From 3bec1c3bb6757a596c725c5d5deb2a149dd9e96e Mon Sep 17 00:00:00 2001
From: timothycarambat <rambat1010@gmail.com>
Date: Sat, 6 Apr 2024 16:10:22 -0700
Subject: [PATCH 1/2] Enable customization of chunk length and overlap

---
 frontend/src/App.jsx                          |   9 +
 .../LLMSelection/LMStudioOptions/index.jsx    |   2 +-
 .../LLMSelection/LocalAiOptions/index.jsx     |   2 +-
 .../src/components/SettingsSidebar/index.jsx  |  18 +-
 .../EmbeddingTextSplitterPreference/index.jsx | 162 ++++++++++++++++++
 .../Steps/LLMPreference/index.jsx             |   2 +-
 .../Steps/VectorDatabaseConnection/index.jsx  |   2 +-
 frontend/src/utils/paths.js                   |   4 +
 server/endpoints/admin.js                     |  14 +-
 server/models/systemSettings.js               |  42 +++++
 .../EmbeddingEngines/azureOpenAi/index.js     |   4 +-
 server/utils/EmbeddingEngines/openAi/index.js |   4 +-
 server/utils/TextSplitter/index.js            |  84 +++++++++
 server/utils/vectorDbProviders/astra/index.js |  17 +-
 .../utils/vectorDbProviders/chroma/index.js   |  17 +-
 server/utils/vectorDbProviders/lance/index.js |  18 +-
 .../utils/vectorDbProviders/milvus/index.js   |  17 +-
 .../utils/vectorDbProviders/pinecone/index.js |  17 +-
 .../utils/vectorDbProviders/qdrant/index.js   |  17 +-
 .../utils/vectorDbProviders/weaviate/index.js |  17 +-
 .../utils/vectorDbProviders/zilliz/index.js   |  17 +-
 21 files changed, 437 insertions(+), 49 deletions(-)
 create mode 100644 frontend/src/pages/GeneralSettings/EmbeddingTextSplitterPreference/index.jsx
 create mode 100644 server/utils/TextSplitter/index.js

diff --git a/frontend/src/App.jsx b/frontend/src/App.jsx
index dbd61623db1..0a5ed65fc85 100644
--- a/frontend/src/App.jsx
+++ b/frontend/src/App.jsx
@@ -35,6 +35,9 @@ const GeneralTranscriptionPreference = lazy(
 const GeneralEmbeddingPreference = lazy(
   () => import("@/pages/GeneralSettings/EmbeddingPreference")
 );
+const EmbeddingTextSplitterPreference = lazy(
+  () => import("@/pages/GeneralSettings/EmbeddingTextSplitterPreference")
+);
 const GeneralVectorDatabase = lazy(
   () => import("@/pages/GeneralSettings/VectorDatabase")
 );
@@ -86,6 +89,12 @@ export default function App() {
                 path="/settings/embedding-preference"
                 element={<AdminRoute Component={GeneralEmbeddingPreference} />}
               />
+              <Route
+                path="/settings/text-splitter-preference"
+                element={
+                  <AdminRoute Component={EmbeddingTextSplitterPreference} />
+                }
+              />
               <Route
                 path="/settings/vector-database"
                 element={<AdminRoute Component={GeneralVectorDatabase} />}
diff --git a/frontend/src/components/LLMSelection/LMStudioOptions/index.jsx b/frontend/src/components/LLMSelection/LMStudioOptions/index.jsx
index c94a99d7871..9a1c59bc73f 100644
--- a/frontend/src/components/LLMSelection/LMStudioOptions/index.jsx
+++ b/frontend/src/components/LLMSelection/LMStudioOptions/index.jsx
@@ -21,7 +21,7 @@ export default function LMStudioOptions({ settings, showAlert = false }) {
             </p>
           </div>
           <a
-            href={paths.settings.embeddingPreference()}
+            href={paths.settings.embedder.modelPreference()}
             className="text-sm md:text-base my-2 underline"
           >
             Manage embedding &rarr;
diff --git a/frontend/src/components/LLMSelection/LocalAiOptions/index.jsx b/frontend/src/components/LLMSelection/LocalAiOptions/index.jsx
index 36b2f2588ec..1304c9e1ba5 100644
--- a/frontend/src/components/LLMSelection/LocalAiOptions/index.jsx
+++ b/frontend/src/components/LLMSelection/LocalAiOptions/index.jsx
@@ -21,7 +21,7 @@ export default function LocalAiOptions({ settings, showAlert = false }) {
             </p>
           </div>
           <a
-            href={paths.settings.embeddingPreference()}
+            href={paths.settings.embedder.modelPreference()}
             className="text-sm md:text-base my-2 underline"
           >
             Manage embedding &rarr;
diff --git a/frontend/src/components/SettingsSidebar/index.jsx b/frontend/src/components/SettingsSidebar/index.jsx
index 40450d4e19a..67797d26619 100644
--- a/frontend/src/components/SettingsSidebar/index.jsx
+++ b/frontend/src/components/SettingsSidebar/index.jsx
@@ -20,6 +20,7 @@ import {
   Barcode,
   ClosedCaptioning,
   EyeSlash,
+  SplitVertical,
 } from "@phosphor-icons/react";
 import useUser from "@/hooks/useUser";
 import { USER_BACKGROUND_COLOR } from "@/utils/constants";
@@ -288,12 +289,25 @@ const SidebarOptions = ({ user = null }) => (
       allowedRole={["admin"]}
     />
     <Option
-      href={paths.settings.embeddingPreference()}
-      btnText="Embedding Model"
+      href={paths.settings.embedder.modelPreference()}
+      childLinks={[paths.settings.embedder.chunkingPreference()]}
+      btnText="Embedder Preferences"
       icon={<FileCode className="h-5 w-5 flex-shrink-0" />}
       user={user}
       flex={true}
       allowedRole={["admin"]}
+      subOptions={
+        <>
+          <Option
+            href={paths.settings.embedder.chunkingPreference()}
+            btnText="Text Splitter & Chunking"
+            icon={<SplitVertical className="h-5 w-5 flex-shrink-0" />}
+            user={user}
+            flex={true}
+            allowedRole={["admin"]}
+          />
+        </>
+      }
     />
     <Option
       href={paths.settings.vectorDatabase()}
diff --git a/frontend/src/pages/GeneralSettings/EmbeddingTextSplitterPreference/index.jsx b/frontend/src/pages/GeneralSettings/EmbeddingTextSplitterPreference/index.jsx
new file mode 100644
index 00000000000..2b123ab1f06
--- /dev/null
+++ b/frontend/src/pages/GeneralSettings/EmbeddingTextSplitterPreference/index.jsx
@@ -0,0 +1,162 @@
+import React, { useEffect, useState } from "react";
+import Sidebar from "@/components/SettingsSidebar";
+import { isMobile } from "react-device-detect";
+import PreLoader from "@/components/Preloader";
+import CTAButton from "@/components/lib/CTAButton";
+import Admin from "@/models/admin";
+import showToast from "@/utils/toast";
+
+function isNullOrNaN(value) {
+  if (value === null) return true;
+  return isNaN(value);
+}
+
+export default function EmbeddingTextSplitterPreference() {
+  const [settings, setSettings] = useState({});
+  const [loading, setLoading] = useState(true);
+  const [saving, setSaving] = useState(false);
+  const [hasChanges, setHasChanges] = useState(false);
+
+  const handleSubmit = async (e) => {
+    e.preventDefault();
+    const form = new FormData(e.target);
+    setSaving(true);
+    await Admin.updateSystemPreferences({
+      text_splitter_chunk_size: isNullOrNaN(
+        form.get("text_splitter_chunk_size")
+      )
+        ? 1000
+        : Number(form.get("text_splitter_chunk_size")),
+      text_splitter_chunk_overlap: isNullOrNaN(
+        form.get("text_splitter_chunk_overlap")
+      )
+        ? 1000
+        : Number(form.get("text_splitter_chunk_overlap")),
+    });
+    setSaving(false);
+    setHasChanges(false);
+    showToast("Text chunking strategy settings saved.", "success");
+  };
+
+  useEffect(() => {
+    async function fetchSettings() {
+      const _settings = (await Admin.systemPreferences())?.settings;
+      setSettings(_settings ?? {});
+      setLoading(false);
+    }
+    fetchSettings();
+  }, []);
+
+  return (
+    <div className="w-screen h-screen overflow-hidden bg-sidebar flex">
+      <Sidebar />
+      {loading ? (
+        <div
+          style={{ height: isMobile ? "100%" : "calc(100% - 32px)" }}
+          className="relative md:ml-[2px] md:mr-[16px] md:my-[16px] md:rounded-[16px] bg-main-gradient w-full h-full overflow-y-scroll"
+        >
+          <div className="w-full h-full flex justify-center items-center">
+            <PreLoader />
+          </div>
+        </div>
+      ) : (
+        <div
+          style={{ height: isMobile ? "100%" : "calc(100% - 32px)" }}
+          className="relative md:ml-[2px] md:mr-[16px] md:my-[16px] md:rounded-[16px] bg-main-gradient w-full h-full overflow-y-scroll"
+        >
+          <form
+            onSubmit={handleSubmit}
+            onChange={() => setHasChanges(true)}
+            className="flex w-full"
+          >
+            <div className="flex flex-col w-full px-1 md:pl-6 md:pr-[50px] md:py-6 py-16">
+              <div className="w-full flex flex-col gap-y-1 pb-4 border-white border-b-2 border-opacity-10">
+                <div className="flex gap-x-4 items-center">
+                  <p className="text-lg leading-6 font-bold text-white">
+                    Text splitting & Chunking Preferences
+                  </p>
+                </div>
+                <p className="text-xs leading-[18px] font-base text-white text-opacity-60">
+                  Sometimes, you may want to change the default way that new
+                  documents are split and chunked before being inserted into
+                  your vector database. <br />
+                  You should only modify this setting if you understand how text
+                  splitting works and it's side effects.
+                </p>
+                <p className="text-xs leading-[18px] font-semibold text-white/80">
+                  Changes here will only apply to{" "}
+                  <i>newly embedded documents</i>, not existing documents.
+                </p>
+              </div>
+              <div className="w-full justify-end flex">
+                {hasChanges && (
+                  <CTAButton className="mt-3 mr-0 -mb-14 z-10">
+                    {saving ? "Saving..." : "Save changes"}
+                  </CTAButton>
+                )}
+              </div>
+
+              <div className="flex flex-col gap-y-4 mt-8">
+                <div className="flex flex-col max-w-[300px]">
+                  <div className="flex flex-col gap-y-2 mb-4">
+                    <label className="text-white text-sm font-semibold block">
+                      Text Chunk Size
+                    </label>
+                    <p className="text-xs text-white/60">
+                      This is the maximum length of characters that can be
+                      present in a single vector.
+                    </p>
+                  </div>
+                  <input
+                    type="number"
+                    name="text_splitter_chunk_size"
+                    min={1}
+                    onWheel={(e) => e?.currentTarget?.blur()}
+                    className="border-none bg-zinc-900 text-white placeholder:text-white/20 text-sm rounded-lg focus:border-white block w-full p-2.5"
+                    placeholder="maximum length of vectorized text"
+                    defaultValue={
+                      isNullOrNaN(settings?.text_splitter_chunk_size)
+                        ? 1000
+                        : Number(settings?.text_splitter_chunk_size)
+                    }
+                    required={true}
+                    autoComplete="off"
+                  />
+                </div>
+              </div>
+
+              <div className="flex flex-col gap-y-4 mt-8">
+                <div className="flex flex-col max-w-[300px]">
+                  <div className="flex flex-col gap-y-2 mb-4">
+                    <label className="text-white text-sm font-semibold block">
+                      Text Chunk Overlap
+                    </label>
+                    <p className="text-xs text-white/60">
+                      This is the maximum overlap of characters that occurs
+                      during chunking between two adjacent text chunks.
+                    </p>
+                  </div>
+                  <input
+                    type="number"
+                    name="text_splitter_chunk_overlap"
+                    min={0}
+                    onWheel={(e) => e?.currentTarget?.blur()}
+                    className="border-none bg-zinc-900 text-white placeholder:text-white/20 text-sm rounded-lg focus:border-white block w-full p-2.5"
+                    placeholder="maximum length of vectorized text"
+                    defaultValue={
+                      isNullOrNaN(settings?.text_splitter_chunk_overlap)
+                        ? 20
+                        : Number(settings?.text_splitter_chunk_overlap)
+                    }
+                    required={true}
+                    autoComplete="off"
+                  />
+                </div>
+              </div>
+            </div>
+          </form>
+        </div>
+      )}
+    </div>
+  );
+}
diff --git a/frontend/src/pages/OnboardingFlow/Steps/LLMPreference/index.jsx b/frontend/src/pages/OnboardingFlow/Steps/LLMPreference/index.jsx
index 33883dc7f0c..0421cfcdfe4 100644
--- a/frontend/src/pages/OnboardingFlow/Steps/LLMPreference/index.jsx
+++ b/frontend/src/pages/OnboardingFlow/Steps/LLMPreference/index.jsx
@@ -190,7 +190,7 @@ export default function LLMPreference({
       showToast(`Failed to save LLM settings: ${error}`, "error");
       return;
     }
-    navigate(paths.onboarding.embeddingPreference());
+    navigate(paths.onboarding.embedder.modelPreference());
   };
 
   useEffect(() => {
diff --git a/frontend/src/pages/OnboardingFlow/Steps/VectorDatabaseConnection/index.jsx b/frontend/src/pages/OnboardingFlow/Steps/VectorDatabaseConnection/index.jsx
index bca7e988792..3b39fa27c6c 100644
--- a/frontend/src/pages/OnboardingFlow/Steps/VectorDatabaseConnection/index.jsx
+++ b/frontend/src/pages/OnboardingFlow/Steps/VectorDatabaseConnection/index.jsx
@@ -118,7 +118,7 @@ export default function VectorDatabaseConnection({
   }
 
   function handleBack() {
-    navigate(paths.onboarding.embeddingPreference());
+    navigate(paths.onboarding.embedder.modelPreference());
   }
 
   const handleSubmit = async (e) => {
diff --git a/frontend/src/utils/paths.js b/frontend/src/utils/paths.js
index af0331a9200..ffbf04c0c73 100644
--- a/frontend/src/utils/paths.js
+++ b/frontend/src/utils/paths.js
@@ -98,6 +98,10 @@ export default {
     transcriptionPreference: () => {
       return "/settings/transcription-preference";
     },
+    embedder: {
+      modelPreference: () => "/settings/embedding-preference",
+      chunkingPreference: () => "/settings/text-splitter-preference",
+    },
     embeddingPreference: () => {
       return "/settings/embedding-preference";
     },
diff --git a/server/endpoints/admin.js b/server/endpoints/admin.js
index 34bd66c3fec..92e6d303c4a 100644
--- a/server/endpoints/admin.js
+++ b/server/endpoints/admin.js
@@ -8,7 +8,10 @@ const { User } = require("../models/user");
 const { DocumentVectors } = require("../models/vectors");
 const { Workspace } = require("../models/workspace");
 const { WorkspaceChats } = require("../models/workspaceChats");
-const { getVectorDbClass } = require("../utils/helpers");
+const {
+  getVectorDbClass,
+  getEmbeddingEngineSelection,
+} = require("../utils/helpers");
 const {
   validRoleSelection,
   canModifyAdmin,
@@ -311,6 +314,7 @@ function adminEndpoints(app) {
     }
   );
 
+  // TODO: Allow specification of which props to get instead of returning all of them all the time.
   app.get(
     "/admin/system-preferences",
     [validatedRequest, flexUserRoleValid([ROLES.admin, ROLES.manager])],
@@ -333,6 +337,14 @@ function adminEndpoints(app) {
           support_email:
             (await SystemSettings.get({ label: "support_email" }))?.value ||
             null,
+          text_splitter_chunk_size:
+            (await SystemSettings.get({ label: "text_splitter_chunk_size" }))
+              ?.value ||
+            getEmbeddingEngineSelection()?.embeddingMaxChunkLength ||
+            null,
+          text_splitter_chunk_overlap:
+            (await SystemSettings.get({ label: "text_splitter_chunk_overlap" }))
+              ?.value || null,
         };
         response.status(200).json({ settings });
       } catch (e) {
diff --git a/server/models/systemSettings.js b/server/models/systemSettings.js
index 080a01f0872..604e43073ff 100644
--- a/server/models/systemSettings.js
+++ b/server/models/systemSettings.js
@@ -5,6 +5,11 @@ process.env.NODE_ENV === "development"
 const { isValidUrl } = require("../utils/http");
 const prisma = require("../utils/prisma");
 
+function isNullOrNaN(value) {
+  if (value === null) return true;
+  return isNaN(value);
+}
+
 const SystemSettings = {
   protectedFields: ["multi_user_mode"],
   supportedFields: [
@@ -15,6 +20,8 @@ const SystemSettings = {
     "telemetry_id",
     "footer_data",
     "support_email",
+    "text_splitter_chunk_size",
+    "text_splitter_chunk_overlap",
   ],
   validations: {
     footer_data: (updates) => {
@@ -28,6 +35,32 @@ const SystemSettings = {
         return JSON.stringify([]);
       }
     },
+    text_splitter_chunk_size: (update) => {
+      try {
+        if (isNullOrNaN(update)) throw new Error("Value is not a number.");
+        if (Number(update) <= 0) throw new Error("Value must be non-zero.");
+        return Number(update);
+      } catch (e) {
+        console.error(
+          `Failed to run validation function on text_splitter_chunk_size`,
+          e.message
+        );
+        return 1000;
+      }
+    },
+    text_splitter_chunk_overlap: (update) => {
+      try {
+        if (isNullOrNaN(update)) throw new Error("Value is not a number");
+        if (Number(update) < 0) throw new Error("Value cannot be less than 0.");
+        return Number(update);
+      } catch (e) {
+        console.error(
+          `Failed to run validation function on text_splitter_chunk_overlap`,
+          e.message
+        );
+        return 20;
+      }
+    },
   },
   currentSettings: async function () {
     const llmProvider = process.env.LLM_PROVIDER;
@@ -84,6 +117,15 @@ const SystemSettings = {
     }
   },
 
+  getValueOrFallback: async function (clause = {}, fallback = null) {
+    try {
+      return (await this.get(clause))?.value ?? fallback;
+    } catch (error) {
+      console.error(error.message);
+      return fallback;
+    }
+  },
+
   where: async function (clause = {}, limit) {
     try {
       const settings = await prisma.system_settings.findMany({
diff --git a/server/utils/EmbeddingEngines/azureOpenAi/index.js b/server/utils/EmbeddingEngines/azureOpenAi/index.js
index 4193e860d1e..1f9362c95d9 100644
--- a/server/utils/EmbeddingEngines/azureOpenAi/index.js
+++ b/server/utils/EmbeddingEngines/azureOpenAi/index.js
@@ -17,7 +17,9 @@ class AzureOpenAiEmbedder {
     // Limit of how many strings we can process in a single pass to stay with resource or network limits
     // https://learn.microsoft.com/en-us/azure/ai-services/openai/faq#i-am-trying-to-use-embeddings-and-received-the-error--invalidrequesterror--too-many-inputs--the-max-number-of-inputs-is-1---how-do-i-fix-this-:~:text=consisting%20of%20up%20to%2016%20inputs%20per%20API%20request
     this.maxConcurrentChunks = 16;
-    this.embeddingMaxChunkLength = 1_000;
+
+    // https://learn.microsoft.com/en-us/answers/questions/1188074/text-embedding-ada-002-token-context-length
+    this.embeddingMaxChunkLength = 2048;
   }
 
   async embedTextInput(textInput) {
diff --git a/server/utils/EmbeddingEngines/openAi/index.js b/server/utils/EmbeddingEngines/openAi/index.js
index b52e78c6f43..49841343a61 100644
--- a/server/utils/EmbeddingEngines/openAi/index.js
+++ b/server/utils/EmbeddingEngines/openAi/index.js
@@ -13,7 +13,9 @@ class OpenAiEmbedder {
 
     // Limit of how many strings we can process in a single pass to stay with resource or network limits
     this.maxConcurrentChunks = 500;
-    this.embeddingMaxChunkLength = 1_000;
+
+    // https://platform.openai.com/docs/guides/embeddings/embedding-models
+    this.embeddingMaxChunkLength = 8_191;
   }
 
   async embedTextInput(textInput) {
diff --git a/server/utils/TextSplitter/index.js b/server/utils/TextSplitter/index.js
new file mode 100644
index 00000000000..d7829827ca0
--- /dev/null
+++ b/server/utils/TextSplitter/index.js
@@ -0,0 +1,84 @@
+function isNullOrNaN(value) {
+  if (value === null) return true;
+  return isNaN(value);
+}
+
+class TextSplitter {
+  #splitter;
+  constructor(config = {}) {
+    /*
+      config can be a ton of things depending on what is required or optional by the specific splitter.
+      Non-splitter related keys
+      {
+        splitByFilename: string, // TODO
+      }
+      ------
+      Default: "RecursiveCharacterTextSplitter"
+      Config: {
+        chunkSize: number,
+        chunkOverlap: number,
+      }
+      ------
+    */
+    this.config = config;
+    this.#splitter = this.#setSplitter(config);
+  }
+
+  log(text, ...args) {
+    console.log(`\x1b[35m[TextSplitter]\x1b[0m ${text}`, ...args);
+  }
+
+  // Does a quick check to determine the text chunk length limit.
+  // Embedder models have hard-set limits that cannot be exceeded, just like an LLM context
+  // so here we want to allow override of the default 1000, but up to the models maximum, which is
+  // sometimes user defined.
+  static determineMaxChunkSize(preferred = null, embedderLimit = 1000) {
+    const prefValue = isNullOrNaN(preferred)
+      ? Number(embedderLimit)
+      : Number(preferred);
+    const limit = Number(embedderLimit);
+    if (prefValue > limit)
+      console.log(
+        `\x1b[43m[WARN]\x1b[0m Text splitter chunk length of ${prefValue} exceeds embedder model max of ${embedderLimit}. Will use ${embedderLimit}.`
+      );
+    return prefValue > limit ? limit : prefValue;
+  }
+
+  #setSplitter(config = {}) {
+    // if (!config?.splitByFilename) {// TODO do something when specific extension is present? }
+    return new RecursiveSplitter({
+      chunkSize: isNaN(config?.chunkSize) ? 1_000 : Number(config?.chunkSize),
+      chunkOverlap: isNaN(config?.chunkOverlap)
+        ? 20
+        : Number(config?.chunkOverlap),
+    });
+  }
+
+  async splitText(documentText) {
+    return this.#splitter._splitText(documentText);
+  }
+}
+
+// Wrapper for Langchain default RecursiveCharacterTextSplitter class.
+class RecursiveSplitter {
+  constructor({ chunkSize, chunkOverlap }) {
+    const {
+      RecursiveCharacterTextSplitter,
+    } = require("langchain/text_splitter");
+    this.log(`Will split with`, { chunkSize, chunkOverlap });
+    this.engine = new RecursiveCharacterTextSplitter({
+      chunkSize,
+      chunkOverlap,
+    });
+  }
+
+  log(text, ...args) {
+    console.log(`\x1b[35m[RecursiveSplitter]\x1b[0m ${text}`, ...args);
+  }
+
+  async _splitText(documentText) {
+    return this.engine.splitText(documentText);
+  }
+}
+
+module.exports.TextSplitter = TextSplitter;
diff --git a/server/utils/vectorDbProviders/astra/index.js b/server/utils/vectorDbProviders/astra/index.js
index df983d4f488..b6f8981bb19 100644
--- a/server/utils/vectorDbProviders/astra/index.js
+++ b/server/utils/vectorDbProviders/astra/index.js
@@ -1,5 +1,5 @@
 const { AstraDB: AstraClient } = require("@datastax/astra-db-ts");
-const { RecursiveCharacterTextSplitter } = require("langchain/text_splitter");
+const { TextSplitter } = require("../../TextSplitter");
 const { storeVectorResult, cachedVectorInformation } = require("../../files");
 const { v4: uuidv4 } = require("uuid");
 const {
@@ -147,10 +147,17 @@ const AstraDB = {
         return { vectorized: true, error: null };
       }
 
-      const textSplitter = new RecursiveCharacterTextSplitter({
-        chunkSize:
-          getEmbeddingEngineSelection()?.embeddingMaxChunkLength || 1_000,
-        chunkOverlap: 20,
+      const textSplitter = new TextSplitter({
+        chunkSize: TextSplitter.determineMaxChunkSize(
+          await SystemSettings.getValueOrFallback({
+            label: "text_splitter_chunk_size",
+          }),
+          getEmbeddingEngineSelection()?.embeddingMaxChunkLength
+        ),
+        chunkOverlap: await SystemSettings.getValueOrFallback(
+          { label: "text_splitter_chunk_overlap" },
+          20
+        ),
       });
       const textChunks = await textSplitter.splitText(pageContent);
 
diff --git a/server/utils/vectorDbProviders/chroma/index.js b/server/utils/vectorDbProviders/chroma/index.js
index 9e3caa7adca..1b9cbb53a4b 100644
--- a/server/utils/vectorDbProviders/chroma/index.js
+++ b/server/utils/vectorDbProviders/chroma/index.js
@@ -1,5 +1,5 @@
 const { ChromaClient } = require("chromadb");
-const { RecursiveCharacterTextSplitter } = require("langchain/text_splitter");
+const { TextSplitter } = require("../../TextSplitter");
 const { storeVectorResult, cachedVectorInformation } = require("../../files");
 const { v4: uuidv4 } = require("uuid");
 const {
@@ -180,10 +180,17 @@ const Chroma = {
       // We have to do this manually as opposed to using LangChains `Chroma.fromDocuments`
       // because we then cannot atomically control our namespace to granularly find/remove documents
       // from vectordb.
-      const textSplitter = new RecursiveCharacterTextSplitter({
-        chunkSize:
-          getEmbeddingEngineSelection()?.embeddingMaxChunkLength || 1_000,
-        chunkOverlap: 20,
+      const textSplitter = new TextSplitter({
+        chunkSize: TextSplitter.determineMaxChunkSize(
+          await SystemSettings.getValueOrFallback({
+            label: "text_splitter_chunk_size",
+          }),
+          getEmbeddingEngineSelection()?.embeddingMaxChunkLength
+        ),
+        chunkOverlap: await SystemSettings.getValueOrFallback(
+          { label: "text_splitter_chunk_overlap" },
+          20
+        ),
       });
       const textChunks = await textSplitter.splitText(pageContent);
 
diff --git a/server/utils/vectorDbProviders/lance/index.js b/server/utils/vectorDbProviders/lance/index.js
index ecf10007f51..f2fc8eee10f 100644
--- a/server/utils/vectorDbProviders/lance/index.js
+++ b/server/utils/vectorDbProviders/lance/index.js
@@ -5,9 +5,10 @@ const {
   getEmbeddingEngineSelection,
 } = require("../../helpers");
 const { OpenAIEmbeddings } = require("langchain/embeddings/openai");
-const { RecursiveCharacterTextSplitter } = require("langchain/text_splitter");
+const { TextSplitter } = require("../../TextSplitter");
 const { storeVectorResult, cachedVectorInformation } = require("../../files");
 const { v4: uuidv4 } = require("uuid");
+const { SystemSettings } = require("../../../models/systemSettings");
 
 const LanceDb = {
   uri: `${
@@ -180,10 +181,17 @@ const LanceDb = {
       // We have to do this manually as opposed to using LangChains `xyz.fromDocuments`
       // because we then cannot atomically control our namespace to granularly find/remove documents
       // from vectordb.
-      const textSplitter = new RecursiveCharacterTextSplitter({
-        chunkSize:
-          getEmbeddingEngineSelection()?.embeddingMaxChunkLength || 1_000,
-        chunkOverlap: 20,
+      const textSplitter = new TextSplitter({
+        chunkSize: TextSplitter.determineMaxChunkSize(
+          await SystemSettings.getValueOrFallback({
+            label: "text_splitter_chunk_size",
+          }),
+          getEmbeddingEngineSelection()?.embeddingMaxChunkLength
+        ),
+        chunkOverlap: await SystemSettings.getValueOrFallback(
+          { label: "text_splitter_chunk_overlap" },
+          20
+        ),
       });
       const textChunks = await textSplitter.splitText(pageContent);
 
diff --git a/server/utils/vectorDbProviders/milvus/index.js b/server/utils/vectorDbProviders/milvus/index.js
index a304e8714e5..3bd5be6d649 100644
--- a/server/utils/vectorDbProviders/milvus/index.js
+++ b/server/utils/vectorDbProviders/milvus/index.js
@@ -4,7 +4,7 @@ const {
   IndexType,
   MilvusClient,
 } = require("@zilliz/milvus2-sdk-node");
-const { RecursiveCharacterTextSplitter } = require("langchain/text_splitter");
+const { TextSplitter } = require("../../TextSplitter");
 const { v4: uuidv4 } = require("uuid");
 const { storeVectorResult, cachedVectorInformation } = require("../../files");
 const {
@@ -182,10 +182,17 @@ const Milvus = {
         return { vectorized: true, error: null };
       }
 
-      const textSplitter = new RecursiveCharacterTextSplitter({
-        chunkSize:
-          getEmbeddingEngineSelection()?.embeddingMaxChunkLength || 1_000,
-        chunkOverlap: 20,
+      const textSplitter = new TextSplitter({
+        chunkSize: TextSplitter.determineMaxChunkSize(
+          await SystemSettings.getValueOrFallback({
+            label: "text_splitter_chunk_size",
+          }),
+          getEmbeddingEngineSelection()?.embeddingMaxChunkLength
+        ),
+        chunkOverlap: await SystemSettings.getValueOrFallback(
+          { label: "text_splitter_chunk_overlap" },
+          20
+        ),
       });
       const textChunks = await textSplitter.splitText(pageContent);
 
diff --git a/server/utils/vectorDbProviders/pinecone/index.js b/server/utils/vectorDbProviders/pinecone/index.js
index b8f288c0619..efcecddc9d4 100644
--- a/server/utils/vectorDbProviders/pinecone/index.js
+++ b/server/utils/vectorDbProviders/pinecone/index.js
@@ -1,5 +1,5 @@
 const { Pinecone } = require("@pinecone-database/pinecone");
-const { RecursiveCharacterTextSplitter } = require("langchain/text_splitter");
+const { TextSplitter } = require("../../TextSplitter");
 const { storeVectorResult, cachedVectorInformation } = require("../../files");
 const { v4: uuidv4 } = require("uuid");
 const {
@@ -125,10 +125,17 @@ const PineconeDB = {
       // because we then cannot atomically control our namespace to granularly find/remove documents
       // from vectordb.
       // https://github.com/hwchase17/langchainjs/blob/2def486af734c0ca87285a48f1a04c057ab74bdf/langchain/src/vectorstores/pinecone.ts#L167
-      const textSplitter = new RecursiveCharacterTextSplitter({
-        chunkSize:
-          getEmbeddingEngineSelection()?.embeddingMaxChunkLength || 1_000,
-        chunkOverlap: 20,
+      const textSplitter = new TextSplitter({
+        chunkSize: TextSplitter.determineMaxChunkSize(
+          await SystemSettings.getValueOrFallback({
+            label: "text_splitter_chunk_size",
+          }),
+          getEmbeddingEngineSelection()?.embeddingMaxChunkLength
+        ),
+        chunkOverlap: await SystemSettings.getValueOrFallback(
+          { label: "text_splitter_chunk_overlap" },
+          20
+        ),
       });
       const textChunks = await textSplitter.splitText(pageContent);
 
diff --git a/server/utils/vectorDbProviders/qdrant/index.js b/server/utils/vectorDbProviders/qdrant/index.js
index e7e00fe64d3..aaca51118f5 100644
--- a/server/utils/vectorDbProviders/qdrant/index.js
+++ b/server/utils/vectorDbProviders/qdrant/index.js
@@ -1,5 +1,5 @@
 const { QdrantClient } = require("@qdrant/js-client-rest");
-const { RecursiveCharacterTextSplitter } = require("langchain/text_splitter");
+const { TextSplitter } = require("../../TextSplitter");
 const { storeVectorResult, cachedVectorInformation } = require("../../files");
 const { v4: uuidv4 } = require("uuid");
 const {
@@ -198,10 +198,17 @@ const QDrant = {
       // We have to do this manually as opposed to using LangChains `Qdrant.fromDocuments`
       // because we then cannot atomically control our namespace to granularly find/remove documents
       // from vectordb.
-      const textSplitter = new RecursiveCharacterTextSplitter({
-        chunkSize:
-          getEmbeddingEngineSelection()?.embeddingMaxChunkLength || 1_000,
-        chunkOverlap: 20,
+      const textSplitter = new TextSplitter({
+        chunkSize: TextSplitter.determineMaxChunkSize(
+          await SystemSettings.getValueOrFallback({
+            label: "text_splitter_chunk_size",
+          }),
+          getEmbeddingEngineSelection()?.embeddingMaxChunkLength
+        ),
+        chunkOverlap: await SystemSettings.getValueOrFallback(
+          { label: "text_splitter_chunk_overlap" },
+          20
+        ),
       });
       const textChunks = await textSplitter.splitText(pageContent);
 
diff --git a/server/utils/vectorDbProviders/weaviate/index.js b/server/utils/vectorDbProviders/weaviate/index.js
index 13668303f9a..35112327b4f 100644
--- a/server/utils/vectorDbProviders/weaviate/index.js
+++ b/server/utils/vectorDbProviders/weaviate/index.js
@@ -1,5 +1,5 @@
 const { default: weaviate } = require("weaviate-ts-client");
-const { RecursiveCharacterTextSplitter } = require("langchain/text_splitter");
+const { TextSplitter } = require("../../TextSplitter");
 const { storeVectorResult, cachedVectorInformation } = require("../../files");
 const { v4: uuidv4 } = require("uuid");
 const {
@@ -241,10 +241,17 @@ const Weaviate = {
       // We have to do this manually as opposed to using LangChains `Chroma.fromDocuments`
       // because we then cannot atomically control our namespace to granularly find/remove documents
       // from vectordb.
-      const textSplitter = new RecursiveCharacterTextSplitter({
-        chunkSize:
-          getEmbeddingEngineSelection()?.embeddingMaxChunkLength || 1_000,
-        chunkOverlap: 20,
+      const textSplitter = new TextSplitter({
+        chunkSize: TextSplitter.determineMaxChunkSize(
+          await SystemSettings.getValueOrFallback({
+            label: "text_splitter_chunk_size",
+          }),
+          getEmbeddingEngineSelection()?.embeddingMaxChunkLength
+        ),
+        chunkOverlap: await SystemSettings.getValueOrFallback(
+          { label: "text_splitter_chunk_overlap" },
+          20
+        ),
       });
       const textChunks = await textSplitter.splitText(pageContent);
 
diff --git a/server/utils/vectorDbProviders/zilliz/index.js b/server/utils/vectorDbProviders/zilliz/index.js
index be0e9e7d40d..72e6829d9a5 100644
--- a/server/utils/vectorDbProviders/zilliz/index.js
+++ b/server/utils/vectorDbProviders/zilliz/index.js
@@ -4,7 +4,7 @@ const {
   IndexType,
   MilvusClient,
 } = require("@zilliz/milvus2-sdk-node");
-const { RecursiveCharacterTextSplitter } = require("langchain/text_splitter");
+const { TextSplitter } = require("../../TextSplitter");
 const { v4: uuidv4 } = require("uuid");
 const { storeVectorResult, cachedVectorInformation } = require("../../files");
 const {
@@ -183,10 +183,17 @@ const Zilliz = {
         return { vectorized: true, error: null };
       }
 
-      const textSplitter = new RecursiveCharacterTextSplitter({
-        chunkSize:
-          getEmbeddingEngineSelection()?.embeddingMaxChunkLength || 1_000,
-        chunkOverlap: 20,
+      const textSplitter = new TextSplitter({
+        chunkSize: TextSplitter.determineMaxChunkSize(
+          await SystemSettings.getValueOrFallback({
+            label: "text_splitter_chunk_size",
+          }),
+          getEmbeddingEngineSelection()?.embeddingMaxChunkLength
+        ),
+        chunkOverlap: await SystemSettings.getValueOrFallback(
+          { label: "text_splitter_chunk_overlap" },
+          20
+        ),
       });
       const textChunks = await textSplitter.splitText(pageContent);
 

From fa458818bca74a568365bee6a7e88db7e2c9b55e Mon Sep 17 00:00:00 2001
From: timothycarambat <rambat1010@gmail.com>
Date: Sat, 6 Apr 2024 16:30:07 -0700
Subject: [PATCH 2/2] fix onboarding link show max limit in UI and prevent
 overlap >= chunk size

---
 .../EmbeddingTextSplitterPreference/index.jsx  | 18 ++++++++++++++++++
 .../Steps/LLMPreference/index.jsx              |  2 +-
 .../Steps/VectorDatabaseConnection/index.jsx   |  2 +-
 server/endpoints/admin.js                      |  2 ++
 4 files changed, 22 insertions(+), 2 deletions(-)

diff --git a/frontend/src/pages/GeneralSettings/EmbeddingTextSplitterPreference/index.jsx b/frontend/src/pages/GeneralSettings/EmbeddingTextSplitterPreference/index.jsx
index 2b123ab1f06..5ee1197f117 100644
--- a/frontend/src/pages/GeneralSettings/EmbeddingTextSplitterPreference/index.jsx
+++ b/frontend/src/pages/GeneralSettings/EmbeddingTextSplitterPreference/index.jsx
@@ -5,6 +5,7 @@ import PreLoader from "@/components/Preloader";
 import CTAButton from "@/components/lib/CTAButton";
 import Admin from "@/models/admin";
 import showToast from "@/utils/toast";
+import { nFormatter, numberWithCommas } from "@/utils/numbers";
 
 function isNullOrNaN(value) {
   if (value === null) return true;
@@ -20,6 +21,18 @@ export default function EmbeddingTextSplitterPreference() {
   const handleSubmit = async (e) => {
     e.preventDefault();
     const form = new FormData(e.target);
+
+    if (
+      Number(form.get("text_splitter_chunk_overlap")) >=
+      Number(form.get("text_splitter_chunk_size"))
+    ) {
+      showToast(
+        "Chunk overlap cannot be larger or equal to chunk size.",
+        "error"
+      );
+      return;
+    }
+
     setSaving(true);
     await Admin.updateSystemPreferences({
       text_splitter_chunk_size: isNullOrNaN(
@@ -111,6 +124,7 @@ export default function EmbeddingTextSplitterPreference() {
                     type="number"
                     name="text_splitter_chunk_size"
                     min={1}
+                    max={settings?.max_embed_chunk_size || 1000}
                     onWheel={(e) => e?.currentTarget?.blur()}
                     className="border-none bg-zinc-900 text-white placeholder:text-white/20 text-sm rounded-lg focus:border-white block w-full p-2.5"
                     placeholder="maximum length of vectorized text"
@@ -122,6 +136,10 @@ export default function EmbeddingTextSplitterPreference() {
                     required={true}
                     autoComplete="off"
                   />
+                  <p className="text-xs text-white/40">
+                    Embed model maximum length is{" "}
+                    {numberWithCommas(settings?.max_embed_chunk_size || 1000)}.
+                  </p>
                 </div>
               </div>
 
diff --git a/frontend/src/pages/OnboardingFlow/Steps/LLMPreference/index.jsx b/frontend/src/pages/OnboardingFlow/Steps/LLMPreference/index.jsx
index 0421cfcdfe4..33883dc7f0c 100644
--- a/frontend/src/pages/OnboardingFlow/Steps/LLMPreference/index.jsx
+++ b/frontend/src/pages/OnboardingFlow/Steps/LLMPreference/index.jsx
@@ -190,7 +190,7 @@ export default function LLMPreference({
       showToast(`Failed to save LLM settings: ${error}`, "error");
       return;
     }
-    navigate(paths.onboarding.embedder.modelPreference());
+    navigate(paths.onboarding.embeddingPreference());
   };
 
   useEffect(() => {
diff --git a/frontend/src/pages/OnboardingFlow/Steps/VectorDatabaseConnection/index.jsx b/frontend/src/pages/OnboardingFlow/Steps/VectorDatabaseConnection/index.jsx
index 3b39fa27c6c..bca7e988792 100644
--- a/frontend/src/pages/OnboardingFlow/Steps/VectorDatabaseConnection/index.jsx
+++ b/frontend/src/pages/OnboardingFlow/Steps/VectorDatabaseConnection/index.jsx
@@ -118,7 +118,7 @@ export default function VectorDatabaseConnection({
   }
 
   function handleBack() {
-    navigate(paths.onboarding.embedder.modelPreference());
+    navigate(paths.onboarding.embeddingPreference());
   }
 
   const handleSubmit = async (e) => {
diff --git a/server/endpoints/admin.js b/server/endpoints/admin.js
index 92e6d303c4a..4bf816a0473 100644
--- a/server/endpoints/admin.js
+++ b/server/endpoints/admin.js
@@ -345,6 +345,8 @@ function adminEndpoints(app) {
           text_splitter_chunk_overlap:
             (await SystemSettings.get({ label: "text_splitter_chunk_overlap" }))
               ?.value || null,
+          max_embed_chunk_size:
+            getEmbeddingEngineSelection()?.embeddingMaxChunkLength || 1000,
         };
         response.status(200).json({ settings });
       } catch (e) {