这是indexloc提供的服务,不要输入任何密码
Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 0 additions & 3 deletions frontend/src/locales/ar/common.js
Original file line number Diff line number Diff line change
Expand Up @@ -416,9 +416,6 @@ const TRANSLATIONS = {
"في بعض الأحيان، قد ترغب في تغيير الطريقة الافتراضية التي يتم بها تقسيم المستندات الجديدة وتقطيعها قبل إدراجها في قاعدة بيانات المتجهة الخاصة بك.",
"desc-end":
"يجب عليك فقط تعديل هذا الإعداد إذا كنت تفهم كيفية عمل تقسيم النص وتأثيراته الجانبية.",
"warn-start": "التغييرات هنا سوف تنطبق فقط على",
"warn-center": "المستندات المضمنة حديثًا",
"warn-end": "، وليس على المستندات الموجودة.",
size: {
title: "حجم قطعة النص",
description:
Expand Down
3 changes: 0 additions & 3 deletions frontend/src/locales/da/common.js
Original file line number Diff line number Diff line change
Expand Up @@ -418,9 +418,6 @@ const TRANSLATIONS = {
"Nogle gange vil du måske ændre den standardmåde, som nye dokumenter deles og opdeles i bidder, inden de indsættes i din vektordatabase.",
"desc-end":
"Du bør kun ændre denne indstilling, hvis du forstår, hvordan tekstopdeling fungerer og dens bivirkninger.",
"warn-start": "Ændringer her vil kun gælde for",
"warn-center": "nyligt indlejrede dokumenter",
"warn-end": ", ikke eksisterende dokumenter.",
size: {
title: "Størrelse på tekstbidder",
description:
Expand Down
3 changes: 0 additions & 3 deletions frontend/src/locales/de/common.js
Original file line number Diff line number Diff line change
Expand Up @@ -607,9 +607,6 @@ const TRANSLATIONS = {
"Manchmal möchten Sie vielleicht die Standardmethode ändern, wie neue Dokumente gesplittet und gechunkt werden, bevor sie in Ihre Vektordatenbank eingefügt werden.",
"desc-end":
"Sie sollten diese Einstellung nur ändern, wenn Sie verstehen, wie Textsplitting funktioniert und welche Nebenwirkungen es hat.",
"warn-start": "Änderungen hier gelten nur für",
"warn-center": "neu eingebettete Dokumente",
"warn-end": ", nicht für bestehende Dokumente.",
size: {
title: "Textchunk-Größe",
description:
Expand Down
3 changes: 0 additions & 3 deletions frontend/src/locales/en/common.js
Original file line number Diff line number Diff line change
Expand Up @@ -631,9 +631,6 @@ const TRANSLATIONS = {
"Sometimes, you may want to change the default way that new documents are split and chunked before being inserted into your vector database.",
"desc-end":
"You should only modify this setting if you understand how text splitting works and it's side effects.",
"warn-start": "Changes here will only apply to",
"warn-center": "newly embedded documents",
"warn-end": ", not existing documents.",
size: {
title: "Text Chunk Size",
description:
Expand Down
3 changes: 0 additions & 3 deletions frontend/src/locales/es/common.js
Original file line number Diff line number Diff line change
Expand Up @@ -415,9 +415,6 @@ const TRANSLATIONS = {
"A veces, es posible que desees cambiar la forma predeterminada en que los nuevos documentos se dividen y fragmentan antes de ser insertados en tu base de datos de vectores.",
"desc-end":
"Solo debes modificar esta configuración si entiendes cómo funciona la división de texto y sus efectos secundarios.",
"warn-start": "Los cambios aquí solo se aplicarán a",
"warn-center": "documentos recién incrustados",
"warn-end": ", no a los documentos existentes.",
size: {
title: "Tamaño del fragmento de texto",
description:
Expand Down
3 changes: 0 additions & 3 deletions frontend/src/locales/et/common.js
Original file line number Diff line number Diff line change
Expand Up @@ -582,9 +582,6 @@ const TRANSLATIONS = {
"desc-start":
"Vahel soovid muuta, kuidas uued dokumendid enne vektoriandmebaasi lisamist tükeldatakse.",
"desc-end": "Muuda seda ainult siis, kui mõistad tekstilõike mõju.",
"warn-start": "Muudatused kehtivad ainult",
"warn-center": "uutele dokumentidele",
"warn-end": ", mitte olemasolevatele.",
size: {
title: "Tekstitüki suurus",
description: "Maksimaalne märgipikkus ühes vektoris.",
Expand Down
3 changes: 0 additions & 3 deletions frontend/src/locales/fa/common.js
Original file line number Diff line number Diff line change
Expand Up @@ -408,9 +408,6 @@ const TRANSLATIONS = {
"تقسیم متن به شما امکان می‌دهد اسناد بزرگ را به بخش‌های کوچک‌تر تقسیم کنید که برای جاسازی و پردازش مناسب‌تر هستند.",
"desc-end":
"سعی کنید تعادلی بین اندازه بخش و همپوشانی ایجاد کنید تا از دست رفتن اطلاعات را به حداقل برسانید.",
"warn-start": "تغییر این مقادیر نیاز به",
"warn-center": "پردازش مجدد تمام اسناد",
"warn-end": "خواهد داشت.",
size: {
title: "حداکثر اندازه بخش",
description:
Expand Down
3 changes: 0 additions & 3 deletions frontend/src/locales/fr/common.js
Original file line number Diff line number Diff line change
Expand Up @@ -416,9 +416,6 @@ const TRANSLATIONS = {
"Parfois, vous voudrez peut-être changer la façon dont les nouveaux documents sont divisés et découpés avant d'être insérés dans votre base de données vectorielle.",
"desc-end":
"Vous ne devez modifier ce paramètre que si vous comprenez comment fonctionne la division du texte et ses effets secondaires.",
"warn-start": "Les changements ici s'appliqueront uniquement aux",
"warn-center": "nouveaux documents intégrés",
"warn-end": ", pas aux documents existants.",
size: {
title: "Taille des segments de texte",
description:
Expand Down
3 changes: 0 additions & 3 deletions frontend/src/locales/he/common.js
Original file line number Diff line number Diff line change
Expand Up @@ -404,9 +404,6 @@ const TRANSLATIONS = {
"לפעמים, ייתכן שתרצה לשנות את הדרך ברירת המחדל שבה מסמכים חדשים מחולקים ומופרדים לפני שהם מוכנסים לבסיס הנתונים הווקטורי שלך.",
"desc-end":
"עליך לשנות הגדרה זו רק אם אתה מבין כיצד חלוקת טקסט פועלת והשפעותיה.",
"warn-start": "שינויים כאן יחולו רק על",
"warn-center": "מסמכים שהוטמעו לאחרונה",
"warn-end": ", לא על מסמכים קיימים.",
size: {
title: "גודל קטע טקסט",
description: "זהו אורך הדמויות המרבי שיכול להיות נוכח בקטור יחיד.",
Expand Down
3 changes: 0 additions & 3 deletions frontend/src/locales/it/common.js
Original file line number Diff line number Diff line change
Expand Up @@ -414,9 +414,6 @@ const TRANSLATIONS = {
"A volte, potresti voler cambiare il modo predefinito in cui i nuovi documenti vengono suddivisi e spezzettati in blocchi prima di essere inseriti nel tuo database vettoriale.",
"desc-end":
"Dovresti modificare questa impostazione solo se capisci come funziona la suddivisione del testo e i suoi effetti collaterali.",
"warn-start": "Le modifiche qui si applicheranno solo a",
"warn-center": "nuovi documenti incorporati",
"warn-end": ", non documenti esistenti.",
size: {
title: "Dimensioni blocco di testo",
description:
Expand Down
3 changes: 0 additions & 3 deletions frontend/src/locales/ja/common.js
Original file line number Diff line number Diff line change
Expand Up @@ -416,9 +416,6 @@ const TRANSLATIONS = {
"新しいドキュメントがベクトルデータベースに挿入される前に、どのように分割およびチャンク化されるかのデフォルトの方法を変更する場合があります。",
"desc-end":
"テキスト分割の仕組みとその副作用を理解している場合にのみ、この設定を変更するべきです。",
"warn-start": "ここでの変更は、",
"warn-center": "新しく埋め込まれるドキュメントにのみ適用されます",
"warn-end": "、既存のドキュメントには適用されません。",
size: {
title: "テキストチャンクサイズ",
description: "1つのベクトルに含まれる最大の文字数です。",
Expand Down
3 changes: 0 additions & 3 deletions frontend/src/locales/ko/common.js
Original file line number Diff line number Diff line change
Expand Up @@ -591,9 +591,6 @@ const TRANSLATIONS = {
"새 문서를 벡터 데이터베이스에 삽입하기 전에 기본 텍스트 분할 방식을 변경할 수 있습니다.",
"desc-end":
"텍스트 분할 방식과 그 영향을 이해하고 있는 경우에만 이 설정을 변경해야 합니다.",
"warn-start": "여기의 변경 사항은",
"warn-center": "새로 임베딩되는 문서",
"warn-end": "에만 적용됩니다. 기존 문서에는 적용되지 않습니다.",
size: {
title: "텍스트 청크 크기",
description: "단일 벡터에 들어갈 수 있는 최대 문자 길이입니다.",
Expand Down
3 changes: 0 additions & 3 deletions frontend/src/locales/lv/common.js
Original file line number Diff line number Diff line change
Expand Up @@ -602,9 +602,6 @@ const TRANSLATIONS = {
"Dažreiz jūs, iespējams, vēlēsieties mainīt noklusējuma veidu, kā jauni dokumenti tiek sadalīti un sagatavoti pirms ievietošanas vektoru datubāzē.",
"desc-end":
"Jums vajadzētu mainīt šo iestatījumu tikai tad, ja saprotat, kā darbojas teksta sadalīšana un tās blakusefekti.",
"warn-start": "Šeit veiktās izmaiņas attieksies tikai uz",
"warn-center": "jauniem iegultajiem dokumentiem",
"warn-end": ", nevis esošajiem dokumentiem.",
size: {
title: "Teksta gabala izmērs",
description:
Expand Down
3 changes: 0 additions & 3 deletions frontend/src/locales/nl/common.js
Original file line number Diff line number Diff line change
Expand Up @@ -411,9 +411,6 @@ const TRANSLATIONS = {
"Soms wil je misschien de standaard manier wijzigen waarop nieuwe documenten worden gesplitst en gechunkt voordat ze in je vector database worden ingevoerd.",
"desc-end":
"Je moet deze instelling alleen wijzigen als je begrijpt hoe tekstsplitsing werkt en de bijbehorende effecten.",
"warn-start": "Wijzigingen hier zijn alleen van toepassing op",
"warn-center": "nieuw ingebedde documenten",
"warn-end": ", niet op bestaande documenten.",
size: {
title: "Tekst Chunk Grootte",
description:
Expand Down
3 changes: 0 additions & 3 deletions frontend/src/locales/pl/common.js
Original file line number Diff line number Diff line change
Expand Up @@ -607,9 +607,6 @@ const TRANSLATIONS = {
"Czasami może zaistnieć potrzeba zmiany domyślnego sposobu, w jaki nowe dokumenty są dzielone i fragmentowane przed wstawieniem ich do wektorowej bazy danych.",
"desc-end":
"Powinieneś modyfikować to ustawienie tylko wtedy, gdy rozumiesz, jak działa dzielenie tekstu i jakie są jego skutki uboczne.",
"warn-start": "Zmiany tutaj będą miały zastosowanie tylko do",
"warn-center": "nowo osadzone dokumenty",
"warn-end": "a nie istniejące dokumenty.",
size: {
title: "Rozmiar fragmentu tekstu",
description:
Expand Down
3 changes: 0 additions & 3 deletions frontend/src/locales/pt_BR/common.js
Original file line number Diff line number Diff line change
Expand Up @@ -588,9 +588,6 @@ const TRANSLATIONS = {
"desc-start":
"Você pode alterar a forma como novos documentos são divididos antes de serem inseridos no banco de dados vetorial.",
"desc-end": "Modifique apenas se entender os efeitos da divisão de texto.",
"warn-start": "Alterações afetarão apenas",
"warn-center": "documentos novos",
"warn-end": ", não os existentes.",
size: {
title: "Tamanho dos Trechos",
description: "Comprimento máximo de caracteres em um único vetor.",
Expand Down
3 changes: 0 additions & 3 deletions frontend/src/locales/ru/common.js
Original file line number Diff line number Diff line change
Expand Up @@ -420,9 +420,6 @@ const TRANSLATIONS = {
"Иногда может понадобиться изменить стандартный способ разделения и сегментации новых документов перед их вставкой в векторную базу данных.",
"desc-end":
"Следует изменять этот параметр только при полном понимании работы разделения текста и его побочных эффектов.",
"warn-start": "Изменения здесь будут применяться только к",
"warn-center": "новым встроенным документам",
"warn-end": ", а не к существующим документам.",
size: {
title: "Размер сегмента текста",
description:
Expand Down
3 changes: 0 additions & 3 deletions frontend/src/locales/tr/common.js
Original file line number Diff line number Diff line change
Expand Up @@ -411,9 +411,6 @@ const TRANSLATIONS = {
"Bazı durumlarda, yeni belgelerin vektör veritabanınıza eklenmeden önce hangi varsayılan yöntemle bölünüp parçalanacağını değiştirmek isteyebilirsiniz.",
"desc-end":
"Metin bölmenin nasıl çalıştığını ve olası yan etkilerini tam olarak bilmiyorsanız bu ayarı değiştirmemelisiniz.",
"warn-start": "Buradaki değişiklikler yalnızca",
"warn-center": "yeni eklenen belgeler",
"warn-end": "için geçerli olacak, mevcut belgelere uygulanmaz.",
size: {
title: "Metin Parça Boyutu",
description:
Expand Down
3 changes: 0 additions & 3 deletions frontend/src/locales/vn/common.js
Original file line number Diff line number Diff line change
Expand Up @@ -410,9 +410,6 @@ const TRANSLATIONS = {
"Sometimes, you may want to change the default way that new documents are split and chunked before being inserted into your vector database.",
"desc-end":
"You should only modify this setting if you understand how text splitting works and it's side effects.",
"warn-start": "Changes here will only apply to",
"warn-center": "newly embedded documents",
"warn-end": ", not existing documents.",
size: {
title: "Text Chunk Size",
description:
Expand Down
3 changes: 0 additions & 3 deletions frontend/src/locales/zh/common.js
Original file line number Diff line number Diff line change
Expand Up @@ -565,9 +565,6 @@ const TRANSLATIONS = {
"desc-start":
"有时,你可能希望更改新文档在插入到矢量数据库之前拆分和分块的默认方式。",
"desc-end": "只有在了解文本拆分的工作原理及其副作用时,才应修改此设置。",
"warn-start": "此处的更改仅适用于",
"warn-center": "新嵌入的文档",
"warn-end": ",而不是现有文档。",
size: {
title: "文本块大小",
description: "这是单个向量中可以存在的字符的最大长度。",
Expand Down
3 changes: 0 additions & 3 deletions frontend/src/locales/zh_TW/common.js
Original file line number Diff line number Diff line change
Expand Up @@ -398,9 +398,6 @@ const TRANSLATIONS = {
"有時您可能需要修改新文件在插入向量資料庫之前的預設分割和區塊化方式。",
"desc-end":
"只有在了解文字分割的運作方式及其副作用的情況下,才應該修改此設定。",
"warn-start": "此處的修改只會套用至",
"warn-center": "新嵌入的文件",
"warn-end": ",而不會套用至現有文件。",
size: {
title: "文字區塊大小",
description: "這是單一向量中可包含的最大字元長度。",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,11 @@ import PreLoader from "@/components/Preloader";
import CTAButton from "@/components/lib/CTAButton";
import Admin from "@/models/admin";
import showToast from "@/utils/toast";
import { nFormatter, numberWithCommas } from "@/utils/numbers";
import { numberWithCommas } from "@/utils/numbers";
import { useTranslation } from "react-i18next";
import { useModal } from "@/hooks/useModal";
import ModalWrapper from "@/components/ModalWrapper";
import ChangeWarningModal from "@/components/ChangeWarning";

function isNullOrNaN(value) {
if (value === null) return true;
Expand All @@ -18,6 +21,7 @@ export default function EmbeddingTextSplitterPreference() {
const [loading, setLoading] = useState(true);
const [saving, setSaving] = useState(false);
const [hasChanges, setHasChanges] = useState(false);
const { isOpen, openModal, closeModal } = useModal();
const { t } = useTranslation();

const handleSubmit = async (e) => {
Expand All @@ -35,22 +39,35 @@ export default function EmbeddingTextSplitterPreference() {
return;
}

openModal();
};

const handleSaveSettings = async () => {
setSaving(true);
await Admin.updateSystemPreferences({
text_splitter_chunk_size: isNullOrNaN(
form.get("text_splitter_chunk_size")
)
? 1000
: Number(form.get("text_splitter_chunk_size")),
text_splitter_chunk_overlap: isNullOrNaN(
form.get("text_splitter_chunk_overlap")
)
? 1000
: Number(form.get("text_splitter_chunk_overlap")),
});
setSaving(false);
setHasChanges(false);
showToast("Text chunking strategy settings saved.", "success");
try {
const form = new FormData(
document.getElementById("text-splitter-chunking-form")
);
await Admin.updateSystemPreferences({
text_splitter_chunk_size: isNullOrNaN(
form.get("text_splitter_chunk_size")
)
? 1000
: Number(form.get("text_splitter_chunk_size")),
text_splitter_chunk_overlap: isNullOrNaN(
form.get("text_splitter_chunk_overlap")
)
? 1000
: Number(form.get("text_splitter_chunk_overlap")),
});
setHasChanges(false);
closeModal();
showToast("Text chunking strategy settings saved.", "success");
} catch (error) {
showToast("Failed to save text chunking strategy settings.", "error");
} finally {
setSaving(false);
}
};

useEffect(() => {
Expand Down Expand Up @@ -83,6 +100,7 @@ export default function EmbeddingTextSplitterPreference() {
onSubmit={handleSubmit}
onChange={() => setHasChanges(true)}
className="flex w-full"
id="text-splitter-chunking-form"
>
<div className="flex flex-col w-full px-1 md:pl-6 md:pr-[50px] md:py-6 py-16">
<div className="w-full flex flex-col gap-y-1 pb-4 border-white light:border-theme-sidebar-border border-b-2 border-opacity-10">
Expand All @@ -95,10 +113,6 @@ export default function EmbeddingTextSplitterPreference() {
{t("text.desc-start")} <br />
{t("text.desc-end")}
</p>
<p className="text-xs leading-[18px] font-semibold text-white/80">
{t("text.warn-start")} <i>{t("text.warn-center")}</i>
{t("text.warn-end")}
</p>
</div>
<div className="w-full justify-end flex">
{hasChanges && (
Expand Down Expand Up @@ -172,6 +186,14 @@ export default function EmbeddingTextSplitterPreference() {
</form>
</div>
)}

<ModalWrapper isOpen={isOpen}>
<ChangeWarningModal
warningText="Changing text splitter settings will clear any previously cached documents.\n\nThese new settings will be applied to all documents when embedding them into a workspace."
onClose={closeModal}
onConfirm={handleSaveSettings}
/>
</ModalWrapper>
</div>
);
}
4 changes: 4 additions & 0 deletions server/models/systemSettings.js
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,8 @@ const SystemSettings = {
try {
if (isNullOrNaN(update)) throw new Error("Value is not a number.");
if (Number(update) <= 0) throw new Error("Value must be non-zero.");
const { purgeEntireVectorCache } = require("../utils/files");
purgeEntireVectorCache();
return Number(update);
} catch (e) {
console.error(
Expand All @@ -86,6 +88,8 @@ const SystemSettings = {
try {
if (isNullOrNaN(update)) throw new Error("Value is not a number");
if (Number(update) < 0) throw new Error("Value cannot be less than 0.");
const { purgeEntireVectorCache } = require("../utils/files");
purgeEntireVectorCache();
return Number(update);
} catch (e) {
console.error(
Expand Down