diff --git a/.gitignore b/.gitignore index f6a7e551f78..deb88990dd3 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,5 @@ v-env aws_cf_deploy_anything_llm.json yarn.lock *.bak + +/.idea/* diff --git a/collector/processSingleFile/index.js b/collector/processSingleFile/index.js index a00b139ed4b..fc72feb0d11 100644 --- a/collector/processSingleFile/index.js +++ b/collector/processSingleFile/index.js @@ -47,8 +47,9 @@ async function processSingleFile(targetFilename, options = {}) { } let processFileAs = fileExtension; + const fileTypeDefault = options["experimental_file_type_default"]; if (!SUPPORTED_FILETYPE_CONVERTERS.hasOwnProperty(fileExtension)) { - if (isTextType(fullFilePath)) { + if (isTextType(fullFilePath) || fileTypeDefault) { console.log( `\x1b[33m[Collector]\x1b[0m The provided filetype of ${fileExtension} does not have a preset and will be processed as .txt.` ); diff --git a/frontend/src/models/experimental/fileTypeDefault.js b/frontend/src/models/experimental/fileTypeDefault.js new file mode 100644 index 00000000000..d717230cf18 --- /dev/null +++ b/frontend/src/models/experimental/fileTypeDefault.js @@ -0,0 +1,24 @@ +import { API_BASE } from "@/utils/constants"; +import { baseHeaders } from "@/utils/request"; + +const FileTypeDefault = { + featureFlag: "experimental_file_type_default", + toggleFeature: async function (updatedStatus = false) { + return await fetch(`${API_BASE}/experimental/toggle-file-type-default`, { + method: "POST", + headers: baseHeaders(), + body: JSON.stringify({ updatedStatus }), + }) + .then((res) => { + if (!res.ok) throw new Error("Could not update status."); + return true; + }) + .then((res) => res) + .catch((e) => { + console.error(e); + return false; + }); + }, +}; + +export default FileTypeDefault; diff --git a/frontend/src/models/system.js b/frontend/src/models/system.js index 663cb21b76f..383898f5f8e 100644 --- a/frontend/src/models/system.js +++ b/frontend/src/models/system.js @@ -2,6 +2,7 @@ import { API_BASE, AUTH_TIMESTAMP, fullApiUrl } from "@/utils/constants"; import { baseHeaders, safeJsonParse } from "@/utils/request"; import DataConnector from "./dataConnector"; import LiveDocumentSync from "./experimental/liveSync"; +import FileTypeDefault from "./experimental/fileTypeDefault"; import AgentPlugins from "./experimental/agentPlugins"; const System = { @@ -738,6 +739,7 @@ const System = { experimentalFeatures: { liveSync: LiveDocumentSync, + fileTypeDefault: FileTypeDefault, agentPlugins: AgentPlugins, }, }; diff --git a/frontend/src/pages/Admin/ExperimentalFeatures/Features/FileTypeDefault/toggle.jsx b/frontend/src/pages/Admin/ExperimentalFeatures/Features/FileTypeDefault/toggle.jsx new file mode 100644 index 00000000000..359f111fadc --- /dev/null +++ b/frontend/src/pages/Admin/ExperimentalFeatures/Features/FileTypeDefault/toggle.jsx @@ -0,0 +1,57 @@ +import System from "@/models/system"; +import showToast from "@/utils/toast"; +import { useState } from "react"; + +export default function FileTypeDefaultToggle({ enabled = false, onToggle }) { + const [status, setStatus] = useState(enabled); + + async function toggleFeatureFlag() { + const updated = + await System.experimentalFeatures.fileTypeDefault.toggleFeature(!status); + if (!updated) { + showToast("Failed to update status of feature.", "error", { + clear: true, + }); + return false; + } + + setStatus(!status); + showToast( + `File type default has been ${!status ? "enabled" : "disabled"}.`, + "success", + { clear: true } + ); + onToggle(); + } + + return ( +
+
+
+

+ File Type Default +

+ +
+
+

+ If the type of an imported file cannot be determined, this setting + changes the default behavior to process the file as text instead of + displaying an error message. +

+

+ This feature only applies when importing file-based content. +

+
+
+
+ ); +} diff --git a/frontend/src/pages/Admin/ExperimentalFeatures/features.js b/frontend/src/pages/Admin/ExperimentalFeatures/features.js index 7dc8251eb07..9f8118991d3 100644 --- a/frontend/src/pages/Admin/ExperimentalFeatures/features.js +++ b/frontend/src/pages/Admin/ExperimentalFeatures/features.js @@ -1,4 +1,5 @@ import LiveSyncToggle from "./Features/LiveSync/toggle"; +import FileTypeDefaultToggle from "./Features/FileTypeDefault/toggle"; export const configurableFeatures = { experimental_live_file_sync: { @@ -6,4 +7,9 @@ export const configurableFeatures = { component: LiveSyncToggle, key: "experimental_live_file_sync", }, + experimental_file_type_default: { + title: "File Type Default", + component: FileTypeDefaultToggle, + key: "experimental_file_type_default", + }, }; diff --git a/server/endpoints/experimental/fileTypeDefault.js b/server/endpoints/experimental/fileTypeDefault.js new file mode 100644 index 00000000000..5f7b87d6303 --- /dev/null +++ b/server/endpoints/experimental/fileTypeDefault.js @@ -0,0 +1,59 @@ +const { EventLogs } = require("../../models/eventLogs"); +const { SystemSettings } = require("../../models/systemSettings"); +const { Telemetry } = require("../../models/telemetry"); +const { reqBody } = require("../../utils/http"); +const { + flexUserRoleValid, + ROLES, +} = require("../../utils/middleware/multiUserProtected"); +const { validatedRequest } = require("../../utils/middleware/validatedRequest"); + +function fileTypeDefaultEndpoints(app) { + if (!app) return; + + app.post( + "/experimental/toggle-file-type-default", + [validatedRequest, flexUserRoleValid([ROLES.admin])], + async (request, response) => { + try { + const { updatedStatus = false } = reqBody(request); + const newStatus = + SystemSettings.validations.experimental_file_type_default( + updatedStatus + ); + const currentStatus = + ( + await SystemSettings.get({ + label: "experimental_file_type_default", + }) + )?.value || "disabled"; + if (currentStatus === newStatus) + return response + .status(200) + .json({ fileTypeDefaultEnabled: newStatus === "enabled" }); + + // Already validated earlier - so can hot update. + await SystemSettings._updateSettings({ + experimental_file_type_default: newStatus, + }); + if (newStatus === "enabled") { + await Telemetry.sendTelemetry("experimental_feature_enabled", { + feature: "file_type_default", + }); + await EventLogs.logEvent("experimental_feature_enabled", { + feature: "file_type_default", + }); + } + + response + .status(200) + .json({ fileTypeDefaultEnabled: newStatus === "enabled" }); + } catch (e) { + console.error(e); + response.status(500).end(); + } + } + ); +} + +module.exports = { fileTypeDefaultEndpoints }; diff --git a/server/endpoints/experimental/index.js b/server/endpoints/experimental/index.js index f7dc8678a80..f9f83e4b7da 100644 --- a/server/endpoints/experimental/index.js +++ b/server/endpoints/experimental/index.js @@ -1,4 +1,5 @@ const { liveSyncEndpoints } = require("./liveSync"); +const { fileTypeDefaultEndpoints } = require("./fileTypeDefault"); const { importedAgentPluginEndpoints } = require("./imported-agent-plugins"); // All endpoints here are not stable and can move around - have breaking changes @@ -6,6 +7,7 @@ const { importedAgentPluginEndpoints } = require("./imported-agent-plugins"); // When a feature is promoted it should be removed from here and added to the appropriate scope. function experimentalEndpoints(router) { liveSyncEndpoints(router); + fileTypeDefaultEndpoints(router); importedAgentPluginEndpoints(router); } diff --git a/server/models/fileTypeDefault.js b/server/models/fileTypeDefault.js new file mode 100644 index 00000000000..f3b964e8b9f --- /dev/null +++ b/server/models/fileTypeDefault.js @@ -0,0 +1,15 @@ +const { SystemSettings } = require("./systemSettings"); + +const FileTypeDefault = { + featureKey: "experimental_file_type_default", + + /** Check if the fileTypeDefault feature is enabled. */ + enabled: async function () { + return ( + (await SystemSettings.get({ label: this.featureKey }))?.value === + "enabled" + ); + }, +}; + +module.exports = { FileTypeDefault }; diff --git a/server/models/systemSettings.js b/server/models/systemSettings.js index 42ce8723bc7..f3d91c0c2bb 100644 --- a/server/models/systemSettings.js +++ b/server/models/systemSettings.js @@ -51,6 +51,7 @@ const SystemSettings = { // beta feature flags "experimental_live_file_sync", + "experimental_file_type_default", // Hub settings "hub_api_key", @@ -158,6 +159,12 @@ const SystemSettings = { if (!["enabled", "disabled"].includes(update)) return "disabled"; return String(update); }, + experimental_file_type_default: (update) => { + if (typeof update === "boolean") + return update === true ? "enabled" : "disabled"; + if (!["enabled", "disabled"].includes(update)) return "disabled"; + return String(update); + }, meta_page_title: (newTitle) => { try { if (typeof newTitle !== "string" || !newTitle) return null; @@ -579,6 +586,9 @@ const SystemSettings = { experimental_live_file_sync: (await SystemSettings.get({ label: "experimental_live_file_sync" })) ?.value === "enabled", + experimental_file_type_default: + (await SystemSettings.get({ label: "experimental_file_type_default" })) + ?.value === "enabled", }; }, diff --git a/server/utils/collectorApi/index.js b/server/utils/collectorApi/index.js index 7f5781918d8..34dff3ff76e 100644 --- a/server/utils/collectorApi/index.js +++ b/server/utils/collectorApi/index.js @@ -1,4 +1,5 @@ const { EncryptionManager } = require("../EncryptionManager"); +const { FileTypeDefault } = require("../../models/fileTypeDefault"); // When running locally will occupy the 0.0.0.0 hostname space but when deployed inside // of docker this endpoint is not exposed so it is only on the Docker instances internal network @@ -45,9 +46,13 @@ class CollectorApi { async processDocument(filename = "") { if (!filename) return false; + let augmentedOptions = this.#attachOptions(); + augmentedOptions[FileTypeDefault.featureKey] = + await FileTypeDefault.enabled(); + const data = JSON.stringify({ filename, - options: this.#attachOptions(), + options: augmentedOptions, }); return await fetch(`${this.endpoint}/process`, {