Mintplex-Labs · wprice-uh · Jan 11, 2025 · Jan 11, 2025 · Jan 11, 2025 · Jan 11, 2025
diff --git a/.gitignore b/.gitignore
@@ -9,3 +9,5 @@ v-env
 aws_cf_deploy_anything_llm.json
 yarn.lock
 *.bak
+
+/.idea/*
diff --git a/collector/processSingleFile/index.js b/collector/processSingleFile/index.js
@@ -47,8 +47,9 @@ async function processSingleFile(targetFilename, options = {}) {
   }
 
   let processFileAs = fileExtension;
+  const fileTypeDefault = options["experimental_file_type_default"];
   if (!SUPPORTED_FILETYPE_CONVERTERS.hasOwnProperty(fileExtension)) {
-    if (isTextType(fullFilePath)) {
+    if (isTextType(fullFilePath) || fileTypeDefault) {
       console.log(
         `\x1b[33m[Collector]\x1b[0m The provided filetype of ${fileExtension} does not have a preset and will be processed as .txt.`
       );

diff --git a/frontend/src/models/experimental/fileTypeDefault.js b/frontend/src/models/experimental/fileTypeDefault.js
@@ -0,0 +1,24 @@
+import { API_BASE } from "@/utils/constants";
+import { baseHeaders } from "@/utils/request";
+
+const FileTypeDefault = {
+  featureFlag: "experimental_file_type_default",
+  toggleFeature: async function (updatedStatus = false) {
+    return await fetch(`${API_BASE}/experimental/toggle-file-type-default`, {
+      method: "POST",
+      headers: baseHeaders(),
+      body: JSON.stringify({ updatedStatus }),
+    })
+      .then((res) => {
+        if (!res.ok) throw new Error("Could not update status.");
+        return true;
+      })
+      .then((res) => res)
+      .catch((e) => {
+        console.error(e);
+        return false;
+      });
+  },
+};
+
+export default FileTypeDefault;
diff --git a/frontend/src/models/system.js b/frontend/src/models/system.js
@@ -2,6 +2,7 @@ import { API_BASE, AUTH_TIMESTAMP, fullApiUrl } from "@/utils/constants";
 import { baseHeaders, safeJsonParse } from "@/utils/request";
 import DataConnector from "./dataConnector";
 import LiveDocumentSync from "./experimental/liveSync";
+import FileTypeDefault from "./experimental/fileTypeDefault";
 import AgentPlugins from "./experimental/agentPlugins";
 
 const System = {
@@ -738,6 +739,7 @@ const System = {
 
   experimentalFeatures: {
     liveSync: LiveDocumentSync,
+    fileTypeDefault: FileTypeDefault,
     agentPlugins: AgentPlugins,
   },
 };

diff --git a/frontend/src/pages/Admin/ExperimentalFeatures/Features/FileTypeDefault/toggle.jsx b/frontend/src/pages/Admin/ExperimentalFeatures/Features/FileTypeDefault/toggle.jsx
@@ -0,0 +1,57 @@
+import System from "@/models/system";
+import showToast from "@/utils/toast";
+import { useState } from "react";
+
+export default function FileTypeDefaultToggle({ enabled = false, onToggle }) {
+  const [status, setStatus] = useState(enabled);
+
+  async function toggleFeatureFlag() {
+    const updated =
+      await System.experimentalFeatures.fileTypeDefault.toggleFeature(!status);
+    if (!updated) {
+      showToast("Failed to update status of feature.", "error", {
+        clear: true,
+      });
+      return false;
+    }
+
+    setStatus(!status);
+    showToast(
+      `File type default has been ${!status ? "enabled" : "disabled"}.`,
+      "success",
+      { clear: true }
+    );
+    onToggle();
+  }
+
+  return (
+    <div className="p-4">
+      <div className="flex flex-col gap-y-6 max-w-[500px]">
+        <div className="flex items-center justify-between">
+          <h2 className="text-theme-text-primary text-md font-bold">
+            File Type Default
+          </h2>
+          <label className="relative inline-flex cursor-pointer items-center">
+            <input
+              type="checkbox"
+              onClick={toggleFeatureFlag}
+              checked={status}
+              className="peer sr-only pointer-events-none"
+            />
+            <div className="peer-disabled:opacity-50 pointer-events-none peer h-6 w-11 rounded-full bg-[#CFCFD0] after:absolute after:left-[2px] after:top-[2px] after:h-5 after:w-5 after:rounded-full after:shadow-xl after:border-none after:bg-white after:box-shadow-md after:transition-all after:content-[''] peer-checked:bg-[#32D583] peer-checked:after:translate-x-full peer-checked:after:border-white peer-focus:outline-none peer-focus:ring-4 peer-focus:ring-transparent"></div>
+          </label>
+        </div>
+        <div className="flex flex-col space-y-4">
+          <p className="text-theme-text-secondary text-sm">
+            If the type of an imported file cannot be determined, this setting
+            changes the default behavior to process the file as text instead of
+            displaying an error message.
+          </p>
+          <p className="text-theme-text-secondary text-xs italic">
+            This feature only applies when importing file-based content.
+          </p>
+        </div>
+      </div>
+    </div>
+  );
+}
diff --git a/frontend/src/pages/Admin/ExperimentalFeatures/features.js b/frontend/src/pages/Admin/ExperimentalFeatures/features.js
@@ -1,9 +1,15 @@
 import LiveSyncToggle from "./Features/LiveSync/toggle";
+import FileTypeDefaultToggle from "./Features/FileTypeDefault/toggle";
 
 export const configurableFeatures = {
   experimental_live_file_sync: {
     title: "Live Document Sync",
     component: LiveSyncToggle,
     key: "experimental_live_file_sync",
   },
+  experimental_file_type_default: {
+    title: "File Type Default",
+    component: FileTypeDefaultToggle,
+    key: "experimental_file_type_default",
+  },
 };
diff --git a/server/endpoints/experimental/fileTypeDefault.js b/server/endpoints/experimental/fileTypeDefault.js
@@ -0,0 +1,59 @@
+const { EventLogs } = require("../../models/eventLogs");
+const { SystemSettings } = require("../../models/systemSettings");
+const { Telemetry } = require("../../models/telemetry");
+const { reqBody } = require("../../utils/http");
+const {
+  flexUserRoleValid,
+  ROLES,
+} = require("../../utils/middleware/multiUserProtected");
+const { validatedRequest } = require("../../utils/middleware/validatedRequest");
+
+function fileTypeDefaultEndpoints(app) {
+  if (!app) return;
+
+  app.post(
+    "/experimental/toggle-file-type-default",
+    [validatedRequest, flexUserRoleValid([ROLES.admin])],
+    async (request, response) => {
+      try {
+        const { updatedStatus = false } = reqBody(request);
+        const newStatus =
+          SystemSettings.validations.experimental_file_type_default(
+            updatedStatus
+          );
+        const currentStatus =
+          (
+            await SystemSettings.get({
+              label: "experimental_file_type_default",
+            })
+          )?.value || "disabled";
+        if (currentStatus === newStatus)
+          return response
+            .status(200)
+            .json({ fileTypeDefaultEnabled: newStatus === "enabled" });
+
+        // Already validated earlier - so can hot update.
+        await SystemSettings._updateSettings({
+          experimental_file_type_default: newStatus,
+        });
+        if (newStatus === "enabled") {
+          await Telemetry.sendTelemetry("experimental_feature_enabled", {
+            feature: "file_type_default",
+          });
+          await EventLogs.logEvent("experimental_feature_enabled", {
+            feature: "file_type_default",
+          });
+        }
+
+        response
+          .status(200)
+          .json({ fileTypeDefaultEnabled: newStatus === "enabled" });
+      } catch (e) {
+        console.error(e);
+        response.status(500).end();
+      }
+    }
+  );
+}
+
+module.exports = { fileTypeDefaultEndpoints };
diff --git a/server/endpoints/experimental/index.js b/server/endpoints/experimental/index.js
@@ -1,11 +1,13 @@
 const { liveSyncEndpoints } = require("./liveSync");
+const { fileTypeDefaultEndpoints } = require("./fileTypeDefault");
 const { importedAgentPluginEndpoints } = require("./imported-agent-plugins");
 
 // All endpoints here are not stable and can move around - have breaking changes
 // or are opt-in features that are not fully released.
 // When a feature is promoted it should be removed from here and added to the appropriate scope.
 function experimentalEndpoints(router) {
   liveSyncEndpoints(router);
+  fileTypeDefaultEndpoints(router);
   importedAgentPluginEndpoints(router);
 }
 

diff --git a/server/models/fileTypeDefault.js b/server/models/fileTypeDefault.js
@@ -0,0 +1,15 @@
+const { SystemSettings } = require("./systemSettings");
+
+const FileTypeDefault = {
+  featureKey: "experimental_file_type_default",
+
+  /** Check if the fileTypeDefault feature is enabled. */
+  enabled: async function () {
+    return (
+      (await SystemSettings.get({ label: this.featureKey }))?.value ===
+      "enabled"
+    );
+  },
+};
+
+module.exports = { FileTypeDefault };
diff --git a/server/models/systemSettings.js b/server/models/systemSettings.js
@@ -51,6 +51,7 @@ const SystemSettings = {
 
     // beta feature flags
     "experimental_live_file_sync",
+    "experimental_file_type_default",
 
     // Hub settings
     "hub_api_key",
@@ -158,6 +159,12 @@ const SystemSettings = {
       if (!["enabled", "disabled"].includes(update)) return "disabled";
       return String(update);
     },
+    experimental_file_type_default: (update) => {
+      if (typeof update === "boolean")
+        return update === true ? "enabled" : "disabled";
+      if (!["enabled", "disabled"].includes(update)) return "disabled";
+      return String(update);
+    },
     meta_page_title: (newTitle) => {
       try {
         if (typeof newTitle !== "string" || !newTitle) return null;
@@ -579,6 +586,9 @@ const SystemSettings = {
       experimental_live_file_sync:
         (await SystemSettings.get({ label: "experimental_live_file_sync" }))
           ?.value === "enabled",
+      experimental_file_type_default:
+        (await SystemSettings.get({ label: "experimental_file_type_default" }))
+          ?.value === "enabled",
     };
   },
 

diff --git a/server/utils/collectorApi/index.js b/server/utils/collectorApi/index.js
@@ -1,4 +1,5 @@
 const { EncryptionManager } = require("../EncryptionManager");
+const { FileTypeDefault } = require("../../models/fileTypeDefault");
 
 // When running locally will occupy the 0.0.0.0 hostname space but when deployed inside
 // of docker this endpoint is not exposed so it is only on the Docker instances internal network
@@ -45,9 +46,13 @@ class CollectorApi {
   async processDocument(filename = "") {
     if (!filename) return false;
 
+    let augmentedOptions = this.#attachOptions();
+    augmentedOptions[FileTypeDefault.featureKey] =
+      await FileTypeDefault.enabled();
+
     const data = JSON.stringify({
       filename,
-      options: this.#attachOptions(),
+      options: augmentedOptions,
     });
 
     return await fetch(`${this.endpoint}/process`, {