From e720a64a2c45528bea63af5b42677b77a70441c8 Mon Sep 17 00:00:00 2001 From: Westopheles Date: Fri, 10 Jan 2025 16:51:52 -1000 Subject: [PATCH 1/6] added ignore for .idea IDE files --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index f6a7e551f78..deb88990dd3 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,5 @@ v-env aws_cf_deploy_anything_llm.json yarn.lock *.bak + +/.idea/* From 1b26858e81269689848b9ac4bd29fc6d7d714614 Mon Sep 17 00:00:00 2001 From: Westopheles Date: Fri, 10 Jan 2025 21:13:21 -1000 Subject: [PATCH 2/6] checkpoint: settings gui works --- frontend/src/App.jsx | 8 ++ .../models/experimental/fileTypeDefault.js | 24 ++++++ frontend/src/models/system.js | 2 + .../Features/FileTypeDefault/manage/index.jsx | 33 ++++++++ .../Features/FileTypeDefault/toggle.jsx | 76 +++++++++++++++++++ .../Admin/ExperimentalFeatures/features.js | 6 ++ frontend/src/utils/paths.js | 3 + .../endpoints/experimental/fileTypeDefault.js | 58 ++++++++++++++ server/endpoints/experimental/index.js | 2 + server/models/fileTypeDefault.js | 15 ++++ server/models/systemSettings.js | 10 +++ 11 files changed, 237 insertions(+) create mode 100644 frontend/src/models/experimental/fileTypeDefault.js create mode 100644 frontend/src/pages/Admin/ExperimentalFeatures/Features/FileTypeDefault/manage/index.jsx create mode 100644 frontend/src/pages/Admin/ExperimentalFeatures/Features/FileTypeDefault/toggle.jsx create mode 100644 server/endpoints/experimental/fileTypeDefault.js create mode 100644 server/models/fileTypeDefault.js diff --git a/frontend/src/App.jsx b/frontend/src/App.jsx index af84603cb8e..ecdb9b726f7 100644 --- a/frontend/src/App.jsx +++ b/frontend/src/App.jsx @@ -67,6 +67,9 @@ const ExperimentalFeatures = lazy( const LiveDocumentSyncManage = lazy( () => import("@/pages/Admin/ExperimentalFeatures/Features/LiveSync/manage") ); +const FileTypeDefaultManage = lazy( + () => import("@/pages/Admin/ExperimentalFeatures/Features/FileTypeDefault/manage") +); const CommunityHubTrending = lazy( () => import("@/pages/GeneralSettings/CommunityHub/Trending") @@ -211,6 +214,11 @@ export default function App() { path="/settings/beta-features/live-document-sync/manage" element={} /> + {/* File Type Default feature */} + } + /> { + if (!res.ok) throw new Error("Could not update status."); + return true; + }) + .then((res) => res) + .catch((e) => { + console.error(e); + return false; + }); + }, +}; + +export default FileTypeDefault; diff --git a/frontend/src/models/system.js b/frontend/src/models/system.js index 663cb21b76f..383898f5f8e 100644 --- a/frontend/src/models/system.js +++ b/frontend/src/models/system.js @@ -2,6 +2,7 @@ import { API_BASE, AUTH_TIMESTAMP, fullApiUrl } from "@/utils/constants"; import { baseHeaders, safeJsonParse } from "@/utils/request"; import DataConnector from "./dataConnector"; import LiveDocumentSync from "./experimental/liveSync"; +import FileTypeDefault from "./experimental/fileTypeDefault"; import AgentPlugins from "./experimental/agentPlugins"; const System = { @@ -738,6 +739,7 @@ const System = { experimentalFeatures: { liveSync: LiveDocumentSync, + fileTypeDefault: FileTypeDefault, agentPlugins: AgentPlugins, }, }; diff --git a/frontend/src/pages/Admin/ExperimentalFeatures/Features/FileTypeDefault/manage/index.jsx b/frontend/src/pages/Admin/ExperimentalFeatures/Features/FileTypeDefault/manage/index.jsx new file mode 100644 index 00000000000..a479c3d2569 --- /dev/null +++ b/frontend/src/pages/Admin/ExperimentalFeatures/Features/FileTypeDefault/manage/index.jsx @@ -0,0 +1,33 @@ +import { useEffect, useState } from "react"; +import Sidebar from "@/components/Sidebar"; +import { isMobile } from "react-device-detect"; +import * as Skeleton from "react-loading-skeleton"; +import "react-loading-skeleton/dist/skeleton.css"; +import System from "@/models/system"; +import KnownFileTypeRow from "./KnownFileTypeRow"; + +export default function FileTypeDefaultManager() { + return ( +
+ +
+
+
+
+

+ Known file types +

+
+

+ Nothing to see here, please move along. +

+
+
+
+
+ ); +} + diff --git a/frontend/src/pages/Admin/ExperimentalFeatures/Features/FileTypeDefault/toggle.jsx b/frontend/src/pages/Admin/ExperimentalFeatures/Features/FileTypeDefault/toggle.jsx new file mode 100644 index 00000000000..6d7a0be9ae4 --- /dev/null +++ b/frontend/src/pages/Admin/ExperimentalFeatures/Features/FileTypeDefault/toggle.jsx @@ -0,0 +1,76 @@ +import System from "@/models/system"; +import paths from "@/utils/paths"; +import showToast from "@/utils/toast"; +import { ArrowSquareOut } from "@phosphor-icons/react"; +import { useState } from "react"; +import { Link } from "react-router-dom"; + +export default function FileTypeDefaultToggle({ enabled = false, onToggle }) { + const [status, setStatus] = useState(enabled); + + async function toggleFeatureFlag() { + const updated = + await System.experimentalFeatures.fileTypeDefault.toggleFeature(!status); + if (!updated) { + showToast("Failed to update status of feature.", "error", { + clear: true, + }); + return false; + } + + setStatus(!status); + showToast( + `File type default has been ${ + !status ? "enabled" : "disabled" + }.`, + "success", + { clear: true } + ); + onToggle(); + } + + return ( +
+
+
+

+ File Type Default +

+ +
+
+

+ If the type of an imported file cannot be determined, this + setting changes the default behavior to process the file as + text instead of displaying an error message. +

+

+ This feature only applies when importing file-based content. +

+
+
+ +
+ ); +} diff --git a/frontend/src/pages/Admin/ExperimentalFeatures/features.js b/frontend/src/pages/Admin/ExperimentalFeatures/features.js index 7dc8251eb07..9f8118991d3 100644 --- a/frontend/src/pages/Admin/ExperimentalFeatures/features.js +++ b/frontend/src/pages/Admin/ExperimentalFeatures/features.js @@ -1,4 +1,5 @@ import LiveSyncToggle from "./Features/LiveSync/toggle"; +import FileTypeDefaultToggle from "./Features/FileTypeDefault/toggle"; export const configurableFeatures = { experimental_live_file_sync: { @@ -6,4 +7,9 @@ export const configurableFeatures = { component: LiveSyncToggle, key: "experimental_live_file_sync", }, + experimental_file_type_default: { + title: "File Type Default", + component: FileTypeDefaultToggle, + key: "experimental_file_type_default", + }, }; diff --git a/frontend/src/utils/paths.js b/frontend/src/utils/paths.js index f6f543b5631..2d6a7c285ec 100644 --- a/frontend/src/utils/paths.js +++ b/frontend/src/utils/paths.js @@ -175,5 +175,8 @@ export default { liveDocumentSync: { manage: () => `/settings/beta-features/live-document-sync/manage`, }, + fileTypeDefault: { + manage: () => `/settings/beta-features/file-type-default/manage`, + }, }, }; diff --git a/server/endpoints/experimental/fileTypeDefault.js b/server/endpoints/experimental/fileTypeDefault.js new file mode 100644 index 00000000000..7d5485fa1f1 --- /dev/null +++ b/server/endpoints/experimental/fileTypeDefault.js @@ -0,0 +1,58 @@ +const { FileTypeDefault } = require("../../models/fileTypeDefault"); +const { Document } = require("../../models/documents"); +const { EventLogs } = require("../../models/eventLogs"); +const { SystemSettings } = require("../../models/systemSettings"); +const { Telemetry } = require("../../models/telemetry"); +const { reqBody } = require("../../utils/http"); +const { + featureFlagEnabled, +} = require("../../utils/middleware/featureFlagEnabled"); +const { + flexUserRoleValid, + ROLES, +} = require("../../utils/middleware/multiUserProtected"); +const { validWorkspaceSlug } = require("../../utils/middleware/validWorkspace"); +const { validatedRequest } = require("../../utils/middleware/validatedRequest"); + +function fileTypeDefaultEndpoints(app) { + if (!app) return; + + app.post( + "/experimental/toggle-file-type-default", + [validatedRequest, flexUserRoleValid([ROLES.admin])], + async (request, response) => { + try { + const { updatedStatus = false } = reqBody(request); + const newStatus = + SystemSettings.validations.experimental_file_type_default(updatedStatus); + const currentStatus = + (await SystemSettings.get({ label: "experimental_file_type_default" })) + ?.value || "disabled"; + if (currentStatus === newStatus) + return response + .status(200) + .json({ fileTypeDefaultEnabled: newStatus === "enabled" }); + + // Already validated earlier - so can hot update. + await SystemSettings._updateSettings({ + experimental_file_type_default: newStatus, + }); + if (newStatus === "enabled") { + await Telemetry.sendTelemetry("experimental_feature_enabled", { + feature: "file_type_default", + }); + await EventLogs.logEvent("experimental_feature_enabled", { + feature: "file_type_default", + }); + } + + response.status(200).json({ fileTypeDefaultEnabled: newStatus === "enabled" }); + } catch (e) { + console.error(e); + response.status(500).end(); + } + } + ); +} + +module.exports = { fileTypeDefaultEndpoints }; diff --git a/server/endpoints/experimental/index.js b/server/endpoints/experimental/index.js index f7dc8678a80..f9f83e4b7da 100644 --- a/server/endpoints/experimental/index.js +++ b/server/endpoints/experimental/index.js @@ -1,4 +1,5 @@ const { liveSyncEndpoints } = require("./liveSync"); +const { fileTypeDefaultEndpoints } = require("./fileTypeDefault"); const { importedAgentPluginEndpoints } = require("./imported-agent-plugins"); // All endpoints here are not stable and can move around - have breaking changes @@ -6,6 +7,7 @@ const { importedAgentPluginEndpoints } = require("./imported-agent-plugins"); // When a feature is promoted it should be removed from here and added to the appropriate scope. function experimentalEndpoints(router) { liveSyncEndpoints(router); + fileTypeDefaultEndpoints(router); importedAgentPluginEndpoints(router); } diff --git a/server/models/fileTypeDefault.js b/server/models/fileTypeDefault.js new file mode 100644 index 00000000000..2d12a2fa010 --- /dev/null +++ b/server/models/fileTypeDefault.js @@ -0,0 +1,15 @@ +const { SystemSettings } = require("./systemSettings"); + +const FileTypeDefault = { + featureKey: "experimental_file_type_default", + + /** Check if the fileTypeDefault feature is enabled. */ + enabled: async function() { + return ( + (await SystemSettings.get({ label: this.featureKey }))?.value === + "enabled" + ); + }, +}; + +module.exports = { FileTypeDefault }; diff --git a/server/models/systemSettings.js b/server/models/systemSettings.js index 1af5d8cdd9d..e0b02f12373 100644 --- a/server/models/systemSettings.js +++ b/server/models/systemSettings.js @@ -49,6 +49,7 @@ const SystemSettings = { // beta feature flags "experimental_live_file_sync", + "experimental_file_type_default", // Hub settings "hub_api_key", @@ -147,6 +148,12 @@ const SystemSettings = { if (!["enabled", "disabled"].includes(update)) return "disabled"; return String(update); }, + experimental_file_type_default: (update) => { + if (typeof update === "boolean") + return update === true ? "enabled" : "disabled"; + if (!["enabled", "disabled"].includes(update)) return "disabled"; + return String(update); + }, meta_page_title: (newTitle) => { try { if (typeof newTitle !== "string" || !newTitle) return null; @@ -572,6 +579,9 @@ const SystemSettings = { experimental_live_file_sync: (await SystemSettings.get({ label: "experimental_live_file_sync" })) ?.value === "enabled", + experimental_file_type_default: + (await SystemSettings.get({ label: "experimental_file_type_default" })) + ?.value === "enabled", }; }, From a840884b44597318375cc9ff2e24ee1c1a2b5b93 Mon Sep 17 00:00:00 2001 From: Westopheles Date: Sat, 11 Jan 2025 13:49:51 -1000 Subject: [PATCH 3/6] updated processSingleFile to respond to fileTypeDefault --- collector/processSingleFile/index.js | 3 ++- server/utils/collectorApi/index.js | 6 +++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/collector/processSingleFile/index.js b/collector/processSingleFile/index.js index a00b139ed4b..b9ebdad1823 100644 --- a/collector/processSingleFile/index.js +++ b/collector/processSingleFile/index.js @@ -47,8 +47,9 @@ async function processSingleFile(targetFilename, options = {}) { } let processFileAs = fileExtension; + const fileTypeDefault = options["fileTypeDefault"]; if (!SUPPORTED_FILETYPE_CONVERTERS.hasOwnProperty(fileExtension)) { - if (isTextType(fullFilePath)) { + if (isTextType(fullFilePath) || fileTypeDefault) { console.log( `\x1b[33m[Collector]\x1b[0m The provided filetype of ${fileExtension} does not have a preset and will be processed as .txt.` ); diff --git a/server/utils/collectorApi/index.js b/server/utils/collectorApi/index.js index 7f5781918d8..2750b3b7a7e 100644 --- a/server/utils/collectorApi/index.js +++ b/server/utils/collectorApi/index.js @@ -1,4 +1,5 @@ const { EncryptionManager } = require("../EncryptionManager"); +const { FileTypeDefault } = require("../../models/fileTypeDefault"); // When running locally will occupy the 0.0.0.0 hostname space but when deployed inside // of docker this endpoint is not exposed so it is only on the Docker instances internal network @@ -45,9 +46,12 @@ class CollectorApi { async processDocument(filename = "") { if (!filename) return false; + let augmentedOptions = this.#attachOptions(); + augmentedOptions['fileTypeDefault'] = await FileTypeDefault.enabled(); + const data = JSON.stringify({ filename, - options: this.#attachOptions(), + options: augmentedOptions, }); return await fetch(`${this.endpoint}/process`, { From 4d4bc83bacfe0633c93dd58be4d1aaa32d64ccd4 Mon Sep 17 00:00:00 2001 From: Westopheles Date: Sat, 11 Jan 2025 13:56:39 -1000 Subject: [PATCH 4/6] removing unused management gui --- frontend/src/App.jsx | 8 ----- .../Features/FileTypeDefault/manage/index.jsx | 33 ------------------- .../Features/FileTypeDefault/toggle.jsx | 17 ---------- frontend/src/utils/paths.js | 3 -- 4 files changed, 61 deletions(-) delete mode 100644 frontend/src/pages/Admin/ExperimentalFeatures/Features/FileTypeDefault/manage/index.jsx diff --git a/frontend/src/App.jsx b/frontend/src/App.jsx index ecdb9b726f7..af84603cb8e 100644 --- a/frontend/src/App.jsx +++ b/frontend/src/App.jsx @@ -67,9 +67,6 @@ const ExperimentalFeatures = lazy( const LiveDocumentSyncManage = lazy( () => import("@/pages/Admin/ExperimentalFeatures/Features/LiveSync/manage") ); -const FileTypeDefaultManage = lazy( - () => import("@/pages/Admin/ExperimentalFeatures/Features/FileTypeDefault/manage") -); const CommunityHubTrending = lazy( () => import("@/pages/GeneralSettings/CommunityHub/Trending") @@ -214,11 +211,6 @@ export default function App() { path="/settings/beta-features/live-document-sync/manage" element={} /> - {/* File Type Default feature */} - } - /> - -
-
-
-
-

- Known file types -

-
-

- Nothing to see here, please move along. -

-
-
-
- - ); -} - diff --git a/frontend/src/pages/Admin/ExperimentalFeatures/Features/FileTypeDefault/toggle.jsx b/frontend/src/pages/Admin/ExperimentalFeatures/Features/FileTypeDefault/toggle.jsx index 6d7a0be9ae4..40faa92395f 100644 --- a/frontend/src/pages/Admin/ExperimentalFeatures/Features/FileTypeDefault/toggle.jsx +++ b/frontend/src/pages/Admin/ExperimentalFeatures/Features/FileTypeDefault/toggle.jsx @@ -1,9 +1,6 @@ import System from "@/models/system"; -import paths from "@/utils/paths"; import showToast from "@/utils/toast"; -import { ArrowSquareOut } from "@phosphor-icons/react"; import { useState } from "react"; -import { Link } from "react-router-dom"; export default function FileTypeDefaultToggle({ enabled = false, onToggle }) { const [status, setStatus] = useState(enabled); @@ -57,20 +54,6 @@ export default function FileTypeDefaultToggle({ enabled = false, onToggle }) {

- ); } diff --git a/frontend/src/utils/paths.js b/frontend/src/utils/paths.js index 2d6a7c285ec..f6f543b5631 100644 --- a/frontend/src/utils/paths.js +++ b/frontend/src/utils/paths.js @@ -175,8 +175,5 @@ export default { liveDocumentSync: { manage: () => `/settings/beta-features/live-document-sync/manage`, }, - fileTypeDefault: { - manage: () => `/settings/beta-features/file-type-default/manage`, - }, }, }; From 756c37c0f2076ab36e812f47879a3d843277a1d0 Mon Sep 17 00:00:00 2001 From: Westopheles Date: Sat, 11 Jan 2025 14:11:38 -1000 Subject: [PATCH 5/6] cleaned up the code --- collector/processSingleFile/index.js | 2 +- server/endpoints/experimental/fileTypeDefault.js | 6 ------ server/utils/collectorApi/index.js | 2 +- 3 files changed, 2 insertions(+), 8 deletions(-) diff --git a/collector/processSingleFile/index.js b/collector/processSingleFile/index.js index b9ebdad1823..fc72feb0d11 100644 --- a/collector/processSingleFile/index.js +++ b/collector/processSingleFile/index.js @@ -47,7 +47,7 @@ async function processSingleFile(targetFilename, options = {}) { } let processFileAs = fileExtension; - const fileTypeDefault = options["fileTypeDefault"]; + const fileTypeDefault = options["experimental_file_type_default"]; if (!SUPPORTED_FILETYPE_CONVERTERS.hasOwnProperty(fileExtension)) { if (isTextType(fullFilePath) || fileTypeDefault) { console.log( diff --git a/server/endpoints/experimental/fileTypeDefault.js b/server/endpoints/experimental/fileTypeDefault.js index 7d5485fa1f1..244e5658a2b 100644 --- a/server/endpoints/experimental/fileTypeDefault.js +++ b/server/endpoints/experimental/fileTypeDefault.js @@ -1,17 +1,11 @@ -const { FileTypeDefault } = require("../../models/fileTypeDefault"); -const { Document } = require("../../models/documents"); const { EventLogs } = require("../../models/eventLogs"); const { SystemSettings } = require("../../models/systemSettings"); const { Telemetry } = require("../../models/telemetry"); const { reqBody } = require("../../utils/http"); -const { - featureFlagEnabled, -} = require("../../utils/middleware/featureFlagEnabled"); const { flexUserRoleValid, ROLES, } = require("../../utils/middleware/multiUserProtected"); -const { validWorkspaceSlug } = require("../../utils/middleware/validWorkspace"); const { validatedRequest } = require("../../utils/middleware/validatedRequest"); function fileTypeDefaultEndpoints(app) { diff --git a/server/utils/collectorApi/index.js b/server/utils/collectorApi/index.js index 2750b3b7a7e..51c20ba9ec8 100644 --- a/server/utils/collectorApi/index.js +++ b/server/utils/collectorApi/index.js @@ -47,7 +47,7 @@ class CollectorApi { if (!filename) return false; let augmentedOptions = this.#attachOptions(); - augmentedOptions['fileTypeDefault'] = await FileTypeDefault.enabled(); + augmentedOptions[FileTypeDefault.featureKey] = await FileTypeDefault.enabled(); const data = JSON.stringify({ filename, From 01315cace1170cfdcbd1bd2a6b112f8f1dfc657d Mon Sep 17 00:00:00 2001 From: Westopheles Date: Sat, 11 Jan 2025 14:26:59 -1000 Subject: [PATCH 6/6] ran yarn lint --- .../Features/FileTypeDefault/toggle.jsx | 10 ++++------ server/endpoints/experimental/fileTypeDefault.js | 15 +++++++++++---- server/models/fileTypeDefault.js | 2 +- server/utils/collectorApi/index.js | 3 ++- 4 files changed, 18 insertions(+), 12 deletions(-) diff --git a/frontend/src/pages/Admin/ExperimentalFeatures/Features/FileTypeDefault/toggle.jsx b/frontend/src/pages/Admin/ExperimentalFeatures/Features/FileTypeDefault/toggle.jsx index 40faa92395f..359f111fadc 100644 --- a/frontend/src/pages/Admin/ExperimentalFeatures/Features/FileTypeDefault/toggle.jsx +++ b/frontend/src/pages/Admin/ExperimentalFeatures/Features/FileTypeDefault/toggle.jsx @@ -17,9 +17,7 @@ export default function FileTypeDefaultToggle({ enabled = false, onToggle }) { setStatus(!status); showToast( - `File type default has been ${ - !status ? "enabled" : "disabled" - }.`, + `File type default has been ${!status ? "enabled" : "disabled"}.`, "success", { clear: true } ); @@ -45,9 +43,9 @@ export default function FileTypeDefaultToggle({ enabled = false, onToggle }) {

- If the type of an imported file cannot be determined, this - setting changes the default behavior to process the file as - text instead of displaying an error message. + If the type of an imported file cannot be determined, this setting + changes the default behavior to process the file as text instead of + displaying an error message.

This feature only applies when importing file-based content. diff --git a/server/endpoints/experimental/fileTypeDefault.js b/server/endpoints/experimental/fileTypeDefault.js index 244e5658a2b..5f7b87d6303 100644 --- a/server/endpoints/experimental/fileTypeDefault.js +++ b/server/endpoints/experimental/fileTypeDefault.js @@ -18,10 +18,15 @@ function fileTypeDefaultEndpoints(app) { try { const { updatedStatus = false } = reqBody(request); const newStatus = - SystemSettings.validations.experimental_file_type_default(updatedStatus); + SystemSettings.validations.experimental_file_type_default( + updatedStatus + ); const currentStatus = - (await SystemSettings.get({ label: "experimental_file_type_default" })) - ?.value || "disabled"; + ( + await SystemSettings.get({ + label: "experimental_file_type_default", + }) + )?.value || "disabled"; if (currentStatus === newStatus) return response .status(200) @@ -40,7 +45,9 @@ function fileTypeDefaultEndpoints(app) { }); } - response.status(200).json({ fileTypeDefaultEnabled: newStatus === "enabled" }); + response + .status(200) + .json({ fileTypeDefaultEnabled: newStatus === "enabled" }); } catch (e) { console.error(e); response.status(500).end(); diff --git a/server/models/fileTypeDefault.js b/server/models/fileTypeDefault.js index 2d12a2fa010..f3b964e8b9f 100644 --- a/server/models/fileTypeDefault.js +++ b/server/models/fileTypeDefault.js @@ -4,7 +4,7 @@ const FileTypeDefault = { featureKey: "experimental_file_type_default", /** Check if the fileTypeDefault feature is enabled. */ - enabled: async function() { + enabled: async function () { return ( (await SystemSettings.get({ label: this.featureKey }))?.value === "enabled" diff --git a/server/utils/collectorApi/index.js b/server/utils/collectorApi/index.js index 51c20ba9ec8..34dff3ff76e 100644 --- a/server/utils/collectorApi/index.js +++ b/server/utils/collectorApi/index.js @@ -47,7 +47,8 @@ class CollectorApi { if (!filename) return false; let augmentedOptions = this.#attachOptions(); - augmentedOptions[FileTypeDefault.featureKey] = await FileTypeDefault.enabled(); + augmentedOptions[FileTypeDefault.featureKey] = + await FileTypeDefault.enabled(); const data = JSON.stringify({ filename,