From f9c6adcdabb788e5f13fe3ee9fb68809403358cd Mon Sep 17 00:00:00 2001 From: shatfield4 Date: Mon, 21 Aug 2023 17:13:31 -0700 Subject: [PATCH 1/3] WIP success fail messages for upload document --- .../Modals/MangeWorkspace/Upload/index.jsx | 10 ++++ server/endpoints/workspaces.js | 52 ++++++++++++------- 2 files changed, 43 insertions(+), 19 deletions(-) diff --git a/frontend/src/components/Modals/MangeWorkspace/Upload/index.jsx b/frontend/src/components/Modals/MangeWorkspace/Upload/index.jsx index 2f56dba7d3e..072b33ac676 100644 --- a/frontend/src/components/Modals/MangeWorkspace/Upload/index.jsx +++ b/frontend/src/components/Modals/MangeWorkspace/Upload/index.jsx @@ -10,6 +10,10 @@ import { Frown } from "react-feather"; export default function UploadToWorkspace({ workspace, fileTypes }) { const [ready, setReady] = useState(null); const [files, setFiles] = useState([]); + const [successMsg, setSuccessMsg] = useState("Success"); + const [errorMsg, setErrorMsg] = useState("Fail"); + + const onDrop = useCallback(async (acceptedFiles, rejections) => { const newAccepted = acceptedFiles.map((file) => { return { @@ -144,6 +148,12 @@ export default function UploadToWorkspace({ workspace, fileTypes }) { {Object.values(fileTypes).flat().join(" ")}

+ {successMsg && ( +

{'Success'}

) + } + {errorMsg && ( +

{'Fail'}

+ )} ); } diff --git a/server/endpoints/workspaces.js b/server/endpoints/workspaces.js index c1b468fbd61..b4fd4489001 100644 --- a/server/endpoints/workspaces.js +++ b/server/endpoints/workspaces.js @@ -71,30 +71,44 @@ function workspaceEndpoints(app) { "/workspace/:slug/upload", fileUploadProgress, handleUploads.single("file"), - async function (request, _) { - const { originalname } = request.file; - const processingOnline = await checkPythonAppAlive(); + async function (request, response) { + try { + const { originalname } = request.file; + + const processingOnline = await checkPythonAppAlive(); + if (!processingOnline) { + console.log( + `Python processing API is not online. Document ${originalname} will not be processed automatically.` + ); + return response + .status(500) + .json({ message: "Python processing API is not online." }); + } - if (!processingOnline) { - console.log( - `Python processing API is not online. Document ${originalname} will not be processed automatically.` - ); - return; - } + const { success, reason } = await processDocument(originalname); + if (!success) { + console.log( + `Python processing API was not able to process document ${originalname}. Reason: ${reason}` + ); + return response + .status(400) + .json({ message: `Document processing failed. Reason: ${reason}` }); + } - const { success, reason } = await processDocument(originalname); - if (!success) { + await Telemetry.sendTelemetry("document_uploaded"); + return response + .status(200) + .json({ + message: `Document ${originalname} uploaded and processed successfully.`, + }); + } catch (error) { console.log( - `Python processing API was not able to process document ${originalname}. Reason: ${reason}` + `Unexpected error while uploading document: ${error.message}` ); - return false; + return response + .status(500) + .json({ message: "Unexpected server error. Please try again." }); } - - console.log( - `Document ${originalname} uploaded processed and successfully. It is now available in documents.` - ); - await Telemetry.sendTelemetry("document_uploaded"); - return; } ); From 7053954579e52da04ce018c13b6a51b790b52c9f Mon Sep 17 00:00:00 2001 From: shatfield4 Date: Tue, 22 Aug 2023 14:43:18 -0700 Subject: [PATCH 2/3] added success/error msgs for uploading feedback and disabled fileUploadProgress in backend --- .../Upload/FileUploadProgress/index.jsx | 15 ++++- .../Modals/MangeWorkspace/Upload/index.jsx | 39 +++++++++++-- frontend/src/models/workspace.js | 4 +- server/endpoints/workspaces.js | 55 +++++++------------ 4 files changed, 69 insertions(+), 44 deletions(-) diff --git a/frontend/src/components/Modals/MangeWorkspace/Upload/FileUploadProgress/index.jsx b/frontend/src/components/Modals/MangeWorkspace/Upload/FileUploadProgress/index.jsx index 8c5054c30a0..5883c4b2095 100644 --- a/frontend/src/components/Modals/MangeWorkspace/Upload/FileUploadProgress/index.jsx +++ b/frontend/src/components/Modals/MangeWorkspace/Upload/FileUploadProgress/index.jsx @@ -10,6 +10,8 @@ function FileUploadProgressComponent({ file, rejected = false, reason = null, + onUploadSuccess, + onUploadError, }) { const [timerMs, setTimerMs] = useState(10); const [status, setStatus] = useState(file?.rejected ? "uploading" : "failed"); @@ -24,9 +26,16 @@ function FileUploadProgressComponent({ }, 100); // Chunk streaming not working in production so we just sit and wait - await Workspace.uploadFile(slug, formData); - setStatus("complete"); - clearInterval(timer); + const { response, data } = await Workspace.uploadFile(slug, formData); + if (!response.ok) { + setStatus("failed"); + clearInterval(timer); + onUploadError(data.error); + } else { + setStatus("complete"); + clearInterval(timer); + onUploadSuccess(); + } } !!file && !rejected && uploadFile(); }, []); diff --git a/frontend/src/components/Modals/MangeWorkspace/Upload/index.jsx b/frontend/src/components/Modals/MangeWorkspace/Upload/index.jsx index 072b33ac676..b7a8bc37c42 100644 --- a/frontend/src/components/Modals/MangeWorkspace/Upload/index.jsx +++ b/frontend/src/components/Modals/MangeWorkspace/Upload/index.jsx @@ -10,9 +10,18 @@ import { Frown } from "react-feather"; export default function UploadToWorkspace({ workspace, fileTypes }) { const [ready, setReady] = useState(null); const [files, setFiles] = useState([]); - const [successMsg, setSuccessMsg] = useState("Success"); - const [errorMsg, setErrorMsg] = useState("Fail"); + const [successMsg, setSuccessMsg] = useState(""); + const [errorMsg, setErrorMsg] = useState(""); + const handleUploadSuccess = () => { + setSuccessMsg("File uploaded successfully"); + setErrorMsg(null); + }; + + const handleUploadError = (message) => { + setErrorMsg(`Upload failed: ${message}`); + setSuccessMsg(null); + }; const onDrop = useCallback(async (acceptedFiles, rejections) => { const newAccepted = acceptedFiles.map((file) => { @@ -41,6 +50,20 @@ export default function UploadToWorkspace({ workspace, fileTypes }) { checkProcessorOnline(); }, []); + useEffect(() => { + if (!!successMsg) { + setTimeout(() => { + setSuccessMsg(""); + }, 3_500); + } + + if (!!errorMsg) { + setTimeout(() => { + setErrorMsg(""); + }, 3_500); + } + }, [successMsg, errorMsg]); + const { getRootProps, getInputProps } = useDropzone({ onDrop, accept: { @@ -137,6 +160,8 @@ export default function UploadToWorkspace({ workspace, fileTypes }) { slug={workspace.slug} rejected={file?.rejected} reason={file?.reason} + onUploadSuccess={handleUploadSuccess} + onUploadError={handleUploadError} /> ))} @@ -149,10 +174,14 @@ export default function UploadToWorkspace({ workspace, fileTypes }) {

{successMsg && ( -

{'Success'}

) - } +

+ {successMsg} +

+ )} {errorMsg && ( -

{'Fail'}

+

+ {errorMsg} +

)} ); diff --git a/frontend/src/models/workspace.js b/frontend/src/models/workspace.js index ac61c71863a..540a6f13489 100644 --- a/frontend/src/models/workspace.js +++ b/frontend/src/models/workspace.js @@ -107,7 +107,9 @@ const Workspace = { body: formData, headers: baseHeaders(), }); - return response; + + const data = await response.json(); + return { response, data }; }, }; diff --git a/server/endpoints/workspaces.js b/server/endpoints/workspaces.js index b4fd4489001..cc45eeb79e2 100644 --- a/server/endpoints/workspaces.js +++ b/server/endpoints/workspaces.js @@ -69,46 +69,31 @@ function workspaceEndpoints(app) { app.post( "/workspace/:slug/upload", - fileUploadProgress, handleUploads.single("file"), async function (request, response) { - try { - const { originalname } = request.file; - - const processingOnline = await checkPythonAppAlive(); - if (!processingOnline) { - console.log( - `Python processing API is not online. Document ${originalname} will not be processed automatically.` - ); - return response - .status(500) - .json({ message: "Python processing API is not online." }); - } + const { originalname } = request.file; + const processingOnline = await checkPythonAppAlive(); - const { success, reason } = await processDocument(originalname); - if (!success) { - console.log( - `Python processing API was not able to process document ${originalname}. Reason: ${reason}` - ); - return response - .status(400) - .json({ message: `Document processing failed. Reason: ${reason}` }); - } - - await Telemetry.sendTelemetry("document_uploaded"); - return response - .status(200) - .json({ - message: `Document ${originalname} uploaded and processed successfully.`, - }); - } catch (error) { - console.log( - `Unexpected error while uploading document: ${error.message}` - ); - return response + if (!processingOnline) { + response .status(500) - .json({ message: "Unexpected server error. Please try again." }); + .json({ + success: false, + error: `Python processing API is not online. Document ${originalname} will not be processed automatically.`, + }) + .end(); } + + const { success, reason } = await processDocument(originalname); + if (!success) { + response.status(500).json({ success: false, error: reason }).end(); + } + + console.log( + `Document ${originalname} uploaded processed and successfully. It is now available in documents.` + ); + await Telemetry.sendTelemetry("document_uploaded"); + response.status(200).json({ success: true, error: null }); } ); From e0ad12d023b32141e18c280db6c8720c979ccc28 Mon Sep 17 00:00:00 2001 From: timothycarambat Date: Tue, 22 Aug 2023 19:17:31 -0700 Subject: [PATCH 3/3] remove unused middleware --- server/endpoints/workspaces.js | 3 --- server/utils/middleware/fileUploadProgress.js | 26 ------------------- 2 files changed, 29 deletions(-) delete mode 100644 server/utils/middleware/fileUploadProgress.js diff --git a/server/endpoints/workspaces.js b/server/endpoints/workspaces.js index cc45eeb79e2..ff8e2aeab54 100644 --- a/server/endpoints/workspaces.js +++ b/server/endpoints/workspaces.js @@ -6,9 +6,6 @@ const { WorkspaceChats } = require("../models/workspaceChats"); const { convertToChatHistory } = require("../utils/chats"); const { getVectorDbClass } = require("../utils/helpers"); const { setupMulter } = require("../utils/files/multer"); -const { - fileUploadProgress, -} = require("../utils/middleware/fileUploadProgress"); const { checkPythonAppAlive, processDocument, diff --git a/server/utils/middleware/fileUploadProgress.js b/server/utils/middleware/fileUploadProgress.js deleted file mode 100644 index ecacfd5e015..00000000000 --- a/server/utils/middleware/fileUploadProgress.js +++ /dev/null @@ -1,26 +0,0 @@ -async function fileUploadProgress(request, response, next) { - let progress = 0; - const fileSize = request.headers["content-length"] - ? parseInt(request.headers["content-length"]) - : 0; - - // Note(tcarambat): While this is chunked it does not stream back to the UI for some reason. - // It just waits for the entire requests to finish. Likely because it is not using EventSource on frontend - // which is limited to GET. - // TODO: Someone smarter than me add streaming here to report back real-time progress. - response.writeHead(200); - request.on("data", (chunk) => { - progress += chunk.length; - const percentage = (progress / fileSize) * 100; - response.write(`${JSON.stringify({ progress, fileSize, percentage })}\n`); - if (progress >= fileSize) { - response.end(); - } - }); - - next(); -} - -module.exports = { - fileUploadProgress, -};