From b9123c41cd49026515817c6a7ec6cf8764abdbb5 Mon Sep 17 00:00:00 2001
From: shatfield4 <seanhatfield5@gmail.com>
Date: Tue, 18 Jun 2024 17:46:41 -0700
Subject: [PATCH 1/2] implement winston logger for backend endpoints

---
 server/endpoints/system.js           | 31 +++++++++++++++-------------
 server/endpoints/workspaceThreads.js | 29 +++++++++++++++++++-------
 server/endpoints/workspaces.js       | 31 ++++++++++++++++------------
 server/index.js                      |  3 ++-
 4 files changed, 58 insertions(+), 36 deletions(-)

diff --git a/server/endpoints/system.js b/server/endpoints/system.js
index 6ab30c5c10a..68f825d94f6 100644
--- a/server/endpoints/system.js
+++ b/server/endpoints/system.js
@@ -51,6 +51,7 @@ const {
   generateRecoveryCodes,
 } = require("../utils/PasswordRecovery");
 const { SlashCommandPresets } = require("../models/slashCommandsPresets");
+const logger = require("../utils/logger");
 
 function systemEndpoints(app) {
   if (!app) return;
@@ -75,7 +76,7 @@ function systemEndpoints(app) {
       const results = await SystemSettings.currentSettings();
       response.status(200).json({ results });
     } catch (e) {
-      console.log(e.message, e);
+      logger.error(e.message, { origin: "/setup-complete" });
       response.sendStatus(500).end();
     }
   });
@@ -98,7 +99,7 @@ function systemEndpoints(app) {
 
         response.sendStatus(200).end();
       } catch (e) {
-        console.log(e.message, e);
+        logger.error(e.message, { origin: "/system/check-token" });
         response.sendStatus(500).end();
       }
     }
@@ -241,7 +242,7 @@ function systemEndpoints(app) {
         });
       }
     } catch (e) {
-      console.log(e.message, e);
+      logger.error(e.message, { origin: "/request-token" });
       response.sendStatus(500).end();
     }
   });
@@ -307,7 +308,7 @@ function systemEndpoints(app) {
           : await VectorDb.totalVectors();
         response.status(200).json({ vectorCount });
       } catch (e) {
-        console.log(e.message, e);
+        logger.error(e.message, { origin: "/system/system-vectors" });
         response.sendStatus(500).end();
       }
     }
@@ -322,7 +323,7 @@ function systemEndpoints(app) {
         await purgeDocument(name);
         response.sendStatus(200).end();
       } catch (e) {
-        console.log(e.message, e);
+        logger.error(e.message, { origin: "/system/remove-document" });
         response.sendStatus(500).end();
       }
     }
@@ -337,7 +338,7 @@ function systemEndpoints(app) {
         for await (const name of names) await purgeDocument(name);
         response.sendStatus(200).end();
       } catch (e) {
-        console.log(e.message, e);
+        logger.error(e.message, { origin: "/system/remove-documents" });
         response.sendStatus(500).end();
       }
     }
@@ -352,7 +353,7 @@ function systemEndpoints(app) {
         await purgeFolder(name);
         response.sendStatus(200).end();
       } catch (e) {
-        console.log(e.message, e);
+        logger.error(e.message, { origin: "/system/remove-folder" });
         response.sendStatus(500).end();
       }
     }
@@ -366,7 +367,7 @@ function systemEndpoints(app) {
         const localFiles = await viewLocalFiles();
         response.status(200).json({ localFiles });
       } catch (e) {
-        console.log(e.message, e);
+        logger.error(e.message, { origin: "/system/local-files" });
         response.sendStatus(500).end();
       }
     }
@@ -380,7 +381,9 @@ function systemEndpoints(app) {
         const online = await new CollectorApi().online();
         response.sendStatus(online ? 200 : 503);
       } catch (e) {
-        console.log(e.message, e);
+        logger.error(e.message, {
+          origin: "/system/document-processing-status",
+        });
         response.sendStatus(500).end();
       }
     }
@@ -399,7 +402,7 @@ function systemEndpoints(app) {
 
         response.status(200).json({ types });
       } catch (e) {
-        console.log(e.message, e);
+        logger.error(e.message, { origin: "/system/accepted-document-types" });
         response.sendStatus(500).end();
       }
     }
@@ -419,7 +422,7 @@ function systemEndpoints(app) {
         if (process.env.NODE_ENV === "production") await dumpENV();
         response.status(200).json({ newValues, error });
       } catch (e) {
-        console.log(e.message, e);
+        logger.error(e.message, { origin: "/system/update-env" });
         response.sendStatus(500).end();
       }
     }
@@ -455,7 +458,7 @@ function systemEndpoints(app) {
         if (process.env.NODE_ENV === "production") await dumpENV();
         response.status(200).json({ success: !error, error });
       } catch (e) {
-        console.log(e.message, e);
+        logger.error(e.message, { origin: "/system/update-password" });
         response.sendStatus(500).end();
       }
     }
@@ -505,7 +508,7 @@ function systemEndpoints(app) {
           multi_user_mode: false,
         });
 
-        console.log(e.message, e);
+        logger.error(e.message, { origin: "/system/enable-multi-user" });
         response.sendStatus(500).end();
       }
     }
@@ -516,7 +519,7 @@ function systemEndpoints(app) {
       const multiUserMode = await SystemSettings.isMultiUserMode();
       response.status(200).json({ multiUserMode });
     } catch (e) {
-      console.log(e.message, e);
+      logger.error(e.message, { origin: "/system/multi-user-mode" });
       response.sendStatus(500).end();
     }
   });
diff --git a/server/endpoints/workspaceThreads.js b/server/endpoints/workspaceThreads.js
index 1c207e5230f..9e1b916cbb6 100644
--- a/server/endpoints/workspaceThreads.js
+++ b/server/endpoints/workspaceThreads.js
@@ -18,6 +18,7 @@ const {
 } = require("../utils/middleware/validWorkspace");
 const { WorkspaceChats } = require("../models/workspaceChats");
 const { convertToChatHistory } = require("../utils/helpers/chat/responses");
+const logger = require("../utils/logger");
 
 function workspaceThreadEndpoints(app) {
   if (!app) return;
@@ -53,7 +54,7 @@ function workspaceThreadEndpoints(app) {
         );
         response.status(200).json({ thread, message });
       } catch (e) {
-        console.log(e.message, e);
+        logger.error(e.message, { origin: "/workspace/:slug/thread/new" });
         response.sendStatus(500).end();
       }
     }
@@ -72,7 +73,7 @@ function workspaceThreadEndpoints(app) {
         });
         response.status(200).json({ threads });
       } catch (e) {
-        console.log(e.message, e);
+        logger.error(e.message, { origin: "/workspace/:slug/threads" });
         response.sendStatus(500).end();
       }
     }
@@ -91,7 +92,9 @@ function workspaceThreadEndpoints(app) {
         await WorkspaceThread.delete({ id: thread.id });
         response.sendStatus(200).end();
       } catch (e) {
-        console.log(e.message, e);
+        logger.error(e.message, {
+          origin: "/workspace/:slug/thread/:threadSlug",
+        });
         response.sendStatus(500).end();
       }
     }
@@ -114,7 +117,9 @@ function workspaceThreadEndpoints(app) {
         });
         response.sendStatus(200).end();
       } catch (e) {
-        console.log(e.message, e);
+        logger.error(e.message, {
+          origin: "/workspace/:slug/thread-bulk-delete",
+        });
         response.sendStatus(500).end();
       }
     }
@@ -145,7 +150,9 @@ function workspaceThreadEndpoints(app) {
 
         response.status(200).json({ history: convertToChatHistory(history) });
       } catch (e) {
-        console.log(e.message, e);
+        logger.error(e.message, {
+          origin: "/workspace/:slug/thread/:threadSlug/chats",
+        });
         response.sendStatus(500).end();
       }
     }
@@ -168,7 +175,9 @@ function workspaceThreadEndpoints(app) {
         );
         response.status(200).json({ thread, message });
       } catch (e) {
-        console.log(e.message, e);
+        logger.error(e.message, {
+          origin: "/workspace/:slug/thread/:threadSlug/update",
+        });
         response.sendStatus(500).end();
       }
     }
@@ -197,7 +206,9 @@ function workspaceThreadEndpoints(app) {
 
         response.sendStatus(200).end();
       } catch (e) {
-        console.log(e.message, e);
+        logger.error(e.message, {
+          origin: "/workspace/:slug/thread/:threadSlug/delete-edited-chats",
+        });
         response.sendStatus(500).end();
       }
     }
@@ -239,7 +250,9 @@ function workspaceThreadEndpoints(app) {
 
         response.sendStatus(200).end();
       } catch (e) {
-        console.log(e.message, e);
+        logger.error(e.message, {
+          origin: "/workspace/:slug/thread/:threadSlug/update-chat",
+        });
         response.sendStatus(500).end();
       }
     }
diff --git a/server/endpoints/workspaces.js b/server/endpoints/workspaces.js
index 6d6f29bbd51..842379a1e98 100644
--- a/server/endpoints/workspaces.js
+++ b/server/endpoints/workspaces.js
@@ -31,6 +31,7 @@ const {
   fetchPfp,
 } = require("../utils/files/pfp");
 const { getTTSProvider } = require("../utils/TextToSpeech");
+const logger = require("../utils/logger");
 
 function workspaceEndpoints(app) {
   if (!app) return;
@@ -68,7 +69,7 @@ function workspaceEndpoints(app) {
 
         response.status(200).json({ workspace, message });
       } catch (e) {
-        console.log(e.message, e);
+        logger.error(e.message, { origin: "/workspace/new" });
         response.sendStatus(500).end();
       }
     }
@@ -97,7 +98,7 @@ function workspaceEndpoints(app) {
         );
         response.status(200).json({ workspace, message });
       } catch (e) {
-        console.log(e.message, e);
+        logger.error(e.message, { origin: "/workspace/:slug/update" });
         response.sendStatus(500).end();
       }
     }
@@ -147,7 +148,7 @@ function workspaceEndpoints(app) {
         );
         response.status(200).json({ success: true, error: null });
       } catch (e) {
-        console.log(e.message, e);
+        logger.error(e.message, { origin: "/workspace/:slug/upload" });
         response.sendStatus(500).end();
       }
     }
@@ -190,7 +191,7 @@ function workspaceEndpoints(app) {
         );
         response.status(200).json({ success: true, error: null });
       } catch (e) {
-        console.log(e.message, e);
+        logger.error(e.message, { origin: "/workspace/:slug/upload-link" });
         response.sendStatus(500).end();
       }
     }
@@ -234,7 +235,9 @@ function workspaceEndpoints(app) {
               : null,
         });
       } catch (e) {
-        console.log(e.message, e);
+        logger.error(e.message, {
+          origin: "/workspace/:slug/update-embeddings",
+        });
         response.sendStatus(500).end();
       }
     }
@@ -277,7 +280,7 @@ function workspaceEndpoints(app) {
         }
         response.sendStatus(200).end();
       } catch (e) {
-        console.log(e.message, e);
+        logger.error(e.message, { origin: "/workspace/:slug" });
         response.sendStatus(500).end();
       }
     }
@@ -318,7 +321,7 @@ function workspaceEndpoints(app) {
         }
         response.sendStatus(200).end();
       } catch (e) {
-        console.log(e.message, e);
+        logger.error(e.message, { origin: "/workspace/:slug/reset-vector-db" });
         response.sendStatus(500).end();
       }
     }
@@ -336,7 +339,7 @@ function workspaceEndpoints(app) {
 
         response.status(200).json({ workspaces });
       } catch (e) {
-        console.log(e.message, e);
+        logger.error(e.message, { origin: "/workspaces" });
         response.sendStatus(500).end();
       }
     }
@@ -355,7 +358,7 @@ function workspaceEndpoints(app) {
 
         response.status(200).json({ workspace });
       } catch (e) {
-        console.log(e.message, e);
+        logger.error(e.message, { origin: "/workspace/:slug" });
         response.sendStatus(500).end();
       }
     }
@@ -382,7 +385,7 @@ function workspaceEndpoints(app) {
           : await WorkspaceChats.forWorkspace(workspace.id);
         response.status(200).json({ history: convertToChatHistory(history) });
       } catch (e) {
-        console.log(e.message, e);
+        logger.error(e.message, { origin: "/workspace/:slug/chats" });
         response.sendStatus(500).end();
       }
     }
@@ -413,7 +416,7 @@ function workspaceEndpoints(app) {
 
         response.sendStatus(200).end();
       } catch (e) {
-        console.log(e.message, e);
+        logger.error(e.message, { origin: "/workspace/:slug/delete-chats" });
         response.sendStatus(500).end();
       }
     }
@@ -437,7 +440,9 @@ function workspaceEndpoints(app) {
 
         response.sendStatus(200).end();
       } catch (e) {
-        console.log(e.message, e);
+        logger.error(e.message, {
+          origin: "/workspace/:slug/delete-edited-chats",
+        });
         response.sendStatus(500).end();
       }
     }
@@ -474,7 +479,7 @@ function workspaceEndpoints(app) {
 
         response.sendStatus(200).end();
       } catch (e) {
-        console.log(e.message, e);
+        logger.error(e.message, { origin: "/workspace/:slug/update-chat" });
         response.sendStatus(500).end();
       }
     }
diff --git a/server/index.js b/server/index.js
index 59d8fec68b4..7ca0263a7d8 100644
--- a/server/index.js
+++ b/server/index.js
@@ -22,6 +22,7 @@ const { bootHTTP, bootSSL } = require("./utils/boot");
 const { workspaceThreadEndpoints } = require("./endpoints/workspaceThreads");
 const { documentEndpoints } = require("./endpoints/document");
 const { agentWebsocket } = require("./endpoints/agentWebsocket");
+const logger = require("./utils/logger");
 const app = express();
 const apiRouter = express.Router();
 const FILE_LIMIT = "3GB";
@@ -104,7 +105,7 @@ if (process.env.NODE_ENV !== "development") {
       }
       return;
     } catch (e) {
-      console.log(e.message, e);
+      logger.error(e.message, { origin: "/v/:command" });
       response.sendStatus(500).end();
     }
   });

From c34ddc35d469d338b9ca97daf4b2a6b37fa3c786 Mon Sep 17 00:00:00 2001
From: shatfield4 <seanhatfield5@gmail.com>
Date: Tue, 18 Jun 2024 21:35:25 -0700
Subject: [PATCH 2/2] implement winston logger for developer api

---
 server/endpoints/api/document/index.js       | 15 +++++++-------
 server/endpoints/api/system/index.js         | 13 ++++++------
 server/endpoints/api/userManagement/index.js |  3 ++-
 server/endpoints/api/workspace/index.js      | 21 +++++++++++---------
 4 files changed, 29 insertions(+), 23 deletions(-)

diff --git a/server/endpoints/api/document/index.js b/server/endpoints/api/document/index.js
index 50601c21a2d..1282bfdd2b2 100644
--- a/server/endpoints/api/document/index.js
+++ b/server/endpoints/api/document/index.js
@@ -13,6 +13,7 @@ const { CollectorApi } = require("../../../utils/collectorApi");
 const fs = require("fs");
 const path = require("path");
 const { Document } = require("../../../models/documents");
+const logger = require("../../../utils/logger");
 const documentsPath =
   process.env.NODE_ENV === "development"
     ? path.resolve(__dirname, "../../../storage/documents")
@@ -115,7 +116,7 @@ function apiDocumentEndpoints(app) {
         });
         response.status(200).json({ success: true, error: null, documents });
       } catch (e) {
-        console.log(e.message, e);
+        logger.error(e.message, { origin: "/v1/document/upload" });
         response.sendStatus(500).end();
       }
     }
@@ -213,7 +214,7 @@ function apiDocumentEndpoints(app) {
         });
         response.status(200).json({ success: true, error: null, documents });
       } catch (e) {
-        console.log(e.message, e);
+        logger.error(e.message, { origin: "/v1/document/upload-link" });
         response.sendStatus(500).end();
       }
     }
@@ -346,7 +347,7 @@ function apiDocumentEndpoints(app) {
         await EventLogs.logEvent("api_raw_document_uploaded");
         response.status(200).json({ success: true, error: null, documents });
       } catch (e) {
-        console.log(e.message, e);
+        logger.error(e.message, { origin: "/v1/document/raw-text" });
         response.sendStatus(500).end();
       }
     }
@@ -391,7 +392,7 @@ function apiDocumentEndpoints(app) {
       const localFiles = await viewLocalFiles();
       response.status(200).json({ localFiles });
     } catch (e) {
-      console.log(e.message, e);
+      logger.error(e.message, { origin: "/v1/documents" });
       response.sendStatus(500).end();
     }
   });
@@ -447,7 +448,7 @@ function apiDocumentEndpoints(app) {
 
         response.status(200).json({ types });
       } catch (e) {
-        console.log(e.message, e);
+        logger.error(e.message, { origin: "/v1/document/accepted-file-types" });
         response.sendStatus(500).end();
       }
     }
@@ -497,7 +498,7 @@ function apiDocumentEndpoints(app) {
           },
         });
       } catch (e) {
-        console.log(e.message, e);
+        logger.error(e.message, { origin: "/v1/document/metadata-schema" });
         response.sendStatus(500).end();
       }
     }
@@ -555,7 +556,7 @@ function apiDocumentEndpoints(app) {
       }
       response.status(200).json({ document });
     } catch (e) {
-      console.log(e.message, e);
+      logger.error(e.message, { origin: "/v1/document/:docName" });
       response.sendStatus(500).end();
     }
   });
diff --git a/server/endpoints/api/system/index.js b/server/endpoints/api/system/index.js
index d129416231f..62f2731c317 100644
--- a/server/endpoints/api/system/index.js
+++ b/server/endpoints/api/system/index.js
@@ -8,6 +8,7 @@ const {
 } = require("../../../utils/helpers/chat/convertTo");
 const { dumpENV, updateENV } = require("../../../utils/helpers/updateENV");
 const { reqBody } = require("../../../utils/http");
+const logger = require("../../../utils/logger");
 const { validApiKey } = require("../../../utils/middleware/validApiKey");
 
 function apiSystemEndpoints(app) {
@@ -29,7 +30,7 @@ function apiSystemEndpoints(app) {
       await dumpENV();
       response.sendStatus(200).end();
     } catch (e) {
-      console.log(e.message, e);
+      logger.error(e.message, { origin: "/v1/system/env-dump" });
       response.sendStatus(500).end();
     }
   });
@@ -66,7 +67,7 @@ function apiSystemEndpoints(app) {
       const settings = await SystemSettings.currentSettings();
       response.status(200).json({ settings });
     } catch (e) {
-      console.log(e.message, e);
+      logger.error(e.message, { origin: "/v1/system" });
       response.sendStatus(500).end();
     }
   });
@@ -98,7 +99,7 @@ function apiSystemEndpoints(app) {
       const vectorCount = await VectorDb.totalVectors();
       response.status(200).json({ vectorCount });
     } catch (e) {
-      console.log(e.message, e);
+      logger.error(e.message, { origin: "/v1/system/vector-count" });
       response.sendStatus(500).end();
     }
   });
@@ -148,7 +149,7 @@ function apiSystemEndpoints(app) {
         if (process.env.NODE_ENV === "production") await dumpENV();
         response.status(200).json({ newValues, error });
       } catch (e) {
-        console.log(e.message, e);
+        logger.error(e.message, { origin: "/v1/system/update-env" });
         response.sendStatus(500).end();
       }
     }
@@ -202,7 +203,7 @@ function apiSystemEndpoints(app) {
         response.setHeader("Content-Type", contentType);
         response.status(200).send(data);
       } catch (e) {
-        console.log(e.message, e);
+        logger.error(e.message, { origin: "/v1/system/export-chats" });
         response.sendStatus(500).end();
       }
     }
@@ -268,7 +269,7 @@ function apiSystemEndpoints(app) {
           .json({ success: true, message: "Documents removed successfully" })
           .end();
       } catch (e) {
-        console.log(e.message, e);
+        logger.error(e.message, { origin: "/v1/system/remove-documents" });
         response.sendStatus(500).end();
       }
     }
diff --git a/server/endpoints/api/userManagement/index.js b/server/endpoints/api/userManagement/index.js
index 656fc580b63..3487e014b4c 100644
--- a/server/endpoints/api/userManagement/index.js
+++ b/server/endpoints/api/userManagement/index.js
@@ -1,5 +1,6 @@
 const { User } = require("../../../models/user");
 const { multiUserMode } = require("../../../utils/http");
+const logger = require("../../../utils/logger");
 const { validApiKey } = require("../../../utils/middleware/validApiKey");
 
 function apiUserManagementEndpoints(app) {
@@ -55,7 +56,7 @@ function apiUserManagementEndpoints(app) {
       }));
       response.status(200).json({ users: filteredUsers });
     } catch (e) {
-      console.log(e.message, e);
+      logger.error(e.message, { origin: "/v1/users" });
       response.sendStatus(500).end();
     }
   });
diff --git a/server/endpoints/api/workspace/index.js b/server/endpoints/api/workspace/index.js
index cbbf1f23634..cb1bffd7cbe 100644
--- a/server/endpoints/api/workspace/index.js
+++ b/server/endpoints/api/workspace/index.js
@@ -17,6 +17,7 @@ const {
   convertToChatHistory,
   writeResponseChunk,
 } = require("../../../utils/helpers/chat/responses");
+const logger = require("../../../utils/logger");
 
 function apiWorkspaceEndpoints(app) {
   if (!app) return;
@@ -79,7 +80,7 @@ function apiWorkspaceEndpoints(app) {
       });
       response.status(200).json({ workspace, message });
     } catch (e) {
-      console.log(e.message, e);
+      logger.error(e.message, { origin: "/v1/workspace/new" });
       response.sendStatus(500).end();
     }
   });
@@ -121,7 +122,7 @@ function apiWorkspaceEndpoints(app) {
       const workspaces = await Workspace.where();
       response.status(200).json({ workspaces });
     } catch (e) {
-      console.log(e.message, e);
+      logger.error(e.message, { origin: "/v1/workspaces" });
       response.sendStatus(500).end();
     }
   });
@@ -170,7 +171,7 @@ function apiWorkspaceEndpoints(app) {
       const workspace = await Workspace.get({ slug });
       response.status(200).json({ workspace });
     } catch (e) {
-      console.log(e.message, e);
+      logger.error(e.message, { origin: "/v1/workspace/:slug" });
       response.sendStatus(500).end();
     }
   });
@@ -221,7 +222,7 @@ function apiWorkspaceEndpoints(app) {
         }
         response.sendStatus(200).end();
       } catch (e) {
-        console.log(e.message, e);
+        logger.error(e.message, { origin: "/v1/workspace/:slug" });
         response.sendStatus(500).end();
       }
     }
@@ -301,7 +302,7 @@ function apiWorkspaceEndpoints(app) {
         );
         response.status(200).json({ workspace, message });
       } catch (e) {
-        console.log(e.message, e);
+        logger.error(e.message, { origin: "/v1/workspace/:slug/update" });
         response.sendStatus(500).end();
       }
     }
@@ -362,7 +363,7 @@ function apiWorkspaceEndpoints(app) {
         const history = await WorkspaceChats.forWorkspace(workspace.id);
         response.status(200).json({ history: convertToChatHistory(history) });
       } catch (e) {
-        console.log(e.message, e);
+        logger.error(e.message, { origin: "/v1/workspace/:slug/chats" });
         response.sendStatus(500).end();
       }
     }
@@ -441,7 +442,9 @@ function apiWorkspaceEndpoints(app) {
         });
         response.status(200).json({ workspace: updatedWorkspace });
       } catch (e) {
-        console.log(e.message, e);
+        logger.error(e.message, {
+          origin: "/v1/workspace/:slug/update-embeddings",
+        });
         response.sendStatus(500).end();
       }
     }
@@ -603,7 +606,7 @@ function apiWorkspaceEndpoints(app) {
         });
         response.status(200).json({ ...result });
       } catch (e) {
-        console.log(e.message, e);
+        logger.error(e.message, { origin: "/v1/workspace/:slug/chat" });
         response.status(500).json({
           id: uuidv4(),
           type: "abort",
@@ -726,7 +729,7 @@ function apiWorkspaceEndpoints(app) {
         });
         response.end();
       } catch (e) {
-        console.log(e.message, e);
+        logger.error(e.message, { origin: "/v1/workspace/:slug/stream-chat" });
         writeResponseChunk(response, {
           id: uuidv4(),
           type: "abort",