From b9123c41cd49026515817c6a7ec6cf8764abdbb5 Mon Sep 17 00:00:00 2001 From: shatfield4 Date: Tue, 18 Jun 2024 17:46:41 -0700 Subject: [PATCH 1/3] implement winston logger for backend endpoints --- server/endpoints/system.js | 31 +++++++++++++++------------- server/endpoints/workspaceThreads.js | 29 +++++++++++++++++++------- server/endpoints/workspaces.js | 31 ++++++++++++++++------------ server/index.js | 3 ++- 4 files changed, 58 insertions(+), 36 deletions(-) diff --git a/server/endpoints/system.js b/server/endpoints/system.js index 6ab30c5c10a..68f825d94f6 100644 --- a/server/endpoints/system.js +++ b/server/endpoints/system.js @@ -51,6 +51,7 @@ const { generateRecoveryCodes, } = require("../utils/PasswordRecovery"); const { SlashCommandPresets } = require("../models/slashCommandsPresets"); +const logger = require("../utils/logger"); function systemEndpoints(app) { if (!app) return; @@ -75,7 +76,7 @@ function systemEndpoints(app) { const results = await SystemSettings.currentSettings(); response.status(200).json({ results }); } catch (e) { - console.log(e.message, e); + logger.error(e.message, { origin: "/setup-complete" }); response.sendStatus(500).end(); } }); @@ -98,7 +99,7 @@ function systemEndpoints(app) { response.sendStatus(200).end(); } catch (e) { - console.log(e.message, e); + logger.error(e.message, { origin: "/system/check-token" }); response.sendStatus(500).end(); } } @@ -241,7 +242,7 @@ function systemEndpoints(app) { }); } } catch (e) { - console.log(e.message, e); + logger.error(e.message, { origin: "/request-token" }); response.sendStatus(500).end(); } }); @@ -307,7 +308,7 @@ function systemEndpoints(app) { : await VectorDb.totalVectors(); response.status(200).json({ vectorCount }); } catch (e) { - console.log(e.message, e); + logger.error(e.message, { origin: "/system/system-vectors" }); response.sendStatus(500).end(); } } @@ -322,7 +323,7 @@ function systemEndpoints(app) { await purgeDocument(name); response.sendStatus(200).end(); } catch (e) { - console.log(e.message, e); + logger.error(e.message, { origin: "/system/remove-document" }); response.sendStatus(500).end(); } } @@ -337,7 +338,7 @@ function systemEndpoints(app) { for await (const name of names) await purgeDocument(name); response.sendStatus(200).end(); } catch (e) { - console.log(e.message, e); + logger.error(e.message, { origin: "/system/remove-documents" }); response.sendStatus(500).end(); } } @@ -352,7 +353,7 @@ function systemEndpoints(app) { await purgeFolder(name); response.sendStatus(200).end(); } catch (e) { - console.log(e.message, e); + logger.error(e.message, { origin: "/system/remove-folder" }); response.sendStatus(500).end(); } } @@ -366,7 +367,7 @@ function systemEndpoints(app) { const localFiles = await viewLocalFiles(); response.status(200).json({ localFiles }); } catch (e) { - console.log(e.message, e); + logger.error(e.message, { origin: "/system/local-files" }); response.sendStatus(500).end(); } } @@ -380,7 +381,9 @@ function systemEndpoints(app) { const online = await new CollectorApi().online(); response.sendStatus(online ? 200 : 503); } catch (e) { - console.log(e.message, e); + logger.error(e.message, { + origin: "/system/document-processing-status", + }); response.sendStatus(500).end(); } } @@ -399,7 +402,7 @@ function systemEndpoints(app) { response.status(200).json({ types }); } catch (e) { - console.log(e.message, e); + logger.error(e.message, { origin: "/system/accepted-document-types" }); response.sendStatus(500).end(); } } @@ -419,7 +422,7 @@ function systemEndpoints(app) { if (process.env.NODE_ENV === "production") await dumpENV(); response.status(200).json({ newValues, error }); } catch (e) { - console.log(e.message, e); + logger.error(e.message, { origin: "/system/update-env" }); response.sendStatus(500).end(); } } @@ -455,7 +458,7 @@ function systemEndpoints(app) { if (process.env.NODE_ENV === "production") await dumpENV(); response.status(200).json({ success: !error, error }); } catch (e) { - console.log(e.message, e); + logger.error(e.message, { origin: "/system/update-password" }); response.sendStatus(500).end(); } } @@ -505,7 +508,7 @@ function systemEndpoints(app) { multi_user_mode: false, }); - console.log(e.message, e); + logger.error(e.message, { origin: "/system/enable-multi-user" }); response.sendStatus(500).end(); } } @@ -516,7 +519,7 @@ function systemEndpoints(app) { const multiUserMode = await SystemSettings.isMultiUserMode(); response.status(200).json({ multiUserMode }); } catch (e) { - console.log(e.message, e); + logger.error(e.message, { origin: "/system/multi-user-mode" }); response.sendStatus(500).end(); } }); diff --git a/server/endpoints/workspaceThreads.js b/server/endpoints/workspaceThreads.js index 1c207e5230f..9e1b916cbb6 100644 --- a/server/endpoints/workspaceThreads.js +++ b/server/endpoints/workspaceThreads.js @@ -18,6 +18,7 @@ const { } = require("../utils/middleware/validWorkspace"); const { WorkspaceChats } = require("../models/workspaceChats"); const { convertToChatHistory } = require("../utils/helpers/chat/responses"); +const logger = require("../utils/logger"); function workspaceThreadEndpoints(app) { if (!app) return; @@ -53,7 +54,7 @@ function workspaceThreadEndpoints(app) { ); response.status(200).json({ thread, message }); } catch (e) { - console.log(e.message, e); + logger.error(e.message, { origin: "/workspace/:slug/thread/new" }); response.sendStatus(500).end(); } } @@ -72,7 +73,7 @@ function workspaceThreadEndpoints(app) { }); response.status(200).json({ threads }); } catch (e) { - console.log(e.message, e); + logger.error(e.message, { origin: "/workspace/:slug/threads" }); response.sendStatus(500).end(); } } @@ -91,7 +92,9 @@ function workspaceThreadEndpoints(app) { await WorkspaceThread.delete({ id: thread.id }); response.sendStatus(200).end(); } catch (e) { - console.log(e.message, e); + logger.error(e.message, { + origin: "/workspace/:slug/thread/:threadSlug", + }); response.sendStatus(500).end(); } } @@ -114,7 +117,9 @@ function workspaceThreadEndpoints(app) { }); response.sendStatus(200).end(); } catch (e) { - console.log(e.message, e); + logger.error(e.message, { + origin: "/workspace/:slug/thread-bulk-delete", + }); response.sendStatus(500).end(); } } @@ -145,7 +150,9 @@ function workspaceThreadEndpoints(app) { response.status(200).json({ history: convertToChatHistory(history) }); } catch (e) { - console.log(e.message, e); + logger.error(e.message, { + origin: "/workspace/:slug/thread/:threadSlug/chats", + }); response.sendStatus(500).end(); } } @@ -168,7 +175,9 @@ function workspaceThreadEndpoints(app) { ); response.status(200).json({ thread, message }); } catch (e) { - console.log(e.message, e); + logger.error(e.message, { + origin: "/workspace/:slug/thread/:threadSlug/update", + }); response.sendStatus(500).end(); } } @@ -197,7 +206,9 @@ function workspaceThreadEndpoints(app) { response.sendStatus(200).end(); } catch (e) { - console.log(e.message, e); + logger.error(e.message, { + origin: "/workspace/:slug/thread/:threadSlug/delete-edited-chats", + }); response.sendStatus(500).end(); } } @@ -239,7 +250,9 @@ function workspaceThreadEndpoints(app) { response.sendStatus(200).end(); } catch (e) { - console.log(e.message, e); + logger.error(e.message, { + origin: "/workspace/:slug/thread/:threadSlug/update-chat", + }); response.sendStatus(500).end(); } } diff --git a/server/endpoints/workspaces.js b/server/endpoints/workspaces.js index 6d6f29bbd51..842379a1e98 100644 --- a/server/endpoints/workspaces.js +++ b/server/endpoints/workspaces.js @@ -31,6 +31,7 @@ const { fetchPfp, } = require("../utils/files/pfp"); const { getTTSProvider } = require("../utils/TextToSpeech"); +const logger = require("../utils/logger"); function workspaceEndpoints(app) { if (!app) return; @@ -68,7 +69,7 @@ function workspaceEndpoints(app) { response.status(200).json({ workspace, message }); } catch (e) { - console.log(e.message, e); + logger.error(e.message, { origin: "/workspace/new" }); response.sendStatus(500).end(); } } @@ -97,7 +98,7 @@ function workspaceEndpoints(app) { ); response.status(200).json({ workspace, message }); } catch (e) { - console.log(e.message, e); + logger.error(e.message, { origin: "/workspace/:slug/update" }); response.sendStatus(500).end(); } } @@ -147,7 +148,7 @@ function workspaceEndpoints(app) { ); response.status(200).json({ success: true, error: null }); } catch (e) { - console.log(e.message, e); + logger.error(e.message, { origin: "/workspace/:slug/upload" }); response.sendStatus(500).end(); } } @@ -190,7 +191,7 @@ function workspaceEndpoints(app) { ); response.status(200).json({ success: true, error: null }); } catch (e) { - console.log(e.message, e); + logger.error(e.message, { origin: "/workspace/:slug/upload-link" }); response.sendStatus(500).end(); } } @@ -234,7 +235,9 @@ function workspaceEndpoints(app) { : null, }); } catch (e) { - console.log(e.message, e); + logger.error(e.message, { + origin: "/workspace/:slug/update-embeddings", + }); response.sendStatus(500).end(); } } @@ -277,7 +280,7 @@ function workspaceEndpoints(app) { } response.sendStatus(200).end(); } catch (e) { - console.log(e.message, e); + logger.error(e.message, { origin: "/workspace/:slug" }); response.sendStatus(500).end(); } } @@ -318,7 +321,7 @@ function workspaceEndpoints(app) { } response.sendStatus(200).end(); } catch (e) { - console.log(e.message, e); + logger.error(e.message, { origin: "/workspace/:slug/reset-vector-db" }); response.sendStatus(500).end(); } } @@ -336,7 +339,7 @@ function workspaceEndpoints(app) { response.status(200).json({ workspaces }); } catch (e) { - console.log(e.message, e); + logger.error(e.message, { origin: "/workspaces" }); response.sendStatus(500).end(); } } @@ -355,7 +358,7 @@ function workspaceEndpoints(app) { response.status(200).json({ workspace }); } catch (e) { - console.log(e.message, e); + logger.error(e.message, { origin: "/workspace/:slug" }); response.sendStatus(500).end(); } } @@ -382,7 +385,7 @@ function workspaceEndpoints(app) { : await WorkspaceChats.forWorkspace(workspace.id); response.status(200).json({ history: convertToChatHistory(history) }); } catch (e) { - console.log(e.message, e); + logger.error(e.message, { origin: "/workspace/:slug/chats" }); response.sendStatus(500).end(); } } @@ -413,7 +416,7 @@ function workspaceEndpoints(app) { response.sendStatus(200).end(); } catch (e) { - console.log(e.message, e); + logger.error(e.message, { origin: "/workspace/:slug/delete-chats" }); response.sendStatus(500).end(); } } @@ -437,7 +440,9 @@ function workspaceEndpoints(app) { response.sendStatus(200).end(); } catch (e) { - console.log(e.message, e); + logger.error(e.message, { + origin: "/workspace/:slug/delete-edited-chats", + }); response.sendStatus(500).end(); } } @@ -474,7 +479,7 @@ function workspaceEndpoints(app) { response.sendStatus(200).end(); } catch (e) { - console.log(e.message, e); + logger.error(e.message, { origin: "/workspace/:slug/update-chat" }); response.sendStatus(500).end(); } } diff --git a/server/index.js b/server/index.js index 59d8fec68b4..7ca0263a7d8 100644 --- a/server/index.js +++ b/server/index.js @@ -22,6 +22,7 @@ const { bootHTTP, bootSSL } = require("./utils/boot"); const { workspaceThreadEndpoints } = require("./endpoints/workspaceThreads"); const { documentEndpoints } = require("./endpoints/document"); const { agentWebsocket } = require("./endpoints/agentWebsocket"); +const logger = require("./utils/logger"); const app = express(); const apiRouter = express.Router(); const FILE_LIMIT = "3GB"; @@ -104,7 +105,7 @@ if (process.env.NODE_ENV !== "development") { } return; } catch (e) { - console.log(e.message, e); + logger.error(e.message, { origin: "/v/:command" }); response.sendStatus(500).end(); } }); From c34ddc35d469d338b9ca97daf4b2a6b37fa3c786 Mon Sep 17 00:00:00 2001 From: shatfield4 Date: Tue, 18 Jun 2024 21:35:25 -0700 Subject: [PATCH 2/3] implement winston logger for developer api --- server/endpoints/api/document/index.js | 15 +++++++------- server/endpoints/api/system/index.js | 13 ++++++------ server/endpoints/api/userManagement/index.js | 3 ++- server/endpoints/api/workspace/index.js | 21 +++++++++++--------- 4 files changed, 29 insertions(+), 23 deletions(-) diff --git a/server/endpoints/api/document/index.js b/server/endpoints/api/document/index.js index 50601c21a2d..1282bfdd2b2 100644 --- a/server/endpoints/api/document/index.js +++ b/server/endpoints/api/document/index.js @@ -13,6 +13,7 @@ const { CollectorApi } = require("../../../utils/collectorApi"); const fs = require("fs"); const path = require("path"); const { Document } = require("../../../models/documents"); +const logger = require("../../../utils/logger"); const documentsPath = process.env.NODE_ENV === "development" ? path.resolve(__dirname, "../../../storage/documents") @@ -115,7 +116,7 @@ function apiDocumentEndpoints(app) { }); response.status(200).json({ success: true, error: null, documents }); } catch (e) { - console.log(e.message, e); + logger.error(e.message, { origin: "/v1/document/upload" }); response.sendStatus(500).end(); } } @@ -213,7 +214,7 @@ function apiDocumentEndpoints(app) { }); response.status(200).json({ success: true, error: null, documents }); } catch (e) { - console.log(e.message, e); + logger.error(e.message, { origin: "/v1/document/upload-link" }); response.sendStatus(500).end(); } } @@ -346,7 +347,7 @@ function apiDocumentEndpoints(app) { await EventLogs.logEvent("api_raw_document_uploaded"); response.status(200).json({ success: true, error: null, documents }); } catch (e) { - console.log(e.message, e); + logger.error(e.message, { origin: "/v1/document/raw-text" }); response.sendStatus(500).end(); } } @@ -391,7 +392,7 @@ function apiDocumentEndpoints(app) { const localFiles = await viewLocalFiles(); response.status(200).json({ localFiles }); } catch (e) { - console.log(e.message, e); + logger.error(e.message, { origin: "/v1/documents" }); response.sendStatus(500).end(); } }); @@ -447,7 +448,7 @@ function apiDocumentEndpoints(app) { response.status(200).json({ types }); } catch (e) { - console.log(e.message, e); + logger.error(e.message, { origin: "/v1/document/accepted-file-types" }); response.sendStatus(500).end(); } } @@ -497,7 +498,7 @@ function apiDocumentEndpoints(app) { }, }); } catch (e) { - console.log(e.message, e); + logger.error(e.message, { origin: "/v1/document/metadata-schema" }); response.sendStatus(500).end(); } } @@ -555,7 +556,7 @@ function apiDocumentEndpoints(app) { } response.status(200).json({ document }); } catch (e) { - console.log(e.message, e); + logger.error(e.message, { origin: "/v1/document/:docName" }); response.sendStatus(500).end(); } }); diff --git a/server/endpoints/api/system/index.js b/server/endpoints/api/system/index.js index d129416231f..62f2731c317 100644 --- a/server/endpoints/api/system/index.js +++ b/server/endpoints/api/system/index.js @@ -8,6 +8,7 @@ const { } = require("../../../utils/helpers/chat/convertTo"); const { dumpENV, updateENV } = require("../../../utils/helpers/updateENV"); const { reqBody } = require("../../../utils/http"); +const logger = require("../../../utils/logger"); const { validApiKey } = require("../../../utils/middleware/validApiKey"); function apiSystemEndpoints(app) { @@ -29,7 +30,7 @@ function apiSystemEndpoints(app) { await dumpENV(); response.sendStatus(200).end(); } catch (e) { - console.log(e.message, e); + logger.error(e.message, { origin: "/v1/system/env-dump" }); response.sendStatus(500).end(); } }); @@ -66,7 +67,7 @@ function apiSystemEndpoints(app) { const settings = await SystemSettings.currentSettings(); response.status(200).json({ settings }); } catch (e) { - console.log(e.message, e); + logger.error(e.message, { origin: "/v1/system" }); response.sendStatus(500).end(); } }); @@ -98,7 +99,7 @@ function apiSystemEndpoints(app) { const vectorCount = await VectorDb.totalVectors(); response.status(200).json({ vectorCount }); } catch (e) { - console.log(e.message, e); + logger.error(e.message, { origin: "/v1/system/vector-count" }); response.sendStatus(500).end(); } }); @@ -148,7 +149,7 @@ function apiSystemEndpoints(app) { if (process.env.NODE_ENV === "production") await dumpENV(); response.status(200).json({ newValues, error }); } catch (e) { - console.log(e.message, e); + logger.error(e.message, { origin: "/v1/system/update-env" }); response.sendStatus(500).end(); } } @@ -202,7 +203,7 @@ function apiSystemEndpoints(app) { response.setHeader("Content-Type", contentType); response.status(200).send(data); } catch (e) { - console.log(e.message, e); + logger.error(e.message, { origin: "/v1/system/export-chats" }); response.sendStatus(500).end(); } } @@ -268,7 +269,7 @@ function apiSystemEndpoints(app) { .json({ success: true, message: "Documents removed successfully" }) .end(); } catch (e) { - console.log(e.message, e); + logger.error(e.message, { origin: "/v1/system/remove-documents" }); response.sendStatus(500).end(); } } diff --git a/server/endpoints/api/userManagement/index.js b/server/endpoints/api/userManagement/index.js index 656fc580b63..3487e014b4c 100644 --- a/server/endpoints/api/userManagement/index.js +++ b/server/endpoints/api/userManagement/index.js @@ -1,5 +1,6 @@ const { User } = require("../../../models/user"); const { multiUserMode } = require("../../../utils/http"); +const logger = require("../../../utils/logger"); const { validApiKey } = require("../../../utils/middleware/validApiKey"); function apiUserManagementEndpoints(app) { @@ -55,7 +56,7 @@ function apiUserManagementEndpoints(app) { })); response.status(200).json({ users: filteredUsers }); } catch (e) { - console.log(e.message, e); + logger.error(e.message, { origin: "/v1/users" }); response.sendStatus(500).end(); } }); diff --git a/server/endpoints/api/workspace/index.js b/server/endpoints/api/workspace/index.js index cbbf1f23634..cb1bffd7cbe 100644 --- a/server/endpoints/api/workspace/index.js +++ b/server/endpoints/api/workspace/index.js @@ -17,6 +17,7 @@ const { convertToChatHistory, writeResponseChunk, } = require("../../../utils/helpers/chat/responses"); +const logger = require("../../../utils/logger"); function apiWorkspaceEndpoints(app) { if (!app) return; @@ -79,7 +80,7 @@ function apiWorkspaceEndpoints(app) { }); response.status(200).json({ workspace, message }); } catch (e) { - console.log(e.message, e); + logger.error(e.message, { origin: "/v1/workspace/new" }); response.sendStatus(500).end(); } }); @@ -121,7 +122,7 @@ function apiWorkspaceEndpoints(app) { const workspaces = await Workspace.where(); response.status(200).json({ workspaces }); } catch (e) { - console.log(e.message, e); + logger.error(e.message, { origin: "/v1/workspaces" }); response.sendStatus(500).end(); } }); @@ -170,7 +171,7 @@ function apiWorkspaceEndpoints(app) { const workspace = await Workspace.get({ slug }); response.status(200).json({ workspace }); } catch (e) { - console.log(e.message, e); + logger.error(e.message, { origin: "/v1/workspace/:slug" }); response.sendStatus(500).end(); } }); @@ -221,7 +222,7 @@ function apiWorkspaceEndpoints(app) { } response.sendStatus(200).end(); } catch (e) { - console.log(e.message, e); + logger.error(e.message, { origin: "/v1/workspace/:slug" }); response.sendStatus(500).end(); } } @@ -301,7 +302,7 @@ function apiWorkspaceEndpoints(app) { ); response.status(200).json({ workspace, message }); } catch (e) { - console.log(e.message, e); + logger.error(e.message, { origin: "/v1/workspace/:slug/update" }); response.sendStatus(500).end(); } } @@ -362,7 +363,7 @@ function apiWorkspaceEndpoints(app) { const history = await WorkspaceChats.forWorkspace(workspace.id); response.status(200).json({ history: convertToChatHistory(history) }); } catch (e) { - console.log(e.message, e); + logger.error(e.message, { origin: "/v1/workspace/:slug/chats" }); response.sendStatus(500).end(); } } @@ -441,7 +442,9 @@ function apiWorkspaceEndpoints(app) { }); response.status(200).json({ workspace: updatedWorkspace }); } catch (e) { - console.log(e.message, e); + logger.error(e.message, { + origin: "/v1/workspace/:slug/update-embeddings", + }); response.sendStatus(500).end(); } } @@ -603,7 +606,7 @@ function apiWorkspaceEndpoints(app) { }); response.status(200).json({ ...result }); } catch (e) { - console.log(e.message, e); + logger.error(e.message, { origin: "/v1/workspace/:slug/chat" }); response.status(500).json({ id: uuidv4(), type: "abort", @@ -726,7 +729,7 @@ function apiWorkspaceEndpoints(app) { }); response.end(); } catch (e) { - console.log(e.message, e); + logger.error(e.message, { origin: "/v1/workspace/:slug/stream-chat" }); writeResponseChunk(response, { id: uuidv4(), type: "abort", From c44e6e168fb7280a1a73e346a92f2e1928761057 Mon Sep 17 00:00:00 2001 From: shatfield4 Date: Wed, 19 Jun 2024 10:32:50 -0700 Subject: [PATCH 3/3] implement winston logger for all utils/agents/etc in backend --- server/models/eventLogs.js | 8 +-- server/models/telemetry.js | 10 +-- server/swagger/init.js | 3 +- server/utils/DocumentManager/index.js | 3 +- .../websocket/websock-branding-collab.js | 32 ++++++---- .../websocket/websock-multi-turn-chat.js | 16 +++-- server/utils/agents/aibitat/plugins/cli.js | 28 +++++---- server/utils/agents/aibitat/plugins/memory.js | 3 +- .../plugins/sql-agent/SQLConnectors/MSSQL.js | 3 +- .../plugins/sql-agent/SQLConnectors/MySQL.js | 3 +- .../sql-agent/SQLConnectors/Postgresql.js | 3 +- .../agents/aibitat/plugins/web-browsing.js | 9 ++- .../utils/agents/aibitat/plugins/websocket.js | 8 ++- .../agents/aibitat/providers/ai-provider.js | 8 +-- server/utils/agents/index.js | 63 +++++++++++++------ server/utils/boot/index.js | 28 +++++---- server/utils/chats/embed.js | 8 ++- server/utils/chats/stream.js | 11 +++- server/utils/collectorApi/index.js | 4 +- server/utils/comKey/index.js | 3 +- server/utils/database/index.js | 36 +++++++---- server/utils/helpers/chat/index.js | 12 ++-- server/utils/helpers/chat/responses.js | 11 +++- server/utils/prisma/migrateFromSqlite.js | 38 ++++++++--- server/utils/telemetry/index.js | 30 ++++++--- 25 files changed, 256 insertions(+), 125 deletions(-) diff --git a/server/models/eventLogs.js b/server/models/eventLogs.js index 51240431a75..c53be284d05 100644 --- a/server/models/eventLogs.js +++ b/server/models/eventLogs.js @@ -1,3 +1,4 @@ +const logger = require("../utils/logger"); const prisma = require("../utils/prisma"); const EventLogs = { @@ -11,13 +12,10 @@ const EventLogs = { occurredAt: new Date(), }, }); - console.log(`\x1b[32m[Event Logged]\x1b[0m - ${event}`); + logger.info(`[Event Logged] - ${event}`, { origin: "EventLogs" }); return { eventLog, message: null }; } catch (error) { - console.error( - `\x1b[31m[Event Logging Failed]\x1b[0m - ${event}`, - error.message - ); + logger.error(error.message, { origin: "EventLogs" }); return { eventLog: null, message: error.message }; } }, diff --git a/server/models/telemetry.js b/server/models/telemetry.js index 011da05e97f..a2dcd33ce23 100644 --- a/server/models/telemetry.js +++ b/server/models/telemetry.js @@ -1,5 +1,6 @@ const { v4 } = require("uuid"); const { SystemSettings } = require("./systemSettings"); +const logger = require("../utils/logger"); const Telemetry = { // Write-only key. It can't read events or any of your other data, so it's safe to use in public apps. @@ -49,11 +50,10 @@ const Telemetry = { // Silence some events to keep logs from being too messy in production // eg: Tool calls from agents spamming the logs. if (!silent) { - console.log(`\x1b[32m[TELEMETRY SENT]\x1b[0m`, { - event, - distinctId, - properties, - }); + logger.info( + `${event} - ${distinctId} - ${JSON.stringify(properties)}`, + { origin: "Telemetry" } + ); } client.capture({ diff --git a/server/swagger/init.js b/server/swagger/init.js index 064814565a3..dc5d109b860 100644 --- a/server/swagger/init.js +++ b/server/swagger/init.js @@ -1,6 +1,7 @@ const swaggerAutogen = require("swagger-autogen")({ openapi: "3.0.0" }); const fs = require("fs"); const path = require("path"); +const logger = require("../utils/logger"); const doc = { info: { @@ -72,5 +73,5 @@ swaggerAutogen(outputFile, endpointsFiles, doc).then(({ data }) => { encoding: "utf-8", flag: "w", }); - console.log(`Swagger-autogen: \x1b[32mPatched servers.url ✔\x1b[0m`); + logger.info("Patched servers.url ✔", { origin: "Swagger-autogen" }); }); diff --git a/server/utils/DocumentManager/index.js b/server/utils/DocumentManager/index.js index 17fd9860ee2..adc36c4656d 100644 --- a/server/utils/DocumentManager/index.js +++ b/server/utils/DocumentManager/index.js @@ -1,5 +1,6 @@ const fs = require("fs"); const path = require("path"); +const logger = require("../logger"); const documentsPath = process.env.NODE_ENV === "development" @@ -14,7 +15,7 @@ class DocumentManager { } log(text, ...args) { - console.log(`\x1b[36m[DocumentManager]\x1b[0m ${text}`, ...args); + logger.info(`${text} ${args}`, { origin: "DocumentManager" }); } async pinnedDocuments() { diff --git a/server/utils/agents/aibitat/example/websocket/websock-branding-collab.js b/server/utils/agents/aibitat/example/websocket/websock-branding-collab.js index 7229bd882d9..dd1f2ed400e 100644 --- a/server/utils/agents/aibitat/example/websocket/websock-branding-collab.js +++ b/server/utils/agents/aibitat/example/websocket/websock-branding-collab.js @@ -3,7 +3,6 @@ // Scraping is enabled, but search requires AGENT_GSE_* keys. const express = require("express"); -const chalk = require("chalk"); const AIbitat = require("../../index.js"); const { websocket, @@ -11,6 +10,7 @@ const { webScraping, } = require("../../plugins/index.js"); const path = require("path"); +const logger = require("../../../../logger/index.js"); const port = 3000; const app = express(); require("express-ws")(app); @@ -31,11 +31,13 @@ app.ws("/ws", function (ws, _response) { }); ws.on("close", function () { - console.log("Socket killed"); + logger.info("Socket killed", { origin: "websock-branding-collab.js" }); return; }); - console.log("Socket online and waiting..."); + logger.info("Socket online and waiting...", { + origin: "websock-branding-collab.js", + }); runAIbitat(ws).catch((error) => { ws.send( JSON.stringify({ @@ -53,11 +55,15 @@ app.all("*", function (_, response) { }); app.listen(port, () => { - console.log(`Testing HTTP/WSS server listening at http://localhost:${port}`); + logger.info(`Testing HTTP/WSS server listening at http://localhost:${port}`, { + origin: "websock-branding-collab.js", + }); }); async function runAIbitat(socket) { - console.log(chalk.blue("Booting AIbitat class & starting agent(s)")); + logger.info("Booting AIbitat class & starting agent(s)", { + origin: "websock-branding-collab.js", + }); const aibitat = new AIbitat({ provider: "openai", @@ -68,19 +74,19 @@ async function runAIbitat(socket) { .use(webScraping.plugin()) .agent("creativeDirector", { role: `You are a Creative Director. Your role is overseeing the entire branding project, ensuring - the client's brief is met, and maintaining consistency across all brand elements, developing the + the client's brief is met, and maintaining consistency across all brand elements, developing the brand strategy, guiding the visual and conceptual direction, and providing overall creative leadership.`, }) .agent("marketResearcher", { - role: `You do competitive market analysis via searching on the internet and learning about - comparative products and services. You can search by using keywords and phrases that you think will lead + role: `You do competitive market analysis via searching on the internet and learning about + comparative products and services. You can search by using keywords and phrases that you think will lead to competitor research that can help find the unique angle and market of the idea.`, functions: ["web-browsing"], }) .agent("PM", { - role: `You are the Project Coordinator. Your role is overseeing the project's progress, timeline, - and budget. Ensure effective communication and coordination among team members, client, and stakeholders. - Your tasks include planning and scheduling project milestones, tracking tasks, and managing any + role: `You are the Project Coordinator. Your role is overseeing the project's progress, timeline, + and budget. Ensure effective communication and coordination among team members, client, and stakeholders. + Your tasks include planning and scheduling project milestones, tracking tasks, and managing any risks or issues that arise.`, interrupt: "ALWAYS", }) @@ -93,8 +99,8 @@ async function runAIbitat(socket) { await aibitat.start({ from: "PM", to: "#branding", - content: `I have an idea for a muslim focused meetup called Chai & Vibes. - I want to focus on professionals that are muslim and are in their 18-30 year old range who live in big cities. + content: `I have an idea for a muslim focused meetup called Chai & Vibes. + I want to focus on professionals that are muslim and are in their 18-30 year old range who live in big cities. Does anything like this exist? How can we differentiate?`, }); } diff --git a/server/utils/agents/aibitat/example/websocket/websock-multi-turn-chat.js b/server/utils/agents/aibitat/example/websocket/websock-multi-turn-chat.js index 6277978cc78..f9928727428 100644 --- a/server/utils/agents/aibitat/example/websocket/websock-multi-turn-chat.js +++ b/server/utils/agents/aibitat/example/websocket/websock-multi-turn-chat.js @@ -3,7 +3,6 @@ // Scraping is enabled, but search requires AGENT_GSE_* keys. const express = require("express"); -const chalk = require("chalk"); const AIbitat = require("../../index.js"); const { websocket, @@ -11,6 +10,7 @@ const { webScraping, } = require("../../plugins/index.js"); const path = require("path"); +const logger = require("../../../../logger/index.js"); const port = 3000; const app = express(); require("express-ws")(app); @@ -31,11 +31,13 @@ app.ws("/ws", function (ws, _response) { }); ws.on("close", function () { - console.log("Socket killed"); + logger.info("Socket killed", { origin: "websock-multi-turn-chat.js" }); return; }); - console.log("Socket online and waiting..."); + logger.info("Socket online and waiting...", { + origin: "websock-multi-turn-chat.js", + }); runAIbitat(ws).catch((error) => { ws.send( JSON.stringify({ @@ -53,7 +55,9 @@ app.all("*", function (_, response) { }); app.listen(port, () => { - console.log(`Testing HTTP/WSS server listening at http://localhost:${port}`); + logger.info(`Testing HTTP/WSS server listening at http://localhost:${port}`, { + origin: "websock-multi-turn-chat.js", + }); }); const Agent = { @@ -66,7 +70,9 @@ async function runAIbitat(socket) { throw new Error( "This example requires a valid OPEN_AI_KEY in the env.development file" ); - console.log(chalk.blue("Booting AIbitat class & starting agent(s)")); + logger.info("Booting AIbitat class & starting agent(s)", { + origin: "websock-multi-turn-chat.js", + }); const aibitat = new AIbitat({ provider: "openai", model: "gpt-4o", diff --git a/server/utils/agents/aibitat/plugins/cli.js b/server/utils/agents/aibitat/plugins/cli.js index 06a60e98e67..9b15395b717 100644 --- a/server/utils/agents/aibitat/plugins/cli.js +++ b/server/utils/agents/aibitat/plugins/cli.js @@ -2,6 +2,7 @@ const { input } = require("@inquirer/prompts"); const chalk = require("chalk"); const { RetryError } = require("../error"); +const logger = require("../../../logger"); /** * Command-line Interface plugin. It prints the messages on the console and asks for feedback @@ -19,9 +20,13 @@ const cli = { let printing = []; aibitat.onError(async (error) => { - console.error(chalk.red(` error: ${error?.message}`)); + logger.error(chalk.red(` error: ${error?.message}`), { + origin: "cli.js", + }); if (error instanceof RetryError) { - console.error(chalk.red(` retrying in 60 seconds...`)); + logger.error(chalk.red(` retrying in 60 seconds...`), { + origin: "cli.js", + }); setTimeout(() => { aibitat.retry(); }, 60000); @@ -30,8 +35,7 @@ const cli = { }); aibitat.onStart(() => { - console.log(); - console.log("🚀 starting chat ...\n"); + logger.info("🚀 starting chat ...\n", { origin: "cli.js" }); printing = [Promise.resolve()]; }); @@ -46,17 +50,17 @@ const cli = { aibitat.onTerminate(async () => { await Promise.all(printing); - console.log("🚀 chat finished"); + logger.info("🚀 chat finished", { origin: "cli.js" }); }); aibitat.onInterrupt(async (node) => { await Promise.all(printing); const feedback = await this.askForFeedback(node); // Add an extra line after the message - console.log(); + logger.info("", { origin: "cli.js" }); if (feedback === "exit") { - console.log("🚀 chat finished"); + logger.info("🚀 chat finished", { origin: "cli.js" }); return process.exit(0); } @@ -80,10 +84,10 @@ const cli = { )} ${replying}:`; if (!simulateStream) { - console.log(reference); - console.log(message.content); + logger.info(reference, { origin: "cli.js" }); + logger.info(message.content, { origin: "cli.js" }); // Add an extra line after the message - console.log(); + logger.info("", { origin: "cli.js" }); return; } @@ -114,8 +118,8 @@ const cli = { } // Add an extra line after the message - console.log(); - console.log(); + logger.info("", { origin: "cli.js" }); + logger.info("", { origin: "cli.js" }); }, /** diff --git a/server/utils/agents/aibitat/plugins/memory.js b/server/utils/agents/aibitat/plugins/memory.js index d31a264d18d..200eda127d3 100644 --- a/server/utils/agents/aibitat/plugins/memory.js +++ b/server/utils/agents/aibitat/plugins/memory.js @@ -1,6 +1,7 @@ const { v4 } = require("uuid"); const { getVectorDbClass, getLLMProvider } = require("../../../helpers"); const { Deduplicator } = require("../utils/dedupe"); +const logger = require("../../../logger"); const memory = { name: "rag-memory", @@ -77,7 +78,7 @@ const memory = { this.tracker.trackRun(this.name, { action, content }); return response; } catch (error) { - console.log(error); + logger.error(error.message, { origin: "memory.handler" }); return `There was an error while calling the function. ${error.message}`; } }, diff --git a/server/utils/agents/aibitat/plugins/sql-agent/SQLConnectors/MSSQL.js b/server/utils/agents/aibitat/plugins/sql-agent/SQLConnectors/MSSQL.js index ed75aa7aa71..f2e95162ff1 100644 --- a/server/utils/agents/aibitat/plugins/sql-agent/SQLConnectors/MSSQL.js +++ b/server/utils/agents/aibitat/plugins/sql-agent/SQLConnectors/MSSQL.js @@ -1,5 +1,6 @@ const mssql = require("mssql"); const UrlPattern = require("url-pattern"); +const logger = require("../../../../../logger"); class MSSQLConnector { #connected = false; @@ -69,7 +70,7 @@ class MSSQLConnector { result.rows = query.recordset; result.count = query.rowsAffected.reduce((sum, a) => sum + a, 0); } catch (err) { - console.log(this.constructor.name, err); + logger.error(err.message, { origin: "MSSQLConnector.runQuery" }); result.error = err.message; } finally { await this._client.close(); diff --git a/server/utils/agents/aibitat/plugins/sql-agent/SQLConnectors/MySQL.js b/server/utils/agents/aibitat/plugins/sql-agent/SQLConnectors/MySQL.js index d9982ab318e..9c745d88673 100644 --- a/server/utils/agents/aibitat/plugins/sql-agent/SQLConnectors/MySQL.js +++ b/server/utils/agents/aibitat/plugins/sql-agent/SQLConnectors/MySQL.js @@ -1,5 +1,6 @@ const mysql = require("mysql2/promise"); const UrlPattern = require("url-pattern"); +const logger = require("../../../../../logger"); class MySQLConnector { #connected = false; @@ -39,7 +40,7 @@ class MySQLConnector { result.rows = query; result.count = query?.length; } catch (err) { - console.log(this.constructor.name, err); + logger.error(err.message, { origin: this.constructor.name }); result.error = err.message; } finally { await this._client.end(); diff --git a/server/utils/agents/aibitat/plugins/sql-agent/SQLConnectors/Postgresql.js b/server/utils/agents/aibitat/plugins/sql-agent/SQLConnectors/Postgresql.js index 463fea51018..2c531da7222 100644 --- a/server/utils/agents/aibitat/plugins/sql-agent/SQLConnectors/Postgresql.js +++ b/server/utils/agents/aibitat/plugins/sql-agent/SQLConnectors/Postgresql.js @@ -1,4 +1,5 @@ const pgSql = require("pg"); +const logger = require("../../../../../logger"); class PostgresSQLConnector { #connected = false; @@ -32,7 +33,7 @@ class PostgresSQLConnector { result.rows = query.rows; result.count = query.rowCount; } catch (err) { - console.log(this.constructor.name, err); + logger.error(err.message, { origin: this.constructor.name }); result.error = err.message; } finally { await this._client.end(); diff --git a/server/utils/agents/aibitat/plugins/web-browsing.js b/server/utils/agents/aibitat/plugins/web-browsing.js index 81314f178f1..df601490d42 100644 --- a/server/utils/agents/aibitat/plugins/web-browsing.js +++ b/server/utils/agents/aibitat/plugins/web-browsing.js @@ -1,4 +1,5 @@ const { SystemSettings } = require("../../../../models/systemSettings"); +const logger = require("../../../logger"); const webBrowsing = { name: "web-browsing", @@ -115,7 +116,9 @@ const webBrowsing = { }); }) .catch((e) => { - console.log(e); + logger.error(e.message, { + origin: "web-browsing._googleSearchEngine", + }); return {}; }); @@ -213,7 +216,9 @@ const webBrowsing = { })); }) .catch((e) => { - console.log(e); + logger.error(e.message, { + origin: "web-browsing._bingWebSearchEngine", + }); return []; }); diff --git a/server/utils/agents/aibitat/plugins/websocket.js b/server/utils/agents/aibitat/plugins/websocket.js index 8c8800ff3e2..2cd6fb2a7ee 100644 --- a/server/utils/agents/aibitat/plugins/websocket.js +++ b/server/utils/agents/aibitat/plugins/websocket.js @@ -1,6 +1,7 @@ const chalk = require("chalk"); const { RetryError } = require("../error"); const { Telemetry } = require("../../../../models/telemetry"); +const logger = require("../../../logger"); const SOCKET_TIMEOUT_MS = 300 * 1_000; // 5 mins /** @@ -129,10 +130,13 @@ const websocket = { }; socketTimeout = setTimeout(() => { - console.log( + logger.info( chalk.red( `Client took too long to respond, chat thread is dead after ${SOCKET_TIMEOUT_MS}ms` - ) + ), + { + origin: "websocket.js", + } ); resolve("exit"); return; diff --git a/server/utils/agents/aibitat/providers/ai-provider.js b/server/utils/agents/aibitat/providers/ai-provider.js index b3a8b1791be..67a0617ae7a 100644 --- a/server/utils/agents/aibitat/providers/ai-provider.js +++ b/server/utils/agents/aibitat/providers/ai-provider.js @@ -14,6 +14,7 @@ const { ChatOpenAI } = require("@langchain/openai"); const { ChatAnthropic } = require("@langchain/anthropic"); const { ChatOllama } = require("@langchain/community/chat_models/ollama"); const { toValidNumber } = require("../../../http"); +const logger = require("../../../logger"); const DEFAULT_WORKSPACE_PROMPT = "You are a helpful ai assistant who can assist the user and use tools available to help answer the users prompts and questions."; @@ -28,10 +29,9 @@ class Provider { } providerLog(text, ...args) { - console.log( - `\x1b[36m[AgentLLM${this?.model ? ` - ${this.model}` : ""}]\x1b[0m ${text}`, - ...args - ); + logger.info(`${this?.model ? ` - ${this.model}` : ""} ${text} ${args}`, { + origin: "AgentLLM", + }); } get client() { diff --git a/server/utils/agents/index.js b/server/utils/agents/index.js index cfa3d6a8ac2..05850b6b2b0 100644 --- a/server/utils/agents/index.js +++ b/server/utils/agents/index.js @@ -6,6 +6,7 @@ const { const { WorkspaceChats } = require("../../models/workspaceChats"); const { safeJsonParse } = require("../http"); const { USER_AGENT, WORKSPACE_AGENT } = require("./defaults"); +const logger = require("../logger"); class AgentHandler { #invocationUUID; @@ -21,11 +22,13 @@ class AgentHandler { } log(text, ...args) { - console.log(`\x1b[36m[AgentHandler]\x1b[0m ${text}`, ...args); + logger.info(text, { origin: "AgentHandler", ...args }); } closeAlert() { - this.log(`End ${this.#invocationUUID}::${this.provider}:${this.model}`); + logger.info(`End ${this.#invocationUUID}::${this.provider}:${this.model}`, { + origin: "AgentHandler", + }); } async #chatHistory(limit = 10) { @@ -62,7 +65,7 @@ class AgentHandler { }); return agentHistory; } catch (e) { - this.log("Error loading chat history", e.message); + logger.error(e.message, { origin: "AgentHandler#chatHistory" }); return []; } } @@ -186,7 +189,12 @@ class AgentHandler { this.provider = this.invocation.workspace.agentProvider || "openai"; this.model = this.invocation.workspace.agentModel || this.#providerDefault(); - this.log(`Start ${this.#invocationUUID}::${this.provider}:${this.model}`); + logger.info( + `Start ${this.#invocationUUID}::${this.provider}:${this.model}`, + { + origin: "AgentHandler#providerSetupAndCheck", + } + ); this.#checkSetup(); } @@ -206,8 +214,9 @@ class AgentHandler { definition.required && (!args.hasOwnProperty(param) || args[param] === null) ) { - this.log( - `'${param}' required parameter for '${pluginName}' plugin is missing. Plugin may not function or crash agent.` + logger.info( + `'${param}' required parameter for '${pluginName}' plugin is missing. Plugin may not function or crash agent.`, + { origin: "AgentHandler#parseCallOptions" } ); continue; } @@ -224,8 +233,9 @@ class AgentHandler { if (name.includes("#")) { const [parent, childPluginName] = name.split("#"); if (!AgentPlugins.hasOwnProperty(parent)) { - this.log( - `${parent} is not a valid plugin. Skipping inclusion to agent cluster.` + logger.info( + `${parent} is not a valid plugin. Skipping inclusion to agent cluster.`, + { origin: "AgentHandler#attachPlugins" } ); continue; } @@ -234,8 +244,9 @@ class AgentHandler { (child) => child.name === childPluginName ); if (!childPlugin) { - this.log( - `${parent} does not have child plugin named ${childPluginName}. Skipping inclusion to agent cluster.` + logger.info( + `${parent} does not have child plugin named ${childPluginName}. Skipping inclusion to agent cluster.`, + { origin: "AgentHandler#attachPlugins" } ); continue; } @@ -246,16 +257,18 @@ class AgentHandler { name ); this.aibitat.use(childPlugin.plugin(callOpts)); - this.log( - `Attached ${parent}:${childPluginName} plugin to Agent cluster` + logger.info( + `Attached ${parent}:${childPluginName} plugin to Agent cluster`, + { origin: "AgentHandler#attachPlugins" } ); continue; } // Load single-stage plugin. if (!AgentPlugins.hasOwnProperty(name)) { - this.log( - `${name} is not a valid plugin. Skipping inclusion to agent cluster.` + logger.info( + `${name} is not a valid plugin. Skipping inclusion to agent cluster.`, + { origin: "AgentHandler#attachPlugins" } ); continue; } @@ -266,13 +279,17 @@ class AgentHandler { ); const AIbitatPlugin = AgentPlugins[name]; this.aibitat.use(AIbitatPlugin.plugin(callOpts)); - this.log(`Attached ${name} plugin to Agent cluster`); + logger.info(`Attached ${name} plugin to Agent cluster`, { + origin: "AgentHandler#attachPlugins", + }); } } async #loadAgents() { // Default User agent and workspace agent - this.log(`Attaching user and default agent to Agent cluster.`); + logger.info(`Attaching user and default agent to Agent cluster.`, { + origin: "AgentHandler#loadAgents", + }); this.aibitat.agent(USER_AGENT.name, await USER_AGENT.getDefinition()); this.aibitat.agent( WORKSPACE_AGENT.name, @@ -307,7 +324,12 @@ class AgentHandler { }); // Attach standard websocket plugin for frontend communication. - this.log(`Attached ${AgentPlugins.websocket.name} plugin to Agent cluster`); + logger.info( + `Attached ${AgentPlugins.websocket.name} plugin to Agent cluster`, + { + origin: "AgentHandler#createAIbitat", + } + ); this.aibitat.use( AgentPlugins.websocket.plugin({ socket: args.socket, @@ -317,8 +339,11 @@ class AgentHandler { ); // Attach standard chat-history plugin for message storage. - this.log( - `Attached ${AgentPlugins.chatHistory.name} plugin to Agent cluster` + logger.info( + `Attached ${AgentPlugins.chatHistory.name} plugin to Agent cluster`, + { + origin: "AgentHandler#createAIbitat", + } ); this.aibitat.use(AgentPlugins.chatHistory.plugin()); diff --git a/server/utils/boot/index.js b/server/utils/boot/index.js index 2022f66e852..1a52e881e6c 100644 --- a/server/utils/boot/index.js +++ b/server/utils/boot/index.js @@ -1,11 +1,14 @@ +const chalk = require("chalk"); const { Telemetry } = require("../../models/telemetry"); const { CommunicationKey } = require("../comKey"); +const logger = require("../logger"); const setupTelemetry = require("../telemetry"); function bootSSL(app, port = 3001) { try { - console.log( - `\x1b[33m[SSL BOOT ENABLED]\x1b[0m Loading the certificate and key for HTTPS mode...` + logger.info( + chalk.yellow(`Loading the certificate and key for HTTPS mode...`), + { origin: "bootSSL" } ); const fs = require("fs"); const https = require("https"); @@ -18,21 +21,20 @@ function bootSSL(app, port = 3001) { .listen(port, async () => { await setupTelemetry(); new CommunicationKey(true); - console.log(`Primary server in HTTPS mode listening on port ${port}`); + logger.info(`Primary server in HTTPS mode listening on port ${port}`, { + origin: "bootSSL", + }); }) .on("error", catchSigTerms); require("express-ws")(app, server); // Apply same certificate + server for WSS connections return { app, server }; } catch (e) { - console.error( - `\x1b[31m[SSL BOOT FAILED]\x1b[0m ${e.message} - falling back to HTTP boot.`, - { - ENABLE_HTTPS: process.env.ENABLE_HTTPS, - HTTPS_KEY_PATH: process.env.HTTPS_KEY_PATH, - HTTPS_CERT_PATH: process.env.HTTPS_CERT_PATH, - stacktrace: e.stack, - } + logger.error( + chalk.red( + `[SSL BOOT FAILED] ${e.message} - falling back to HTTP boot. ${{ ENABLE_HTTPS: process.env.ENABLE_HTTPS, HTTPS_KEY_PATH: process.env.HTTPS_KEY_PATH, HTTPS_CERT_PATH: process.env.HTTPS_CERT_PATH }}` + ), + { origin: "bootSSL" } ); return bootHTTP(app, port); } @@ -45,7 +47,9 @@ function bootHTTP(app, port = 3001) { .listen(port, async () => { await setupTelemetry(); new CommunicationKey(true); - console.log(`Primary server in HTTP mode listening on port ${port}`); + logger.info(`Primary server in HTTP mode listening on port ${port}`, { + origin: "bootHTTP", + }); }) .on("error", catchSigTerms); diff --git a/server/utils/chats/embed.js b/server/utils/chats/embed.js index 8488aedd514..f8cbb1b76c9 100644 --- a/server/utils/chats/embed.js +++ b/server/utils/chats/embed.js @@ -7,6 +7,7 @@ const { writeResponseChunk, } = require("../helpers/chat/responses"); const { DocumentManager } = require("../DocumentManager"); +const logger = require("../logger"); async function streamChatWithForEmbed( response, @@ -165,8 +166,11 @@ async function streamChatWithForEmbed( // If streaming is not explicitly enabled for connector // we do regular waiting of a response and send a single chunk. if (LLMConnector.streamingEnabled() !== true) { - console.log( - `\x1b[31m[STREAMING DISABLED]\x1b[0m Streaming is not available for ${LLMConnector.constructor.name}. Will use regular chat method.` + logger.info( + `Streaming is not available for ${LLMConnector.constructor.name}. Will use regular chat method.`, + { + origin: "streamChatWithForEmbed", + } ); completeText = await LLMConnector.getChatCompletion(messages, { temperature: embed.workspace?.openAiTemp ?? LLMConnector.defaultTemp, diff --git a/server/utils/chats/stream.js b/server/utils/chats/stream.js index ced9a971094..4478b2b66aa 100644 --- a/server/utils/chats/stream.js +++ b/server/utils/chats/stream.js @@ -11,6 +11,8 @@ const { recentChatHistory, sourceIdentifier, } = require("./index"); +const chalk = require("chalk"); +const logger = require("../logger"); const VALID_CHAT_MODE = ["chat", "query"]; @@ -205,8 +207,13 @@ async function streamChatWithWorkspace( // If streaming is not explicitly enabled for connector // we do regular waiting of a response and send a single chunk. if (LLMConnector.streamingEnabled() !== true) { - console.log( - `\x1b[31m[STREAMING DISABLED]\x1b[0m Streaming is not available for ${LLMConnector.constructor.name}. Will use regular chat method.` + logger.info( + chalk.yellow( + `Streaming is not available for ${LLMConnector.constructor.name}. Will use regular chat method.` + ), + { + origin: "streamChatWithWorkspace", + } ); completeText = await LLMConnector.getChatCompletion(messages, { temperature: workspace?.openAiTemp ?? LLMConnector.defaultTemp, diff --git a/server/utils/collectorApi/index.js b/server/utils/collectorApi/index.js index 6971f640d20..b45f685f74a 100644 --- a/server/utils/collectorApi/index.js +++ b/server/utils/collectorApi/index.js @@ -3,6 +3,8 @@ // so no additional security is needed on the endpoint directly. Auth is done however by the express // middleware prior to leaving the node-side of the application so that is good enough >:) +const logger = require("../logger"); + class CollectorApi { constructor() { const { CommunicationKey } = require("../comKey"); @@ -11,7 +13,7 @@ class CollectorApi { } log(text, ...args) { - console.log(`\x1b[36m[CollectorApi]\x1b[0m ${text}`, ...args); + logger.info(`${text} ${args}`, { origin: "CollectorApi" }); } #attachOptions() { diff --git a/server/utils/comKey/index.js b/server/utils/comKey/index.js index aec436bbd69..901a62bd788 100644 --- a/server/utils/comKey/index.js +++ b/server/utils/comKey/index.js @@ -1,6 +1,7 @@ const crypto = require("crypto"); const fs = require("fs"); const path = require("path"); +const logger = require("../logger"); const keyPath = process.env.NODE_ENV === "development" ? path.resolve(__dirname, `../../storage/comkey`) @@ -30,7 +31,7 @@ class CommunicationKey { } log(text, ...args) { - console.log(`\x1b[36m[CommunicationKey]\x1b[0m ${text}`, ...args); + logger.info(`${text} ${args}`, { origin: "CommunicationKey" }); } #readPrivateKey() { diff --git a/server/utils/database/index.js b/server/utils/database/index.js index 75f5f7116de..fc7a34cf5a8 100644 --- a/server/utils/database/index.js +++ b/server/utils/database/index.js @@ -1,5 +1,7 @@ +const chalk = require("chalk"); const { getGitVersion } = require("../../endpoints/utils"); const { Telemetry } = require("../../models/telemetry"); +const logger = require("../logger"); function checkColumnTemplate(tablename = null, column = null) { if (!tablename || !column) @@ -35,7 +37,9 @@ async function checkForMigrations(model, db) { if (toMigrate.length === 0) return; - console.log(`Running ${toMigrate.length} migrations`, toMigrate); + logger.info(`Running ${toMigrate.length} migrations ${toMigrate}`, { + origin: "checkForMigrations", + }); await db.exec(toMigrate.join(";\n")); return; } @@ -48,9 +52,9 @@ async function checkForMigrations(model, db) { async function validateTablePragmas(force = false) { try { if (process.env.NODE_ENV !== "development" && force === false) { - console.log( - `\x1b[34m[MIGRATIONS STUBBED]\x1b[0m Please ping /migrate once server starts to run migrations` - ); + logger.info("Please ping /migrate once server starts to run migrations", { + origin: "MIGRATIONS STUBBED", + }); return; } const { SystemSettings } = require("../../models/systemSettings"); @@ -86,21 +90,31 @@ async function validateTablePragmas(force = false) { // You can see all Telemetry events by ctrl+f `Telemetry.sendEvent` calls to verify this claim. async function setupTelemetry() { if (process.env.DISABLE_TELEMETRY === "true") { - console.log( - `\x1b[31m[TELEMETRY DISABLED]\x1b[0m Telemetry is marked as disabled - no events will send. Telemetry helps Mintplex Labs Inc improve AnythingLLM.` + logger.info( + chalk.red( + "Telemetry is marked as disabled - no events will send. Telemetry helps Mintplex Labs Inc improve AnythingLLM." + ), + { + origin: "TELEMETRY DISABLED", + } ); return true; } if (Telemetry.isDev()) { - console.log( - `\x1b[33m[TELEMETRY STUBBED]\x1b[0m Anonymous Telemetry stubbed in development.` - ); + logger.info(chalk.yellow("Anonymous Telemetry stubbed in development."), { + origin: "TELEMETRY STUBBED", + }); return; } - console.log( - `\x1b[32m[TELEMETRY ENABLED]\x1b[0m Anonymous Telemetry enabled. Telemetry helps Mintplex Labs Inc improve AnythingLLM.` + logger.info( + chalk.green( + "Anonymous Telemetry enabled. Telemetry helps Mintplex Labs Inc improve AnythingLLM." + ), + { + origin: "TELEMETRY ENABLED", + } ); await Telemetry.findOrCreateId(); await Telemetry.sendTelemetry("server_boot", { diff --git a/server/utils/helpers/chat/index.js b/server/utils/helpers/chat/index.js index 6f565efe14e..6c589cf0546 100644 --- a/server/utils/helpers/chat/index.js +++ b/server/utils/helpers/chat/index.js @@ -1,5 +1,6 @@ const { sourceIdentifier } = require("../../chats"); const { safeJsonParse } = require("../../http"); +const logger = require("../../logger"); const { TokenManager } = require("../tiktoken"); const { convertToPromptHistory } = require("./responses"); @@ -21,7 +22,7 @@ You may think: "Doesn't this result in massive data loss?" - yes & no. Under the use cases we expect the tool to be used, which is mostly chatting with documents, we are able to use this approach with minimal blowback on the quality of responses. -We accomplish this by taking a rate-limit approach that is proportional to the model capacity. Since we support more than openAI models, this needs to +We accomplish this by taking a rate-limit approach that is proportional to the model capacity. Since we support more than openAI models, this needs to be generic and reliance on a "better summary" model just is not a luxury we can afford. The added latency overhead during prompting is also unacceptable. In general: system: at best 15% of token capacity @@ -37,7 +38,7 @@ we handle overflows by taking an aggressive path for two main cases. 2. Context window is exceeded in regular use. - We do not touch prompt since it is very likely to be <70% of window. - We check system prompt is not outrageous - if it is we cannonball it and keep context if present. -- We check a sliding window of history, only allowing up to 15% of the history to pass through if it fits, with a +- We check a sliding window of history, only allowing up to 15% of the history to pass through if it fits, with a preference for recent history if we can cannonball to fit it, otherwise it is omitted. We end up with a rather large prompt that fits through a given window with a lot of room for response in most use-cases. @@ -337,10 +338,11 @@ function cannonball({ truncText + tokenManager.bytesFromTokens(rightChunks); - console.log( + logger.info( `Cannonball results ${initialInputSize} -> ${tokenManager.countFromString( truncatedText - )} tokens.` + )} tokens.`, + { origin: "cannonball" } ); return truncatedText; } @@ -395,7 +397,7 @@ function fillSourceWindow({ } const log = (text, ...args) => { - console.log(`\x1b[36m[fillSourceWindow]\x1b[0m ${text}`, ...args); + logger.info(`${text} ${args}`, { origin: "fillSourceWindow" }); }; log( diff --git a/server/utils/helpers/chat/responses.js b/server/utils/helpers/chat/responses.js index 609b18190fe..a321408bb5f 100644 --- a/server/utils/helpers/chat/responses.js +++ b/server/utils/helpers/chat/responses.js @@ -1,9 +1,16 @@ const { v4: uuidv4 } = require("uuid"); const moment = require("moment"); +const logger = require("../../logger"); +const chalk = require("chalk"); function clientAbortedHandler(resolve, fullText) { - console.log( - "\x1b[43m\x1b[34m[STREAM ABORTED]\x1b[0m Client requested to abort stream. Exiting LLM stream handler early." + logger.info( + chalk.red( + "Client requested to abort stream. Exiting LLM stream handler early.", + { + origin: "STREAM ABORTED", + } + ) ); resolve(fullText); return; diff --git a/server/utils/prisma/migrateFromSqlite.js b/server/utils/prisma/migrateFromSqlite.js index 4e5e379cf45..7f6d65391cc 100644 --- a/server/utils/prisma/migrateFromSqlite.js +++ b/server/utils/prisma/migrateFromSqlite.js @@ -2,6 +2,7 @@ const { PrismaClient } = require("@prisma/client"); const execSync = require("child_process").execSync; const fs = require("fs"); const path = require("path"); +const logger = require("../logger"); require("dotenv").config(); const DATABASE_PATH = process.env.DB_URL || "../../storage/anythingllm.db"; @@ -14,7 +15,9 @@ const BACKUP_PATH = path.join( function backupDatabase() { try { fs.copyFileSync(DATABASE_PATH, BACKUP_PATH); - console.log("Database backup created successfully."); + logger.info("Database backup created successfully.", { + origin: "migrateFromSqlite.js", + }); } catch (error) { console.error("Failed to create database backup:", error); } @@ -27,14 +30,18 @@ const prisma = new PrismaClient(); // Reset the prisma database and prepare it for migration of data from sqlite function resetAndMigrateDatabase() { try { - console.log("Resetting and migrating the database..."); + logger.info("Resetting and migrating the database...", { + origin: "migrateFromSqlite.js", + }); execSync("cd ../.. && npx prisma migrate reset --skip-seed --force", { stdio: "inherit", }); execSync("cd ../.. && npx prisma migrate dev --name init", { stdio: "inherit", }); - console.log("Database reset and initial migration completed successfully"); + logger.info("Database reset and initial migration completed successfully", { + origin: "migrateFromSqlite.js", + }); } catch (error) { console.error("Failed to reset and migrate the database:", error); } @@ -45,7 +52,9 @@ resetAndMigrateDatabase(); // Migrate data from sqlite to prisma async function migrateData() { try { - console.log("Starting data migration..."); + logger.info("Starting data migration...", { + origin: "migrateFromSqlite.js", + }); var legacyMap = { users: { count: 0, @@ -234,7 +243,9 @@ async function migrateData() { }); }); - console.log("Data migration completed successfully"); + logger.info("Data migration completed successfully", { + origin: "migrateFromSqlite.js", + }); } catch (error) { console.error("Data migration failed:", error); } finally { @@ -255,8 +266,11 @@ async function migrateTable(tableName, migrateRowFunc) { `SELECT COUNT(*) as count FROM sqlite_master WHERE name='${tableName}'` ); if (count === 0) { - console.log( - `${tableName} does not exist in legacy DB - nothing to migrate - skipping.` + logger.info( + `${tableName} does not exist in legacy DB - nothing to migrate - skipping.`, + { + origin: "migrateFromSqlite.js", + } ); return; } @@ -270,8 +284,14 @@ async function migrateTable(tableName, migrateRowFunc) { upserts.push(row); } } catch (e) { - console.error(e); - console.log({ tableName, upserts }); + logger.error( + `Error migrating ${tableName}: ${e.message} - ${JSON.stringify( + e + )} - ${JSON.stringify(upserts)}`, + { + origin: "migrateFromSqlite.js", + } + ); } finally { await db.close(); } diff --git a/server/utils/telemetry/index.js b/server/utils/telemetry/index.js index 1945de3fe88..52bb7055929 100644 --- a/server/utils/telemetry/index.js +++ b/server/utils/telemetry/index.js @@ -1,5 +1,7 @@ +const chalk = require("chalk"); const { getGitVersion } = require("../../endpoints/utils"); const { Telemetry } = require("../../models/telemetry"); +const logger = require("../logger"); // Telemetry is anonymized and your data is never read. This can be disabled by setting // DISABLE_TELEMETRY=true in the `.env` of however you setup. Telemetry helps us determine use @@ -7,21 +9,35 @@ const { Telemetry } = require("../../models/telemetry"); // You can see all Telemetry events by ctrl+f `Telemetry.sendEvent` calls to verify this claim. async function setupTelemetry() { if (process.env.DISABLE_TELEMETRY === "true") { - console.log( - `\x1b[31m[TELEMETRY DISABLED]\x1b[0m Telemetry is marked as disabled - no events will send. Telemetry helps Mintplex Labs Inc improve AnythingLLM.` + logger.info( + chalk.red( + "Telemetry is marked as disabled - no events will send. Telemetry helps Mintplex Labs Inc improve AnythingLLM." + ), + { + origin: "TELEMETRY DISABLED", + } ); return true; } if (Telemetry.isDev()) { - console.log( - `\x1b[33m[TELEMETRY STUBBED]\x1b[0m Anonymous Telemetry stubbed in development.` + logger.info( + chalk.yellow( + "Telemetry is marked as stubbed in development - no events will send. Telemetry helps Mintplex Labs Inc improve AnythingLLM." + ), + { + origin: "TELEMETRY STUBBED", + } ); return; } - - console.log( - `\x1b[32m[TELEMETRY ENABLED]\x1b[0m Anonymous Telemetry enabled. Telemetry helps Mintplex Labs Inc improve AnythingLLM.` + logger.info( + chalk.green( + "Anonymous Telemetry enabled. Telemetry helps Mintplex Labs Inc improve AnythingLLM." + ), + { + origin: "TELEMETRY ENABLED", + } ); await Telemetry.findOrCreateId(); await Telemetry.sendTelemetry("server_boot", {