From a028afbf7e1a571fee4ff05067fd7f556c46f245 Mon Sep 17 00:00:00 2001 From: shatfield4 Date: Tue, 8 Jul 2025 17:36:54 -0700 Subject: [PATCH 1/4] wip handle bigints in message response --- server/models/workspaceChats.js | 3 ++- server/utils/chats/stream.js | 7 ++++++- server/utils/helpers/chat/responses.js | 14 +++++++++++++- 3 files changed, 21 insertions(+), 3 deletions(-) diff --git a/server/models/workspaceChats.js b/server/models/workspaceChats.js index dde98d1e899..e48807be71d 100644 --- a/server/models/workspaceChats.js +++ b/server/models/workspaceChats.js @@ -1,4 +1,5 @@ const prisma = require("../utils/prisma"); +const { safeJSONStringify } = require("../utils/helpers/chat/responses"); const WorkspaceChats = { new: async function ({ @@ -15,7 +16,7 @@ const WorkspaceChats = { data: { workspaceId, prompt, - response: JSON.stringify(response), + response: safeJSONStringify(response), user_id: user?.id || null, thread_id: threadId, api_session_id: apiSessionId, diff --git a/server/utils/chats/stream.js b/server/utils/chats/stream.js index 0ecf86c8141..3eaaf87b4f8 100644 --- a/server/utils/chats/stream.js +++ b/server/utils/chats/stream.js @@ -256,6 +256,11 @@ async function streamChatWithWorkspace( } if (completeText?.length > 0) { + // BigInt test + if (process.env.NODE_ENV === 'development') { + metrics.test_bigint = BigInt(Number.MAX_SAFE_INTEGER) + BigInt(1); + } + const { chat } = await WorkspaceChats.new({ workspaceId: workspace.id, prompt: message, @@ -275,7 +280,7 @@ async function streamChatWithWorkspace( type: "finalizeResponseStream", close: true, error: false, - chatId: chat.id, + chatId: chat?.id || null, metrics, }); return; diff --git a/server/utils/helpers/chat/responses.js b/server/utils/helpers/chat/responses.js index 16a1e9af439..957e3320b05 100644 --- a/server/utils/helpers/chat/responses.js +++ b/server/utils/helpers/chat/responses.js @@ -207,8 +207,19 @@ function convertToPromptHistory(history = []) { return formattedHistory.flat(); } +/** + * Safely stringifies an object containing BigInt values + * @param {Object} obj - Object to stringify + * @returns {string} JSON string with BigInt values converted to strings + */ +function safeJSONStringify(obj) { + return JSON.stringify(obj, (_, value) => + typeof value === 'bigint' ? value.toString() : value + ); +} + function writeResponseChunk(response, data) { - response.write(`data: ${JSON.stringify(data)}\n\n`); + response.write(`data: ${safeJSONStringify(data)}\n\n`); return; } @@ -262,4 +273,5 @@ module.exports = { writeResponseChunk, clientAbortedHandler, formatChatHistory, + safeJSONStringify, }; From b5a8880ecaf741c1980146f7b868c25bc8d8fe91 Mon Sep 17 00:00:00 2001 From: shatfield4 Date: Tue, 8 Jul 2025 18:14:37 -0700 Subject: [PATCH 2/4] extend bigint protoype to handle bigint stringification + add test --- .../safeJSONStringify.test.js | 47 +++++++++++++++++++ server/utils/chats/stream.js | 5 -- server/utils/helpers/chat/responses.js | 23 +++++++-- 3 files changed, 65 insertions(+), 10 deletions(-) create mode 100644 server/__tests__/utils/safeJSONStringify/safeJSONStringify.test.js diff --git a/server/__tests__/utils/safeJSONStringify/safeJSONStringify.test.js b/server/__tests__/utils/safeJSONStringify/safeJSONStringify.test.js new file mode 100644 index 00000000000..8f77b464d66 --- /dev/null +++ b/server/__tests__/utils/safeJSONStringify/safeJSONStringify.test.js @@ -0,0 +1,47 @@ +/* eslint-env jest */ +const { safeJSONStringify } = require("../../../utils/helpers/chat/responses"); + +describe("safeJSONStringify", () => { + test("handles regular objects without BigInt", () => { + const obj = { a: 1, b: "test", c: true, d: null }; + expect(safeJSONStringify(obj)).toBe(JSON.stringify(obj)); + }); + + test("converts BigInt to string", () => { + const bigInt = BigInt(Number.MAX_SAFE_INTEGER) + BigInt(1); + expect(safeJSONStringify(bigInt)).toBe(`"${bigInt.toString()}"`); + }); + + test("handles nested BigInt values", () => { + const obj = { + metrics: { + tokens: BigInt(123), + nested: { moreBigInt: BigInt(456) } + }, + normal: "value" + }; + expect(safeJSONStringify(obj)).toBe( + '{"metrics":{"tokens":"123","nested":{"moreBigInt":"456"}},"normal":"value"}' + ); + }); + + test("handles arrays with BigInt", () => { + const arr = [BigInt(1), 2, BigInt(3)]; + expect(safeJSONStringify(arr)).toBe('["1",2,"3"]'); + }); + + test("handles mixed complex objects", () => { + const obj = { + id: 1, + bigNums: [BigInt(123), BigInt(456)], + nested: { + more: { huge: BigInt(Number.MAX_SAFE_INTEGER) + BigInt(1) } + }, + normal: { str: "test", num: 42, bool: true, nil: null } + }; + const result = JSON.parse(safeJSONStringify(obj)); // Should parse back without errors + expect(typeof result.bigNums[0]).toBe("string"); + expect(typeof result.nested.more.huge).toBe("string"); + expect(result.normal).toEqual({ str: "test", num: 42, bool: true, nil: null }); + }); +}); \ No newline at end of file diff --git a/server/utils/chats/stream.js b/server/utils/chats/stream.js index 3eaaf87b4f8..ef5dee97676 100644 --- a/server/utils/chats/stream.js +++ b/server/utils/chats/stream.js @@ -256,11 +256,6 @@ async function streamChatWithWorkspace( } if (completeText?.length > 0) { - // BigInt test - if (process.env.NODE_ENV === 'development') { - metrics.test_bigint = BigInt(Number.MAX_SAFE_INTEGER) + BigInt(1); - } - const { chat } = await WorkspaceChats.new({ workspaceId: workspace.id, prompt: message, diff --git a/server/utils/helpers/chat/responses.js b/server/utils/helpers/chat/responses.js index 957e3320b05..d1e0598b6e9 100644 --- a/server/utils/helpers/chat/responses.js +++ b/server/utils/helpers/chat/responses.js @@ -208,14 +208,27 @@ function convertToPromptHistory(history = []) { } /** - * Safely stringifies an object containing BigInt values - * @param {Object} obj - Object to stringify + * Safely stringifies any object containing BigInt values + * @param {*} obj - Anything to stringify that might contain BigInt values * @returns {string} JSON string with BigInt values converted to strings */ function safeJSONStringify(obj) { - return JSON.stringify(obj, (_, value) => - typeof value === 'bigint' ? value.toString() : value - ); + // Temporarily extend BigInt prototype for this stringify operation + const originalToJSON = BigInt.prototype.toJSON; + BigInt.prototype.toJSON = function () { + return this.toString(); + }; + + try { + return JSON.stringify(obj); + } finally { + // Restore original behavior + if (originalToJSON) { + BigInt.prototype.toJSON = originalToJSON; + } else { + delete BigInt.prototype.toJSON; + } + } } function writeResponseChunk(response, data) { From b6a81f8df9e14cb81f137cd06855344c1c483cca Mon Sep 17 00:00:00 2001 From: timothycarambat Date: Thu, 10 Jul 2025 12:23:14 -0700 Subject: [PATCH 3/4] unset unrelated file --- server/utils/chats/stream.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/utils/chats/stream.js b/server/utils/chats/stream.js index ef5dee97676..0ecf86c8141 100644 --- a/server/utils/chats/stream.js +++ b/server/utils/chats/stream.js @@ -275,7 +275,7 @@ async function streamChatWithWorkspace( type: "finalizeResponseStream", close: true, error: false, - chatId: chat?.id || null, + chatId: chat.id, metrics, }); return; From 0809c1ee5a3e42eaea1b246dbaada1250e85b0bd Mon Sep 17 00:00:00 2001 From: timothycarambat Date: Thu, 10 Jul 2025 12:33:08 -0700 Subject: [PATCH 4/4] update tests, simplify implementation; --- .../YoutubeLoader/youtube-transcript.test.js | 8 +++----- .../safeJSONStringify.test.js | 17 ++++++++++++++-- server/utils/helpers/chat/responses.js | 20 ++++--------------- 3 files changed, 22 insertions(+), 23 deletions(-) diff --git a/collector/__tests__/utils/extensions/YoutubeTranscript/YoutubeLoader/youtube-transcript.test.js b/collector/__tests__/utils/extensions/YoutubeTranscript/YoutubeLoader/youtube-transcript.test.js index 2423e736bee..1fca742fd53 100644 --- a/collector/__tests__/utils/extensions/YoutubeTranscript/YoutubeLoader/youtube-transcript.test.js +++ b/collector/__tests__/utils/extensions/YoutubeTranscript/YoutubeLoader/youtube-transcript.test.js @@ -10,9 +10,7 @@ describe("YoutubeTranscript", () => { expect(transcript).toBeDefined(); expect(typeof transcript).toBe("string"); expect(transcript.length).toBeGreaterThan(0); - - // Log the results for debugging purposes - console.log("Success! Transcript length:", transcript.length); - console.log("First 200 characters:", transcript.substring(0, 200) + "..."); - }, 30000); // 30 second timeout for network request + // console.log("Success! Transcript length:", transcript.length); + // console.log("First 200 characters:", transcript.substring(0, 200) + "..."); + }, 30000); }); diff --git a/server/__tests__/utils/safeJSONStringify/safeJSONStringify.test.js b/server/__tests__/utils/safeJSONStringify/safeJSONStringify.test.js index 8f77b464d66..2165698035f 100644 --- a/server/__tests__/utils/safeJSONStringify/safeJSONStringify.test.js +++ b/server/__tests__/utils/safeJSONStringify/safeJSONStringify.test.js @@ -37,11 +37,24 @@ describe("safeJSONStringify", () => { nested: { more: { huge: BigInt(Number.MAX_SAFE_INTEGER) + BigInt(1) } }, - normal: { str: "test", num: 42, bool: true, nil: null } + normal: { str: "test", num: 42, bool: true, nil: null, sub_arr: ["alpha", "beta", "gamma", 1, 2, BigInt(Number.MAX_SAFE_INTEGER) + BigInt(1), { map: { a: BigInt(Number.MAX_SAFE_INTEGER) + BigInt(1) } }] } }; const result = JSON.parse(safeJSONStringify(obj)); // Should parse back without errors expect(typeof result.bigNums[0]).toBe("string"); + expect(result.bigNums[0]).toEqual("123"); expect(typeof result.nested.more.huge).toBe("string"); - expect(result.normal).toEqual({ str: "test", num: 42, bool: true, nil: null }); + expect(result.normal).toEqual({ str: "test", num: 42, bool: true, nil: null, sub_arr: ["alpha", "beta", "gamma", 1, 2, (BigInt(Number.MAX_SAFE_INTEGER) + BigInt(1)).toString(), { map: { a: (BigInt(Number.MAX_SAFE_INTEGER) + BigInt(1)).toString() } }] }); + expect(result.normal.sub_arr[6].map.a).toEqual((BigInt(Number.MAX_SAFE_INTEGER) + BigInt(1)).toString()); + }); + + test("handles invariants", () => { + expect(safeJSONStringify({})).toBe("{}"); + expect(safeJSONStringify(null)).toBe("null"); + expect(safeJSONStringify(undefined)).toBe(undefined); + expect(safeJSONStringify(true)).toBe("true"); + expect(safeJSONStringify(false)).toBe("false"); + expect(safeJSONStringify(0)).toBe("0"); + expect(safeJSONStringify(1)).toBe("1"); + expect(safeJSONStringify(-1)).toBe("-1"); }); }); \ No newline at end of file diff --git a/server/utils/helpers/chat/responses.js b/server/utils/helpers/chat/responses.js index d1e0598b6e9..465d56fd2f6 100644 --- a/server/utils/helpers/chat/responses.js +++ b/server/utils/helpers/chat/responses.js @@ -213,22 +213,10 @@ function convertToPromptHistory(history = []) { * @returns {string} JSON string with BigInt values converted to strings */ function safeJSONStringify(obj) { - // Temporarily extend BigInt prototype for this stringify operation - const originalToJSON = BigInt.prototype.toJSON; - BigInt.prototype.toJSON = function () { - return this.toString(); - }; - - try { - return JSON.stringify(obj); - } finally { - // Restore original behavior - if (originalToJSON) { - BigInt.prototype.toJSON = originalToJSON; - } else { - delete BigInt.prototype.toJSON; - } - } + return JSON.stringify(obj, (_, value) => { + if (typeof value === "bigint") return value.toString(); + return value; + }); } function writeResponseChunk(response, data) {