From ac29747ac3a2cbd9cd45abb872f213dc32591008 Mon Sep 17 00:00:00 2001
From: timothycarambat <rambat1010@gmail.com>
Date: Thu, 4 Jan 2024 15:27:09 -0800
Subject: [PATCH 1/2] Handle special token in TikToken resolves #525

---
 server/utils/helpers/tiktoken.js | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/server/utils/helpers/tiktoken.js b/server/utils/helpers/tiktoken.js
index ad1cdd44437..862b9b866e9 100644
--- a/server/utils/helpers/tiktoken.js
+++ b/server/utils/helpers/tiktoken.js
@@ -17,7 +17,7 @@ class TokenManager {
   }
 
   tokensFromString(input = "") {
-    const tokens = this.encoder.encode(input);
+    const tokens = this.encoder.encode(input, undefined, []);
     return tokens;
   }
 
@@ -27,7 +27,7 @@ class TokenManager {
   }
 
   countFromString(input = "") {
-    const tokens = this.encoder.encode(input);
+    const tokens = this.encoder.encode(input, undefined, []);
     return tokens.length;
   }
 

From 9264d527795b381292d83d78edac78fb9c4323da Mon Sep 17 00:00:00 2001
From: timothycarambat <rambat1010@gmail.com>
Date: Thu, 4 Jan 2024 15:46:29 -0800
Subject: [PATCH 2/2] remove duplicate method add clarification comment on
 implementation

---
 server/utils/helpers/chat/index.js |  2 +-
 server/utils/helpers/tiktoken.js   | 12 ++++--------
 2 files changed, 5 insertions(+), 9 deletions(-)

diff --git a/server/utils/helpers/chat/index.js b/server/utils/helpers/chat/index.js
index ed7eab90fc8..b969201eb5e 100644
--- a/server/utils/helpers/chat/index.js
+++ b/server/utils/helpers/chat/index.js
@@ -300,7 +300,7 @@ function cannonball({
   // if the delta is the token difference between where our prompt is in size
   // and where we ideally need to land.
   const delta = initialInputSize - targetTokenSize;
-  const tokenChunks = tokenManager.tokensFromString(input);
+  const tokenChunks = tokenManager.countFromString(input);
   const middleIdx = Math.floor(tokenChunks.length / 2);
 
   // middle truncate the text going left and right of midpoint
diff --git a/server/utils/helpers/tiktoken.js b/server/utils/helpers/tiktoken.js
index 862b9b866e9..134755492da 100644
--- a/server/utils/helpers/tiktoken.js
+++ b/server/utils/helpers/tiktoken.js
@@ -3,12 +3,11 @@ const { getEncodingNameForModel, getEncoding } = require("js-tiktoken");
 class TokenManager {
   constructor(model = "gpt-3.5-turbo") {
     this.model = model;
-    this.encoderName = this.getEncodingFromModel(model);
+    this.encoderName = this.#getEncodingFromModel(model);
     this.encoder = getEncoding(this.encoderName);
-    this.buffer = 50;
   }
 
-  getEncodingFromModel(model) {
+  #getEncodingFromModel(model) {
     try {
       return getEncodingNameForModel(model);
     } catch {
@@ -16,16 +15,13 @@ class TokenManager {
     }
   }
 
-  tokensFromString(input = "") {
-    const tokens = this.encoder.encode(input, undefined, []);
-    return tokens;
-  }
-
   bytesFromTokens(tokens = []) {
     const bytes = this.encoder.decode(tokens);
     return bytes;
   }
 
+  // Pass in an empty array of disallowedSpecials to handle all tokens as text and to be tokenized.
+  // https://github.com/openai/tiktoken/blob/9e79899bc248d5313c7dd73562b5e211d728723d/tiktoken/core.py#L91C20-L91C38
   countFromString(input = "") {
     const tokens = this.encoder.encode(input, undefined, []);
     return tokens.length;