From 40c214ae574c9da152826b5748322f9938d16001 Mon Sep 17 00:00:00 2001
From: timothycarambat <rambat1010@gmail.com>
Date: Mon, 13 May 2024 22:59:28 -0700
Subject: [PATCH 1/4] Add Speech-to-text and Text-to-speech providers

---
 .vscode/settings.json                         |  1 +
 docker/.env.example                           | 13 ++++
 frontend/package.json                         |  1 +
 frontend/src/App.jsx                          |  7 ++
 .../src/components/SettingsSidebar/index.jsx  |  9 +++
 .../HistoricalMessage/Actions/index.jsx       | 65 +------------------
 .../ChatContainer/PromptInput/index.jsx       |  5 ++
 frontend/src/models/system.js                 |  2 +-
 frontend/src/models/workspace.js              | 17 ++++-
 frontend/src/utils/paths.js                   |  3 +
 frontend/yarn.lock                            |  5 ++
 server/.env.example                           | 13 ++++
 server/endpoints/workspaces.js                | 50 +++++++++++++-
 server/models/systemSettings.js               | 11 ++++
 server/package.json                           |  1 +
 server/utils/helpers/customModels.js          | 30 +++++++++
 server/utils/helpers/updateENV.js             | 31 +++++++++
 server/yarn.lock                              | 65 ++++++++++++++++++-
 18 files changed, 261 insertions(+), 68 deletions(-)

diff --git a/.vscode/settings.json b/.vscode/settings.json
index 110c4fa6ed5..4930aa2d153 100644
--- a/.vscode/settings.json
+++ b/.vscode/settings.json
@@ -11,6 +11,7 @@
     "cooldowns",
     "Deduplicator",
     "Dockerized",
+    "elevenlabs",
     "Embeddable",
     "epub",
     "GROQ",
diff --git a/docker/.env.example b/docker/.env.example
index 8cfa2aea872..70059ea5135 100644
--- a/docker/.env.example
+++ b/docker/.env.example
@@ -171,6 +171,19 @@ GID='1000'
 # WHISPER_PROVIDER="openai"
 # OPEN_AI_KEY=sk-xxxxxxxx
 
+###########################################
+######## TTS/STT Model Selection ##########
+###########################################
+# TTS_PROVIDER="native"
+
+# TTS_PROVIDER="openai"
+# TTS_OPEN_AI_KEY=sk-example
+# TTS_OPEN_AI_VOICE_MODEL=nova
+
+# TTS_PROVIDER="elevenlabs"
+# TTS_ELEVEN_LABS_KEY=
+# TTS_ELEVEN_LABS_VOICE_MODEL=21m00Tcm4TlvDq8ikWAM # Rachel
+
 # CLOUD DEPLOYMENT VARIRABLES ONLY
 # AUTH_TOKEN="hunter2" # This is the password to your application if remote hosting.
 # DISABLE_TELEMETRY="false"
diff --git a/frontend/package.json b/frontend/package.json
index ded06aa9c77..11e612fcdfd 100644
--- a/frontend/package.json
+++ b/frontend/package.json
@@ -28,6 +28,7 @@
     "react-dropzone": "^14.2.3",
     "react-loading-skeleton": "^3.1.0",
     "react-router-dom": "^6.3.0",
+    "react-speech-recognition": "^3.10.0",
     "react-tag-input-component": "^2.0.2",
     "react-toastify": "^9.1.3",
     "react-tooltip": "^5.25.2",
diff --git a/frontend/src/App.jsx b/frontend/src/App.jsx
index 0a5ed65fc85..b29e6eea925 100644
--- a/frontend/src/App.jsx
+++ b/frontend/src/App.jsx
@@ -32,6 +32,9 @@ const GeneralLLMPreference = lazy(
 const GeneralTranscriptionPreference = lazy(
   () => import("@/pages/GeneralSettings/TranscriptionPreference")
 );
+const GeneralAudioPreference = lazy(
+  () => import("@/pages/GeneralSettings/AudioPreference")
+);
 const GeneralEmbeddingPreference = lazy(
   () => import("@/pages/GeneralSettings/EmbeddingPreference")
 );
@@ -85,6 +88,10 @@ export default function App() {
                   <AdminRoute Component={GeneralTranscriptionPreference} />
                 }
               />
+              <Route
+                path="/settings/audio-preference"
+                element={<AdminRoute Component={GeneralAudioPreference} />}
+              />
               <Route
                 path="/settings/embedding-preference"
                 element={<AdminRoute Component={GeneralEmbeddingPreference} />}
diff --git a/frontend/src/components/SettingsSidebar/index.jsx b/frontend/src/components/SettingsSidebar/index.jsx
index 67797d26619..6b8f79e5edd 100644
--- a/frontend/src/components/SettingsSidebar/index.jsx
+++ b/frontend/src/components/SettingsSidebar/index.jsx
@@ -21,6 +21,7 @@ import {
   ClosedCaptioning,
   EyeSlash,
   SplitVertical,
+  Microphone,
 } from "@phosphor-icons/react";
 import useUser from "@/hooks/useUser";
 import { USER_BACKGROUND_COLOR } from "@/utils/constants";
@@ -280,6 +281,14 @@ const SidebarOptions = ({ user = null }) => (
       flex={true}
       allowedRole={["admin"]}
     />
+    <Option
+      href={paths.settings.audioPreference()}
+      btnText="Voice and Speech Support"
+      icon={<Microphone className="h-5 w-5 flex-shrink-0" />}
+      user={user}
+      flex={true}
+      allowedRole={["admin"]}
+    />
     <Option
       href={paths.settings.transcriptionPreference()}
       btnText="Transcription Model"
diff --git a/frontend/src/components/WorkspaceChat/ChatContainer/ChatHistory/HistoricalMessage/Actions/index.jsx b/frontend/src/components/WorkspaceChat/ChatContainer/ChatHistory/HistoricalMessage/Actions/index.jsx
index 3bdee472d8e..52ae1466a82 100644
--- a/frontend/src/components/WorkspaceChat/ChatContainer/ChatHistory/HistoricalMessage/Actions/index.jsx
+++ b/frontend/src/components/WorkspaceChat/ChatContainer/ChatHistory/HistoricalMessage/Actions/index.jsx
@@ -1,4 +1,4 @@
-import React, { memo, useEffect, useState } from "react";
+import React, { memo, useState } from "react";
 import useCopyText from "@/hooks/useCopyText";
 import {
   Check,
@@ -6,11 +6,10 @@ import {
   ThumbsUp,
   ThumbsDown,
   ArrowsClockwise,
-  SpeakerHigh,
-  PauseCircle,
 } from "@phosphor-icons/react";
 import { Tooltip } from "react-tooltip";
 import Workspace from "@/models/workspace";
+import TTSMessage from "./TTSButton";
 
 const Actions = ({
   message,
@@ -60,7 +59,7 @@ const Actions = ({
           </>
         )}
       </div>
-      <TTSMessage message={message} />
+      <TTSMessage slug={slug} chatId={chatId} message={message} />
     </div>
   );
 };
@@ -149,62 +148,4 @@ function RegenerateMessage({ regenerateMessage, chatId }) {
   );
 }
 
-function TTSMessage({ message }) {
-  const [speaking, setSpeaking] = useState(false);
-  const [supported, setSupported] = useState(false);
-  useEffect(() => {
-    setSupported("speechSynthesis" in window);
-  }, []);
-
-  function endSpeechUtterance() {
-    window.speechSynthesis?.cancel();
-    setSpeaking(false);
-    return;
-  }
-
-  function speakMessage() {
-    // if the user is pausing this particular message
-    // while the synth if speaking we can end it.
-    // If they are clicking another message's TTS
-    // we need to ignore that until they pause the one that is playing.
-    if (window.speechSynthesis.speaking && speaking) {
-      endSpeechUtterance();
-      return;
-    }
-
-    if (window.speechSynthesis.speaking && !speaking) return;
-    const utterance = new SpeechSynthesisUtterance(message);
-    utterance.addEventListener("end", endSpeechUtterance);
-    window.speechSynthesis.speak(utterance);
-    setSpeaking(true);
-  }
-
-  if (!supported) return null;
-  return (
-    <div className="mt-3 relative">
-      <button
-        onClick={speakMessage}
-        data-tooltip-id="message-to-speech"
-        data-tooltip-content={
-          speaking ? "Pause TTS speech of message" : "TTS Speak message"
-        }
-        className="border-none text-zinc-300"
-        aria-label={speaking ? "Pause speech" : "Speak message"}
-      >
-        {speaking ? (
-          <PauseCircle size={18} className="mb-1" />
-        ) : (
-          <SpeakerHigh size={18} className="mb-1" />
-        )}
-      </button>
-      <Tooltip
-        id="message-to-speech"
-        place="bottom"
-        delayShow={300}
-        className="tooltip !text-xs"
-      />
-    </div>
-  );
-}
-
 export default memo(Actions);
diff --git a/frontend/src/components/WorkspaceChat/ChatContainer/PromptInput/index.jsx b/frontend/src/components/WorkspaceChat/ChatContainer/PromptInput/index.jsx
index 98ad11f8f98..df08bcc7c46 100644
--- a/frontend/src/components/WorkspaceChat/ChatContainer/PromptInput/index.jsx
+++ b/frontend/src/components/WorkspaceChat/ChatContainer/PromptInput/index.jsx
@@ -12,6 +12,7 @@ import AvailableAgentsButton, {
   useAvailableAgents,
 } from "./AgentMenu";
 import TextSizeButton from "./TextSizeMenu";
+import SpeechToText from "./SpeechToText";
 
 export const PROMPT_INPUT_EVENT = "set_prompt_input";
 export default function PromptInput({
@@ -34,6 +35,7 @@ export default function PromptInput({
   function handlePromptUpdate(e) {
     setPromptInput(e?.detail ?? "");
   }
+
   useEffect(() => {
     if (!!window)
       window.addEventListener(PROMPT_INPUT_EVENT, handlePromptUpdate);
@@ -156,6 +158,9 @@ export default function PromptInput({
                 />
                 <TextSizeButton />
               </div>
+              <div className="flex gap-x-2">
+                <SpeechToText sendCommand={sendCommand} />
+              </div>
             </div>
           </div>
         </div>
diff --git a/frontend/src/models/system.js b/frontend/src/models/system.js
index e64b0119986..f8f12344849 100644
--- a/frontend/src/models/system.js
+++ b/frontend/src/models/system.js
@@ -332,7 +332,7 @@ const System = {
       })
       .then((blob) => (blob ? URL.createObjectURL(blob) : null))
       .catch((e) => {
-        console.log(e);
+        // console.log(e);
         return null;
       });
   },
diff --git a/frontend/src/models/workspace.js b/frontend/src/models/workspace.js
index 91f4a2db332..64732c0441e 100644
--- a/frontend/src/models/workspace.js
+++ b/frontend/src/models/workspace.js
@@ -272,6 +272,21 @@ const Workspace = {
         return false;
       });
   },
+  ttsMessage: async function (slug, chatId) {
+    return await fetch(`${API_BASE}/workspace/${slug}/tts/${chatId}`, {
+      method: "GET",
+      cache: "no-cache",
+      headers: baseHeaders(),
+    })
+      .then((res) => {
+        if (res.ok && res.status !== 204) return res.blob();
+        throw new Error("Failed to fetch TTS.");
+      })
+      .then((blob) => (blob ? URL.createObjectURL(blob) : null))
+      .catch((e) => {
+        return null;
+      });
+  },
   threads: WorkspaceThread,
 
   uploadPfp: async function (formData, slug) {
@@ -302,7 +317,7 @@ const Workspace = {
       })
       .then((blob) => (blob ? URL.createObjectURL(blob) : null))
       .catch((e) => {
-        console.log(e);
+        // console.log(e);
         return null;
       });
   },
diff --git a/frontend/src/utils/paths.js b/frontend/src/utils/paths.js
index 4dc4d528504..cc2b69eee57 100644
--- a/frontend/src/utils/paths.js
+++ b/frontend/src/utils/paths.js
@@ -98,6 +98,9 @@ export default {
     transcriptionPreference: () => {
       return "/settings/transcription-preference";
     },
+    audioPreference: () => {
+      return "/settings/audio-preference";
+    },
     embedder: {
       modelPreference: () => "/settings/embedding-preference",
       chunkingPreference: () => "/settings/text-splitter-preference",
diff --git a/frontend/yarn.lock b/frontend/yarn.lock
index bd12e9fa348..93bdc0884fe 100644
--- a/frontend/yarn.lock
+++ b/frontend/yarn.lock
@@ -2841,6 +2841,11 @@ react-smooth@^4.0.0:
     prop-types "^15.8.1"
     react-transition-group "^4.4.5"
 
+react-speech-recognition@^3.10.0:
+  version "3.10.0"
+  resolved "https://registry.yarnpkg.com/react-speech-recognition/-/react-speech-recognition-3.10.0.tgz#7aa43bb28d78b92671864dabba3a70489ccad27b"
+  integrity sha512-EVSr4Ik8l9urwdPiK2r0+ADrLyDDrjB0qBRdUWO+w2MfwEBrj6NuRmy1GD3x7BU/V6/hab0pl8Lupen0zwlJyw==
+
 react-tag-input-component@^2.0.2:
   version "2.0.2"
   resolved "https://registry.yarnpkg.com/react-tag-input-component/-/react-tag-input-component-2.0.2.tgz#f62f013c6a535141dd1c6c3a88858223170150f1"
diff --git a/server/.env.example b/server/.env.example
index 290a07096fc..5e0233b7b4e 100644
--- a/server/.env.example
+++ b/server/.env.example
@@ -168,6 +168,19 @@ WHISPER_PROVIDER="local"
 # WHISPER_PROVIDER="openai"
 # OPEN_AI_KEY=sk-xxxxxxxx
 
+###########################################
+######## TTS/STT Model Selection ##########
+###########################################
+TTS_PROVIDER="native"
+
+# TTS_PROVIDER="openai"
+# TTS_OPEN_AI_KEY=sk-example
+# TTS_OPEN_AI_VOICE_MODEL=nova
+
+# TTS_PROVIDER="elevenlabs"
+# TTS_ELEVEN_LABS_KEY=
+# TTS_ELEVEN_LABS_VOICE_MODEL=21m00Tcm4TlvDq8ikWAM # Rachel
+
 # CLOUD DEPLOYMENT VARIRABLES ONLY
 # AUTH_TOKEN="hunter2" # This is the password to your application if remote hosting.
 # STORAGE_DIR= # absolute filesystem path with no trailing slash
diff --git a/server/endpoints/workspaces.js b/server/endpoints/workspaces.js
index c22c679a0c8..81cbd615418 100644
--- a/server/endpoints/workspaces.js
+++ b/server/endpoints/workspaces.js
@@ -1,6 +1,11 @@
 const path = require("path");
 const fs = require("fs");
-const { reqBody, multiUserMode, userFromSession } = require("../utils/http");
+const {
+  reqBody,
+  multiUserMode,
+  userFromSession,
+  safeJsonParse,
+} = require("../utils/http");
 const { normalizePath } = require("../utils/files");
 const { Workspace } = require("../models/workspace");
 const { Document } = require("../models/documents");
@@ -25,6 +30,7 @@ const {
   determineWorkspacePfpFilepath,
   fetchPfp,
 } = require("../utils/files/pfp");
+const { getTTSProvider } = require("../utils/TextToSpeech");
 
 function workspaceEndpoints(app) {
   if (!app) return;
@@ -506,6 +512,48 @@ function workspaceEndpoints(app) {
     }
   );
 
+  app.get(
+    "/workspace/:slug/tts/:chatId",
+    [validatedRequest, flexUserRoleValid([ROLES.all]), validWorkspaceSlug],
+    async function (request, response) {
+      try {
+        const { chatId } = request.params;
+        const workspace = response.locals.workspace;
+        const cacheKey = `${workspace.slug}:${chatId}`;
+        const wsChat = await WorkspaceChats.get({
+          id: Number(chatId),
+          workspaceId: workspace.id,
+        });
+
+        const cachedResponse = responseCache.get(cacheKey);
+        if (cachedResponse) {
+          response.writeHead(200, {
+            "Content-Type": cachedResponse.mime || "audio/mpeg",
+          });
+          response.end(cachedResponse.buffer);
+          return;
+        }
+
+        const text = safeJsonParse(wsChat.response, null)?.text;
+        if (!text) return response.sendStatus(204).end();
+
+        const TTSProvider = getTTSProvider();
+        const buffer = await TTSProvider.ttsBuffer(text);
+        if (buffer === null) return response.sendStatus(204).end();
+
+        responseCache.set(cacheKey, { buffer, mime: "audio/mpeg" });
+        response.writeHead(200, {
+          "Content-Type": "audio/mpeg",
+        });
+        response.end(buffer);
+        return;
+      } catch (error) {
+        console.error("Error processing the TTS request:", error);
+        response.status(500).json({ message: "TTS could not be completed" });
+      }
+    }
+  );
+
   app.get(
     "/workspace/:slug/pfp",
     [validatedRequest, flexUserRoleValid([ROLES.all])],
diff --git a/server/models/systemSettings.js b/server/models/systemSettings.js
index 904c448d55b..248ca8cd79b 100644
--- a/server/models/systemSettings.js
+++ b/server/models/systemSettings.js
@@ -131,6 +131,17 @@ const SystemSettings = {
       // --------------------------------------------------------
       WhisperProvider: process.env.WHISPER_PROVIDER || "local",
 
+      // --------------------------------------------------------
+      // TTS/STT  Selection Settings & Configs
+      // - Currently the only 3rd party is OpenAI or the native browser-built in
+      // --------------------------------------------------------
+      TextToSpeechProvider: process.env.TTS_PROVIDER || "native",
+      TTSOpenAIKey: !!process.env.TTS_OPEN_AI_KEY,
+      TTSOpenAIVoiceModel: process.env.TTS_OPEN_AI_VOICE_MODEL,
+      // Eleven Labs TTS
+      TTSElevenLabsKey: !!process.env.TTS_ELEVEN_LABS_KEY,
+      TTSElevenLabsVoiceModel: process.env.TTS_ELEVEN_LABS_VOICE_MODEL,
+
       // --------------------------------------------------------
       // Agent Settings & Configs
       // --------------------------------------------------------
diff --git a/server/package.json b/server/package.json
index edee71b023c..73b947c461e 100644
--- a/server/package.json
+++ b/server/package.json
@@ -44,6 +44,7 @@
     "cohere-ai": "^7.9.5",
     "cors": "^2.8.5",
     "dotenv": "^16.0.3",
+    "elevenlabs": "^0.5.0",
     "express": "^4.18.2",
     "express-ws": "^5.0.2",
     "extract-json-from-string": "^1.0.1",
diff --git a/server/utils/helpers/customModels.js b/server/utils/helpers/customModels.js
index b7aae93bef5..caf5a77c755 100644
--- a/server/utils/helpers/customModels.js
+++ b/server/utils/helpers/customModels.js
@@ -4,6 +4,7 @@ const {
 } = require("../AiProviders/openRouter");
 const { perplexityModels } = require("../AiProviders/perplexity");
 const { togetherAiModels } = require("../AiProviders/togetherAi");
+const { ElevenLabsTTS } = require("../TextToSpeech/elevenLabs");
 const SUPPORT_CUSTOM_MODELS = [
   "openai",
   "localai",
@@ -15,6 +16,7 @@ const SUPPORT_CUSTOM_MODELS = [
   "openrouter",
   "lmstudio",
   "koboldcpp",
+  "elevenlabs-tts",
 ];
 
 async function getCustomModels(provider = "", apiKey = null, basePath = null) {
@@ -42,6 +44,8 @@ async function getCustomModels(provider = "", apiKey = null, basePath = null) {
       return await getLMStudioModels(basePath);
     case "koboldcpp":
       return await getKoboldCPPModels(basePath);
+    case "elevenlabs-tts":
+      return await getElevenLabsModels(apiKey);
     default:
       return { models: [], error: "Invalid provider for custom models" };
   }
@@ -321,6 +325,32 @@ function nativeLLMModels() {
   return { models: files, error: null };
 }
 
+async function getElevenLabsModels(apiKey = null) {
+  const models = (await ElevenLabsTTS.voices(apiKey)).map((model) => {
+    return {
+      id: model.voice_id,
+      organization: model.category,
+      name: model.name,
+    };
+  });
+
+  if (models.length === 0) {
+    return {
+      models: [
+        {
+          id: "21m00Tcm4TlvDq8ikWAM",
+          organization: "premade",
+          name: "Rachel (default)",
+        },
+      ],
+      error: null,
+    };
+  }
+
+  if (models.length > 0 && !!apiKey) process.env.TTS_ELEVEN_LABS_KEY = apiKey;
+  return { models, error: null };
+}
+
 module.exports = {
   getCustomModels,
 };
diff --git a/server/utils/helpers/updateENV.js b/server/utils/helpers/updateENV.js
index 947fbc62492..beb03a566e7 100644
--- a/server/utils/helpers/updateENV.js
+++ b/server/utils/helpers/updateENV.js
@@ -366,6 +366,32 @@ const KEY_MAPPING = {
     envKey: "AGENT_SERPER_DEV_KEY",
     checks: [],
   },
+
+  // TTS/STT Integration ENVS
+  TextToSpeechProvider: {
+    envKey: "TTS_PROVIDER",
+    checks: [supportedTTSProvider],
+  },
+
+  // TTS OpenAI
+  TTSOpenAIKey: {
+    envKey: "TTS_OPEN_AI_KEY",
+    checks: [validOpenAIKey],
+  },
+  TTSOpenAIVoiceModel: {
+    envKey: "TTS_OPEN_AI_VOICE_MODEL",
+    checks: [],
+  },
+
+  // TTS EleventLabs
+  TTSElevenLabsKey: {
+    envKey: "TTS_ELEVEN_LABS_KEY",
+    checks: [isNotEmpty],
+  },
+  TTSElevenLabsVoiceModel: {
+    envKey: "TTS_ELEVEN_LABS_VOICE_MODEL",
+    checks: [],
+  },
 };
 
 function isNotEmpty(input = "") {
@@ -419,6 +445,11 @@ function validOllamaLLMBasePath(input = "") {
   }
 }
 
+function supportedTTSProvider(input = "") {
+  const validSelection = ["native", "openai", "elevenlabs"].includes(input);
+  return validSelection ? null : `${input} is not a valid TTS provider.`;
+}
+
 function supportedLLM(input = "") {
   const validSelection = [
     "openai",
diff --git a/server/yarn.lock b/server/yarn.lock
index 5edd09a351f..9e4f184d5f0 100644
--- a/server/yarn.lock
+++ b/server/yarn.lock
@@ -1901,6 +1901,11 @@ combined-stream@^1.0.8:
   dependencies:
     delayed-stream "~1.0.0"
 
+command-exists@^1.2.9:
+  version "1.2.9"
+  resolved "https://registry.yarnpkg.com/command-exists/-/command-exists-1.2.9.tgz#c50725af3808c8ab0260fd60b01fbfa25b954f69"
+  integrity sha512-LTQ/SGc+s0Xc0Fu5WaKnR0YiygZkm9eKFvyS+fRsU7/ZWFF8ykFM6Pc9aCVf1+xasOOZpO3BAVgVrKvsqKHV7w==
+
 command-line-args@5.2.1, command-line-args@^5.2.1:
   version "5.2.1"
   resolved "https://registry.yarnpkg.com/command-line-args/-/command-line-args-5.2.1.tgz#c44c32e437a57d7c51157696893c5909e9cec42e"
@@ -2255,6 +2260,18 @@ ee-first@1.1.1:
   resolved "https://registry.yarnpkg.com/ee-first/-/ee-first-1.1.1.tgz#590c61156b0ae2f4f0255732a158b266bc56b21d"
   integrity sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==
 
+elevenlabs@^0.5.0:
+  version "0.5.0"
+  resolved "https://registry.yarnpkg.com/elevenlabs/-/elevenlabs-0.5.0.tgz#07eb1a943b0ab99b925875bd5c57833a3a024e58"
+  integrity sha512-jfex4ecuWIlyAUuMrMJAJNa5MLziqYQOCDw4ZYuoc9PCYLxtHwaYBWpZoDhnYMcceLI7rRRvmbLMcT9HlVMfHA==
+  dependencies:
+    command-exists "^1.2.9"
+    execa "^5.1.1"
+    form-data "4.0.0"
+    node-fetch "2.7.0"
+    qs "6.11.2"
+    url-join "4.0.1"
+
 emoji-regex@^10.2.1:
   version "10.3.0"
   resolved "https://registry.yarnpkg.com/emoji-regex/-/emoji-regex-10.3.0.tgz#76998b9268409eb3dae3de989254d456e70cfe23"
@@ -2605,6 +2622,21 @@ eventemitter3@^4.0.4:
   resolved "https://registry.yarnpkg.com/eventemitter3/-/eventemitter3-4.0.7.tgz#2de9b68f6528d5644ef5c59526a1b4a07306169f"
   integrity sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw==
 
+execa@^5.1.1:
+  version "5.1.1"
+  resolved "https://registry.yarnpkg.com/execa/-/execa-5.1.1.tgz#f80ad9cbf4298f7bd1d4c9555c21e93741c411dd"
+  integrity sha512-8uSpZZocAZRBAPIEINJj3Lo9HyGitllczc27Eh5YYojjMFMn8yHMDMaUHE2Jqfq05D/wucwI4JGURyXt1vchyg==
+  dependencies:
+    cross-spawn "^7.0.3"
+    get-stream "^6.0.0"
+    human-signals "^2.1.0"
+    is-stream "^2.0.0"
+    merge-stream "^2.0.0"
+    npm-run-path "^4.0.1"
+    onetime "^5.1.2"
+    signal-exit "^3.0.3"
+    strip-final-newline "^2.0.0"
+
 expand-template@^2.0.3:
   version "2.0.3"
   resolved "https://registry.yarnpkg.com/expand-template/-/expand-template-2.0.3.tgz#6e14b3fcee0f3a6340ecb57d2e8918692052a47c"
@@ -3024,6 +3056,11 @@ get-stream@^5.1.0:
   dependencies:
     pump "^3.0.0"
 
+get-stream@^6.0.0:
+  version "6.0.1"
+  resolved "https://registry.yarnpkg.com/get-stream/-/get-stream-6.0.1.tgz#a262d8eef67aced57c2852ad6167526a43cbf7b7"
+  integrity sha512-ts6Wi+2j3jQjqi70w5AlN8DFnkSwC+MqmxEzdEALB2qXZYV3X/b1CTfgPLGJNMeAWxdPfU8FO1ms3NUfaHCPYg==
+
 get-symbol-description@^1.0.2:
   version "1.0.2"
   resolved "https://registry.yarnpkg.com/get-symbol-description/-/get-symbol-description-1.0.2.tgz#533744d5aa20aca4e079c8e5daf7fd44202821f5"
@@ -3297,6 +3334,11 @@ https-proxy-agent@^7.0.0:
     agent-base "^7.0.2"
     debug "4"
 
+human-signals@^2.1.0:
+  version "2.1.0"
+  resolved "https://registry.yarnpkg.com/human-signals/-/human-signals-2.1.0.tgz#dc91fcba42e4d06e4abaed33b3e7a3c02f514ea0"
+  integrity sha512-B4FFZ6q/T2jhhksgkbEW3HBvWIfDW85snkQgawt07S7J5QXTk6BkNV+0yAeZrM5QpMAdYlocGoljn0sJ/WQkFw==
+
 humanize-ms@^1.2.1:
   version "1.2.1"
   resolved "https://registry.yarnpkg.com/humanize-ms/-/humanize-ms-1.2.1.tgz#c46e3159a293f6b896da29316d8b6fe8bb79bbed"
@@ -4092,6 +4134,11 @@ merge-descriptors@1.0.1:
   resolved "https://registry.yarnpkg.com/merge-descriptors/-/merge-descriptors-1.0.1.tgz#b00aaa556dd8b44568150ec9d1b953f3f90cbb61"
   integrity sha512-cCi6g3/Zr1iqQi6ySbseM1Xvooa98N0w31jzUYrXPX2xqObmFGHJ0tQ5u74H3mVh7wLouTseZyYIq39g8cNp1w==
 
+merge-stream@^2.0.0:
+  version "2.0.0"
+  resolved "https://registry.yarnpkg.com/merge-stream/-/merge-stream-2.0.0.tgz#52823629a14dd00c9770fb6ad47dc6310f2c1f60"
+  integrity sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w==
+
 methods@~1.1.2:
   version "1.1.2"
   resolved "https://registry.yarnpkg.com/methods/-/methods-1.1.2.tgz#5529a4d67654134edcc5266656835b0f851afcee"
@@ -4455,6 +4502,13 @@ normalize-path@^3.0.0, normalize-path@~3.0.0:
   resolved "https://registry.yarnpkg.com/normalize-path/-/normalize-path-3.0.0.tgz#0dcd69ff23a1c9b11fd0978316644a0388216a65"
   integrity sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==
 
+npm-run-path@^4.0.1:
+  version "4.0.1"
+  resolved "https://registry.yarnpkg.com/npm-run-path/-/npm-run-path-4.0.1.tgz#b7ecd1e5ed53da8e37a55e1c2269e0b97ed748ea"
+  integrity sha512-S48WzZW777zhNIrn7gxOlISNAqi9ZC/uQFnRdbeIHhZhCA6UqpkOT8T1G7BvfdgP4Er8gF4sUbaS0i7QvIfCWw==
+  dependencies:
+    path-key "^3.0.0"
+
 npmlog@^5.0.1:
   version "5.0.1"
   resolved "https://registry.yarnpkg.com/npmlog/-/npmlog-5.0.1.tgz#f06678e80e29419ad67ab964e0fa69959c1eb8b0"
@@ -4593,7 +4647,7 @@ one-time@^1.0.0:
   dependencies:
     fn.name "1.x.x"
 
-onetime@^5.1.0:
+onetime@^5.1.0, onetime@^5.1.2:
   version "5.1.2"
   resolved "https://registry.yarnpkg.com/onetime/-/onetime-5.1.2.tgz#d0e96ebb56b07476df1dd9c4806e5237985ca45e"
   integrity sha512-kbpaSSGJTWdAY5KPVeMOKXSrPtr8C8C7wodJbcsd51jRnmD+GZu8Y0VoU6Dm5Z4vWr0Ig/1NKuWRKf7j5aaYSg==
@@ -4774,7 +4828,7 @@ path-is-absolute@^1.0.0:
   resolved "https://registry.yarnpkg.com/path-is-absolute/-/path-is-absolute-1.0.1.tgz#174b9268735534ffbc7ace6bf53a5a9e1b5c5f5f"
   integrity sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==
 
-path-key@^3.1.0:
+path-key@^3.0.0, path-key@^3.1.0:
   version "3.1.1"
   resolved "https://registry.yarnpkg.com/path-key/-/path-key-3.1.1.tgz#581f6ade658cbba65a0d3380de7753295054f375"
   integrity sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==
@@ -5322,7 +5376,7 @@ side-channel@^1.0.4, side-channel@^1.0.6:
     get-intrinsic "^1.2.4"
     object-inspect "^1.13.1"
 
-signal-exit@^3.0.0, signal-exit@^3.0.2, signal-exit@^3.0.7:
+signal-exit@^3.0.0, signal-exit@^3.0.2, signal-exit@^3.0.3, signal-exit@^3.0.7:
   version "3.0.7"
   resolved "https://registry.yarnpkg.com/signal-exit/-/signal-exit-3.0.7.tgz#a9a1767f8af84155114eaabd73f99273c8f59ad9"
   integrity sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==
@@ -5559,6 +5613,11 @@ strip-ansi@^7.0.1, strip-ansi@^7.1.0:
   dependencies:
     ansi-regex "^6.0.1"
 
+strip-final-newline@^2.0.0:
+  version "2.0.0"
+  resolved "https://registry.yarnpkg.com/strip-final-newline/-/strip-final-newline-2.0.0.tgz#89b852fb2fcbe936f6f4b3187afb0a12c1ab58ad"
+  integrity sha512-BrpvfNAE3dcvq7ll3xVumzjKjZQ5tI1sEUIKr3Uoks0XUl45St3FlatVqef9prk4jRDzhW6WZg+3bk93y6pLjA==
+
 strip-json-comments@^3.1.1:
   version "3.1.1"
   resolved "https://registry.yarnpkg.com/strip-json-comments/-/strip-json-comments-3.1.1.tgz#31f1281b3832630434831c310c01cccda8cbe006"

From 0c660a23a00d1300430eb2f6bc9cbe3a67a9a213 Mon Sep 17 00:00:00 2001
From: timothycarambat <rambat1010@gmail.com>
Date: Mon, 13 May 2024 23:03:13 -0700
Subject: [PATCH 2/4] add files and update comment

---
 .../SpeechToText/BrowserNative/index.jsx      |   9 +
 .../TextToSpeech/BrowserNative/index.jsx      |   9 +
 .../TextToSpeech/ElevenLabsOptions/index.jsx  | 107 +++++++++
 .../TextToSpeech/OpenAiOptions/index.jsx      |  45 ++++
 .../Actions/TTSButton/asyncTts.jsx            |  94 ++++++++
 .../Actions/TTSButton/index.jsx               |  23 ++
 .../Actions/TTSButton/native.jsx              |  61 +++++
 .../PromptInput/SpeechToText/index.jsx        |  82 +++++++
 .../src/media/ttsproviders/elevenlabs.png     | Bin 0 -> 6422 bytes
 .../GeneralSettings/AudioPreference/index.jsx |  45 ++++
 .../GeneralSettings/AudioPreference/stt.jsx   | 191 ++++++++++++++++
 .../GeneralSettings/AudioPreference/tts.jsx   | 209 ++++++++++++++++++
 server/utils/TextToSpeech/elevenLabs/index.js |  54 +++++
 server/utils/TextToSpeech/index.js            |  15 ++
 server/utils/TextToSpeech/openAi/index.js     |  29 +++
 server/utils/helpers/updateENV.js             |   2 +-
 16 files changed, 974 insertions(+), 1 deletion(-)
 create mode 100644 frontend/src/components/SpeechToText/BrowserNative/index.jsx
 create mode 100644 frontend/src/components/TextToSpeech/BrowserNative/index.jsx
 create mode 100644 frontend/src/components/TextToSpeech/ElevenLabsOptions/index.jsx
 create mode 100644 frontend/src/components/TextToSpeech/OpenAiOptions/index.jsx
 create mode 100644 frontend/src/components/WorkspaceChat/ChatContainer/ChatHistory/HistoricalMessage/Actions/TTSButton/asyncTts.jsx
 create mode 100644 frontend/src/components/WorkspaceChat/ChatContainer/ChatHistory/HistoricalMessage/Actions/TTSButton/index.jsx
 create mode 100644 frontend/src/components/WorkspaceChat/ChatContainer/ChatHistory/HistoricalMessage/Actions/TTSButton/native.jsx
 create mode 100644 frontend/src/components/WorkspaceChat/ChatContainer/PromptInput/SpeechToText/index.jsx
 create mode 100644 frontend/src/media/ttsproviders/elevenlabs.png
 create mode 100644 frontend/src/pages/GeneralSettings/AudioPreference/index.jsx
 create mode 100644 frontend/src/pages/GeneralSettings/AudioPreference/stt.jsx
 create mode 100644 frontend/src/pages/GeneralSettings/AudioPreference/tts.jsx
 create mode 100644 server/utils/TextToSpeech/elevenLabs/index.js
 create mode 100644 server/utils/TextToSpeech/index.js
 create mode 100644 server/utils/TextToSpeech/openAi/index.js

diff --git a/frontend/src/components/SpeechToText/BrowserNative/index.jsx b/frontend/src/components/SpeechToText/BrowserNative/index.jsx
new file mode 100644
index 00000000000..1e9bcb3c2e5
--- /dev/null
+++ b/frontend/src/components/SpeechToText/BrowserNative/index.jsx
@@ -0,0 +1,9 @@
+export default function BrowserNative() {
+  return (
+    <div className="w-full h-10 items-center flex">
+      <p className="text-sm font-base text-white text-opacity-60">
+        There is no configuration needed for this provider.
+      </p>
+    </div>
+  );
+}
diff --git a/frontend/src/components/TextToSpeech/BrowserNative/index.jsx b/frontend/src/components/TextToSpeech/BrowserNative/index.jsx
new file mode 100644
index 00000000000..1e9bcb3c2e5
--- /dev/null
+++ b/frontend/src/components/TextToSpeech/BrowserNative/index.jsx
@@ -0,0 +1,9 @@
+export default function BrowserNative() {
+  return (
+    <div className="w-full h-10 items-center flex">
+      <p className="text-sm font-base text-white text-opacity-60">
+        There is no configuration needed for this provider.
+      </p>
+    </div>
+  );
+}
diff --git a/frontend/src/components/TextToSpeech/ElevenLabsOptions/index.jsx b/frontend/src/components/TextToSpeech/ElevenLabsOptions/index.jsx
new file mode 100644
index 00000000000..5b4232e8436
--- /dev/null
+++ b/frontend/src/components/TextToSpeech/ElevenLabsOptions/index.jsx
@@ -0,0 +1,107 @@
+import { useState, useEffect } from "react";
+import System from "@/models/system";
+
+export default function ElevenLabsOptions({ settings }) {
+  const [inputValue, setInputValue] = useState(settings?.TTSElevenLabsKey);
+  const [openAIKey, setOpenAIKey] = useState(settings?.TTSElevenLabsKey);
+
+  return (
+    <div className="flex gap-x-4">
+      <div className="flex flex-col w-60">
+        <label className="text-white text-sm font-semibold block mb-4">
+          API Key
+        </label>
+        <input
+          type="password"
+          name="TTSElevenLabsKey"
+          className="bg-zinc-900 text-white placeholder:text-white/20 text-sm rounded-lg focus:border-white block w-full p-2.5"
+          placeholder="ElevenLabs API Key"
+          defaultValue={settings?.TTSElevenLabsKey ? "*".repeat(20) : ""}
+          required={true}
+          autoComplete="off"
+          spellCheck={false}
+          onChange={(e) => setInputValue(e.target.value)}
+          onBlur={() => setOpenAIKey(inputValue)}
+        />
+      </div>
+      {!settings?.credentialsOnly && (
+        <ElevenLabsModelSelection settings={settings} apiKey={openAIKey} />
+      )}
+    </div>
+  );
+}
+
+function ElevenLabsModelSelection({ apiKey, settings }) {
+  const [groupedModels, setGroupedModels] = useState({});
+  const [loading, setLoading] = useState(true);
+
+  useEffect(() => {
+    async function findCustomModels() {
+      setLoading(true);
+      const { models } = await System.customModels(
+        "elevenlabs-tts",
+        typeof apiKey === "boolean" ? null : apiKey
+      );
+
+      if (models?.length > 0) {
+        const modelsByOrganization = models.reduce((acc, model) => {
+          acc[model.organization] = acc[model.organization] || [];
+          acc[model.organization].push(model);
+          return acc;
+        }, {});
+        setGroupedModels(modelsByOrganization);
+      }
+
+      setLoading(false);
+    }
+    findCustomModels();
+  }, [apiKey]);
+
+  if (loading) {
+    return (
+      <div className="flex flex-col w-60">
+        <label className="text-white text-sm font-semibold block mb-4">
+          Chat Model Selection
+        </label>
+        <select
+          name="OpenAiModelPref"
+          disabled={true}
+          className="bg-zinc-900 border-gray-500 text-white text-sm rounded-lg block w-full p-2.5"
+        >
+          <option disabled={true} selected={true}>
+            -- loading available models --
+          </option>
+        </select>
+      </div>
+    );
+  }
+
+  return (
+    <div className="flex flex-col w-60">
+      <label className="text-white text-sm font-semibold block mb-4">
+        Chat Model Selection
+      </label>
+      <select
+        name="OpenAiModelPref"
+        required={true}
+        className="bg-zinc-900 border-gray-500 text-white text-sm rounded-lg block w-full p-2.5"
+      >
+        {Object.keys(groupedModels)
+          .sort()
+          .map((organization) => (
+            <optgroup key={organization} label={organization}>
+              {groupedModels[organization].map((model) => (
+                <option
+                  key={model.id}
+                  value={model.id}
+                  selected={settings?.OpenAiModelPref === model.id}
+                >
+                  {model.name}
+                </option>
+              ))}
+            </optgroup>
+          ))}
+      </select>
+    </div>
+  );
+}
diff --git a/frontend/src/components/TextToSpeech/OpenAiOptions/index.jsx b/frontend/src/components/TextToSpeech/OpenAiOptions/index.jsx
new file mode 100644
index 00000000000..4183a4e5800
--- /dev/null
+++ b/frontend/src/components/TextToSpeech/OpenAiOptions/index.jsx
@@ -0,0 +1,45 @@
+function toProperCase(string) {
+  return string.replace(/\w\S*/g, function (txt) {
+    return txt.charAt(0).toUpperCase() + txt.substr(1).toLowerCase();
+  });
+}
+
+export default function OpenAiTextToSpeechOptions({ settings }) {
+  const apiKey = settings?.TTSOpenAIKey;
+
+  return (
+    <div className="flex gap-x-4">
+      <div className="flex flex-col w-60">
+        <label className="text-white text-sm font-semibold block mb-4">
+          API Key
+        </label>
+        <input
+          type="password"
+          name="TTSOpenAIKey"
+          className="bg-zinc-900 text-white placeholder:text-white/20 text-sm rounded-lg focus:border-white block w-full p-2.5"
+          placeholder="OpenAI API Key"
+          defaultValue={apiKey ? "*".repeat(20) : ""}
+          required={true}
+          autoComplete="off"
+          spellCheck={false}
+        />
+      </div>
+      <div className="flex flex-col w-60">
+        <label className="text-white text-sm font-semibold block mb-4">
+          Voice Model
+        </label>
+        <select
+          name="TTSOpenAIVoiceModel"
+          defaultValue={settings?.TTSOpenAIVoiceModel ?? "alloy"}
+          className="bg-zinc-900 border-gray-500 text-white text-sm rounded-lg block w-full p-2.5"
+        >
+          {["alloy", "echo", "fable", "onyx", "nova", "shimmer"].map(
+            (voice) => {
+              return <option value={voice}>{toProperCase(voice)}</option>;
+            }
+          )}
+        </select>
+      </div>
+    </div>
+  );
+}
diff --git a/frontend/src/components/WorkspaceChat/ChatContainer/ChatHistory/HistoricalMessage/Actions/TTSButton/asyncTts.jsx b/frontend/src/components/WorkspaceChat/ChatContainer/ChatHistory/HistoricalMessage/Actions/TTSButton/asyncTts.jsx
new file mode 100644
index 00000000000..e58d4df3041
--- /dev/null
+++ b/frontend/src/components/WorkspaceChat/ChatContainer/ChatHistory/HistoricalMessage/Actions/TTSButton/asyncTts.jsx
@@ -0,0 +1,94 @@
+import { useEffect, useState, useRef } from "react";
+import { SpeakerHigh, PauseCircle, CircleNotch } from "@phosphor-icons/react";
+import { Tooltip } from "react-tooltip";
+import Workspace from "@/models/workspace";
+import showToast from "@/utils/toast";
+
+export default function AsyncTTSMessage({ slug, chatId }) {
+  const playerRef = useRef(null);
+  const [speaking, setSpeaking] = useState(false);
+  const [loading, setLoading] = useState(false);
+  const [audioSrc, setAudioSrc] = useState(null);
+
+  function speakMessage() {
+    if (speaking) {
+      playerRef?.current?.pause();
+      return;
+    }
+
+    try {
+      if (!audioSrc) {
+        setLoading(true);
+        Workspace.ttsMessage(slug, chatId)
+          .then((audioBlob) => {
+            if (!audioBlob)
+              throw new Error("Failed to load or play TTS message response.");
+            setAudioSrc(audioBlob);
+          })
+          .catch((e) => showToast(e.message, "error", { clear: true }))
+          .finally(() => setLoading(false));
+      } else {
+        playerRef.current.play();
+      }
+    } catch (e) {
+      console.error(e);
+      setLoading(false);
+      setSpeaking(false);
+    }
+  }
+
+  useEffect(() => {
+    function setupPlayer() {
+      if (!playerRef) return;
+      playerRef.current.addEventListener("play", () => {
+        setSpeaking(true);
+      });
+
+      playerRef.current.addEventListener("pause", () => {
+        playerRef.current.currentTime = 0;
+        setSpeaking(false);
+      });
+    }
+    setupPlayer();
+  }, []);
+
+  if (!chatId) return null;
+  return (
+    <div className="mt-3 relative">
+      <button
+        onClick={speakMessage}
+        data-tooltip-id="message-to-speech"
+        data-tooltip-content={
+          speaking ? "Pause TTS speech of message" : "TTS Speak message"
+        }
+        className="border-none text-zinc-300"
+        aria-label={speaking ? "Pause speech" : "Speak message"}
+      >
+        {speaking ? (
+          <PauseCircle size={18} className="mb-1" />
+        ) : (
+          <>
+            {loading ? (
+              <CircleNotch size={18} className="mb-1 animate-spin" />
+            ) : (
+              <SpeakerHigh size={18} className="mb-1" />
+            )}
+          </>
+        )}
+        <audio
+          ref={playerRef}
+          hidden={true}
+          src={audioSrc}
+          autoPlay={true}
+          controls={false}
+        />
+      </button>
+      <Tooltip
+        id="message-to-speech"
+        place="bottom"
+        delayShow={300}
+        className="tooltip !text-xs"
+      />
+    </div>
+  );
+}
diff --git a/frontend/src/components/WorkspaceChat/ChatContainer/ChatHistory/HistoricalMessage/Actions/TTSButton/index.jsx b/frontend/src/components/WorkspaceChat/ChatContainer/ChatHistory/HistoricalMessage/Actions/TTSButton/index.jsx
new file mode 100644
index 00000000000..644a57afc38
--- /dev/null
+++ b/frontend/src/components/WorkspaceChat/ChatContainer/ChatHistory/HistoricalMessage/Actions/TTSButton/index.jsx
@@ -0,0 +1,23 @@
+import { useEffect, useState } from "react";
+import NativeTTSMessage from "./native";
+import AsyncTTSMessage from "./asyncTts";
+import System from "@/models/system";
+
+export default function TTSMessage({ slug, chatId, message }) {
+  const [provider, setProvider] = useState("native");
+  const [loading, setLoading] = useState(true);
+
+  useEffect(() => {
+    async function getSettings() {
+      const _settings = await System.keys();
+      setProvider(_settings?.TextToSpeechProvider ?? "native");
+      setLoading(false);
+    }
+    getSettings();
+  }, []);
+
+  if (loading) return null;
+  if (provider !== "native")
+    return <AsyncTTSMessage slug={slug} chatId={chatId} />;
+  return <NativeTTSMessage message={message} />;
+}
diff --git a/frontend/src/components/WorkspaceChat/ChatContainer/ChatHistory/HistoricalMessage/Actions/TTSButton/native.jsx b/frontend/src/components/WorkspaceChat/ChatContainer/ChatHistory/HistoricalMessage/Actions/TTSButton/native.jsx
new file mode 100644
index 00000000000..5d9d4fe5318
--- /dev/null
+++ b/frontend/src/components/WorkspaceChat/ChatContainer/ChatHistory/HistoricalMessage/Actions/TTSButton/native.jsx
@@ -0,0 +1,61 @@
+import React, { useEffect, useState } from "react";
+import { SpeakerHigh, PauseCircle } from "@phosphor-icons/react";
+import { Tooltip } from "react-tooltip";
+
+export default function NativeTTSMessage({ message }) {
+  const [speaking, setSpeaking] = useState(false);
+  const [supported, setSupported] = useState(false);
+  useEffect(() => {
+    setSupported("speechSynthesis" in window);
+  }, []);
+
+  function endSpeechUtterance() {
+    window.speechSynthesis?.cancel();
+    setSpeaking(false);
+    return;
+  }
+
+  function speakMessage() {
+    // if the user is pausing this particular message
+    // while the synth if speaking we can end it.
+    // If they are clicking another message's TTS
+    // we need to ignore that until they pause the one that is playing.
+    if (window.speechSynthesis.speaking && speaking) {
+      endSpeechUtterance();
+      return;
+    }
+
+    if (window.speechSynthesis.speaking && !speaking) return;
+    const utterance = new SpeechSynthesisUtterance(message);
+    utterance.addEventListener("end", endSpeechUtterance);
+    window.speechSynthesis.speak(utterance);
+    setSpeaking(true);
+  }
+
+  if (!supported) return null;
+  return (
+    <div className="mt-3 relative">
+      <button
+        onClick={speakMessage}
+        data-tooltip-id="message-to-speech"
+        data-tooltip-content={
+          speaking ? "Pause TTS speech of message" : "TTS Speak message"
+        }
+        className="border-none text-zinc-300"
+        aria-label={speaking ? "Pause speech" : "Speak message"}
+      >
+        {speaking ? (
+          <PauseCircle size={18} className="mb-1" />
+        ) : (
+          <SpeakerHigh size={18} className="mb-1" />
+        )}
+      </button>
+      <Tooltip
+        id="message-to-speech"
+        place="bottom"
+        delayShow={300}
+        className="tooltip !text-xs"
+      />
+    </div>
+  );
+}
diff --git a/frontend/src/components/WorkspaceChat/ChatContainer/PromptInput/SpeechToText/index.jsx b/frontend/src/components/WorkspaceChat/ChatContainer/PromptInput/SpeechToText/index.jsx
new file mode 100644
index 00000000000..6cbcfbf8d5a
--- /dev/null
+++ b/frontend/src/components/WorkspaceChat/ChatContainer/PromptInput/SpeechToText/index.jsx
@@ -0,0 +1,82 @@
+import { useEffect } from "react";
+import { Microphone } from "@phosphor-icons/react";
+import { Tooltip } from "react-tooltip";
+import _regeneratorRuntime from "regenerator-runtime";
+import SpeechRecognition, {
+  useSpeechRecognition,
+} from "react-speech-recognition";
+
+let timeout;
+const SILENCE_INTERVAL = 3_200; // wait in seconds of silence before closing.
+export default function SpeechToText({ sendCommand }) {
+  const {
+    transcript,
+    listening,
+    resetTranscript,
+    browserSupportsSpeechRecognition,
+    browserSupportsContinuousListening,
+    isMicrophoneAvailable,
+  } = useSpeechRecognition({
+    clearTranscriptOnListen: true,
+  });
+
+  function startSTTSession() {
+    if (!isMicrophoneAvailable) {
+      alert(
+        "AnythingLLM does not have access to microphone. Please enable for this site to use this feature."
+      );
+      return;
+    }
+
+    resetTranscript();
+    SpeechRecognition.startListening({
+      continuous: browserSupportsContinuousListening,
+      language: window?.navigator?.language ?? "en-US",
+    });
+  }
+
+  function endTTSSession() {
+    SpeechRecognition.stopListening();
+    if (transcript.length > 0) {
+      sendCommand(transcript, true);
+    }
+
+    resetTranscript();
+    clearTimeout(timeout);
+  }
+
+  useEffect(() => {
+    if (transcript?.length > 0) {
+      sendCommand(transcript, false);
+      clearTimeout(timeout);
+      timeout = setTimeout(() => {
+        endTTSSession();
+      }, SILENCE_INTERVAL);
+    }
+  }, [transcript]);
+
+  if (!browserSupportsSpeechRecognition) return null;
+  return (
+    <div
+      id="text-size-btn"
+      data-tooltip-id="tooltip-text-size-btn"
+      data-tooltip-content="Speak your prompt"
+      aria-label="Speak your prompt"
+      onClick={listening ? endTTSSession : startSTTSession}
+      className={`relative flex justify-center items-center opacity-60 hover:opacity-100 cursor-pointer ${
+        !!listening ? "!opacity-100" : ""
+      }`}
+    >
+      <Microphone
+        weight="fill"
+        className="w-6 h-6 pointer-events-none text-white"
+      />
+      <Tooltip
+        id="tooltip-text-size-btn"
+        place="top"
+        delayShow={300}
+        className="tooltip !text-xs z-99"
+      />
+    </div>
+  );
+}
diff --git a/frontend/src/media/ttsproviders/elevenlabs.png b/frontend/src/media/ttsproviders/elevenlabs.png
new file mode 100644
index 0000000000000000000000000000000000000000..b1047e422e3c7a855caec5ef2bae1fcc647495e5
GIT binary patch
literal 6422
zcmeI1*Ebvhx5XtQf*>MFhLGsPAbONU^fo%f=)veEdM6QyUSbT<TXdrgMje8vqnBaS
zNQmBhzxf{ST6f)l;ogV6&OYbuyqx`8>qKg4D3Ox_NC^lC$W@f(bqEOVfc~Qgg#RWQ
z`N2y6${i0KCD}XpK2XTNhUC36)PsP4jQT&iLy(q1Pe5?@S_`bFaC>`ub8|C2JzZK_
zT2@w8TwFXqKYw<1*4^D*Qc^NEH+On^+SAj6L?SCHDk>{0|NQxLetteRHDzI8p`@gw
ztgO7YwsvuG5g#8<OG`^nPftfjmz9;ZwY6n!ZJm>ob9HrfeSN*Ux(WaQo<4ngd3o92
z-_OCpaqr$eA|fJUV&bKxrKqT=yLa#2zkfe1Ep2mi)6me6kdW~2-@m%Lx<y4rSS<F=
zojd=Qh=_<tO-<F&(J3e>`1b7^7Z(>NCudbv)rSus1_lQD`ueuFw=**{Gcq!EcXwxI
zX1;&_{^`>vFE1}YKfmVY=IZL|n3xz}Utb>|pYZVT@$qpu9PZ%Y@b29^7Z(>i9>2W2
z{Pyiza&q#A4<Cw%h<y3-WpQzlnwt9Y<HwAQj1v<RPo6xXp`p>!(-RgJW@l%A|Nebp
zVIc(t1tle=o15F4H*X$1cpxt?-`?Kd($YdkMkXU86BieUKp>8fj|Bt-c6N61^78EM
z?Sq4ZpFe-j%F3Fao_=z2lAD_w85ub_Ik~^T|MlzFjg1XbQqt(?=%l2imoHzAj*kBN
z^~=b}NL5u;Qc`knZ_mog%F)rWuC6XSJDY@r<ml)K0)f=m*U!$*y1KePdh`eig;G&b
ziHnN^fk0+vW>ZsB4-bzQFJ5$Xbd;BuD<~*POG`g{_6&_iySuwDEG#4>Bn%A=v9Ylo
z92`hVN!8ZYe*E~6mzQ^aeSK_f?DOZ(OiWCjot??a$zU+}_3PK%+}y*%!|Lklp`oFI
zf`Z!G+AtVQOib+e@8A6V{0s~XT3T8^fBy9J^fWg&Z)<A<fk0tlVId(QEG#UFii#Kv
zrm3mP*x2~+@X+7i-^t0z+1XiNU%#QDAtfbcWo5<A&aSbsQBzZshlfW)LqknX&C=3x
zWMss|#6&_uLQYQ3#>VEyk01H@`LeRIW@cuhqN3j3-Y66*ARr(yF)=naHYg~lwY7C{
zaL~ZOptrZTtE($8Fc62s@$vD!di6?0MMX$R$kx^t6m#T5KtO|1k(bf)&DhQ)H+j;v
z_`^2g^&gFeCyRTZUlW2BwQ7{Wa_R~^qXT;)=KUQ$RDnRSoGzBFOD}L;e_{YU=rgFJ
z8(0QrSJPEaMuFuqa;}lHok89wJHI|0W=luPt({&6wqRy6Q!~#cBqfC|@3Qxr%3&o5
zX-JZY2w->KLJ72J3I5-5X+qIK*p+WY^ebH0*3H{EkXXM?&cr~lUI8uomt6(R2L~WV
zZsW+CCV$Iu9*pQUP)#K<&^Cg48Ac*H%<}|ukC7;Aa+(LDx>ov>t9{r;=!LN9>vY?T
z)mR;y4*etuM2H)h4M^*^{yfr#mnj&e8-D_wT;DcX4gaQCYNK2RusqT}aX~K^ts(@e
zEAnblGxIho^g;1PF#BnK_^Q4>jxbr0%{B3=VjA{T_*NVm4CLC(ZX|<plq_qIQOYH)
zaH+NaH4o%GKs1HqfH+ZLBVLSrASnxdM0|S>d<U)#QGwbF@u<J2rG|;aS28zh$0Jc0
z{r$=wwBse9<Y)M%q-T(?d7ttu%j{$rd$c}(EcO!NJ^r!LnpiOWd>$w_avS({jR?|^
zBR#485m%kE%Tr2r`>^hwAo6KH%k_;1Z?4vZ5U8mzSQl~(OLx^XH#agp{R&dOZ{qM)
zvvg%tR6D9ZZTPvrvl%hOV{T1}h^8cXQsX?VVZ4%`JAmql*RgO8?XoUEcqxc%U$g6v
z*;vzwoy9H!>cS%WZ#xA&N}T4UQN_b#Wvp=l(~_{UacH*xh4sjf1xZ*zapP+-o9P;<
zix1&C-aCEEJYgEI$40h^eKzkSN0C$Q8k@*tUKeH|`K-B-?BE?|WTZ}84doDjEH7Q}
zM)h>_%Rn0$o9Mp0{m9@9Il52Zx+utLpFT+Sx*>)k88a;7jhbb$H1H(rr%mO|GL_7a
z$XHWBX2IKwwrIP#;8fU3?tCTk1VyLw<kQLADyn6S)Q4qJh4$EwJ=MSZ5lhPE3w!7j
ztA<<B7ke{hj!(n{w=WCp@OdQNnE-aPc!eapEGl>CwE&e04;4GT0K8ocE;Nx=fL^;^
z2gF6ltH<|>|J|on)1DSgVTYrQIq-7)FC*)>2NeaNE-hd5-{udj&lJ9YtR48s15O4F
z`&scj$=WWPCRF5a;wkKzz-aM)m|pbEI?@U2Hucx{ecosN7vsg}X`XnF=mc~CCMHUW
z&j2~kF?9JBNV*wS{lb7g!*O1!9e48NtHNqwHm(ZrTQXtnsmSJ0_IeiuU!&<Y@D7~G
zaBZ;Fg?(dRVIf2bZ@pH(4tfem{_ZObE&P62aa&h+^jjA4aaoD-Ti9Kw;8%_j8tWgH
zNTWt@b<Il5<IfGoCePJ4@?HybsyD9aC7UTBI(l^;#tx2Q**wtu2T%^zB+YWAj58|7
z{ED^~Rg0&%Gkt?#fLU7t2scsvjQ)4q0{^{)fe&w-R~@H!{L9r+$gCr)`*v)VP3G;5
zypz&qgj4-@k;#rlc3^8ov=H>FA%h<;|7(I<mqc>f?(OYtxI9kpIDHp!9ekI;Y9q^V
z2*7(Sq=e@ed=(cGJ~a9Wf3{Xry>X{Bg>Fl_+o_AZG{q(b#^jjFx!0ef=#?5|dBI!s
zyVu>U=FI1h6b<3K`TR)ayWiwYUeNZUaeAQrZZM>7=A1`A+M)GZmv=)#90yMx;?L;!
zm7A)bTpmu~@wn8CW0Preq()ScQ@H>6wSgW~)8)}lV-=tHL$XJ8@C;me&IOC4+fdOb
zlvD2Ua;KfQ1-b!aJ{080UIre%@)nNWYvS?p>fgU`Z_&uAi`j0vj>Mv?i_E?SDkDE>
zBCD~23=9uaan&Opwk`N5BK`-bCetWshB56crhql3fT227J|NayBdcF0wW{Fd5@-G2
zv?RFMO~Fi<r(}87ZA*or;8752Y+Czl!&AT&%cFYGUG)Wc@(Kx0<I3cLf;W1}M4`K5
zbYtZ!PBVh~CD}H&*<k^E<zyGow4F)P&Nhx2p0rP(><e|iszXDMwg}df4^shMRo`oa
z9=~qr0qnpi76M|1ANccY?7v$(P#N5sZ#I-vN;}uD@3Ftli}&vjs4NR<Qcs?3Obe~c
z2-;*UCqVgwPIw-=0$;gF_Of_fK@2rbDS*bB$1`1++R))6^73DqHStD@*34E5Z?-gl
znhamCvs4Q)2}mcoV=dryTwvE{Hwk`i2KS)1Qlf0nLN%bx(_J1M)HmPR^Syh`nVg1>
z*|KY_(sla4OZL(>_q~O(flPfyfblJ^J`U`JBVXO%upun<E_XBtUf}zDh%r4!SWh#C
z3EO>_<kMhtTqVT1&U}~k9^M)eBj1GNPdFY~MQGp-39Sr^xIGEocxX3Hl2>`3>ZJ=R
zp1YWc9c+XhZv~23aSs+>?-c;bbIIm?n+?CuHh1z=gOGE#!j~6o{6$hxpN+Iv*$e2p
zTw^U&pbqt|g%cdq^`(Eve@nI;>)^OQ^Bcb@HO%-$aU)xRSmAfEFeN@rWbY9oBeZk0
zIS6%De3jEfCE9&1<ptbi7})N%O_=d3eP<O|^Prtcfg*9@9_jrtgcih1Ya9z&V0ET_
z)Q%Sau=O2y7)Icjk~pOe(K-A^S=%26d0h}wVsEj<O&iG;gYJ3pah@YXIh!Wu2+vbq
zKEX55=R2UC9IVOKywrVUvq3yDQ=e0pICX2WKJPjxxGC&#I@MD>8vpanTEB)<Qt$4w
zjv3G&6yKdtk-i*L10{!^x_s8QiYioLSW=AWucjQgBrjIC&k~7_fuioofE>v1iJG`W
zKc%C1{3I_x`ba#Mw}SC-ibV#xA#FDiO(XpsKXqBo;9Wk%rboNVd_JHfFbhgO+V;tn
zul@CvF2o}vZ;7t0TOzmcY98Mxm((;_ykBb<Ct^jNcSR(U?MOFm-FNiI3`YeH{P|u)
zr#JnmGc`=Zd7*?qz2s&5bRvh|Kw6h)Qp99q7vf&^8sk;(C#MV^wa2DwiU3z=dx^)r
zqd#|W_u~)JiJ?UYD=7f4I@5M>*4SSYHfjvi$$pN8<SOc+!eF5SeLbkgOV1JmaB?u(
zJI#MdCK;63XF#dEq4!MeRf&b=p?iZf3ud>K4DEGfXs+oct6v)iueA2b1cY2L+`j~u
zfFP<+`H-!YQ6`0r9H*_|Vw_)l*66O8o>DN1UK-yTH^S}40Lx}^tCc%2C|qpZe7_h+
zZqMWsZrEsgl?sBK&FxhpQ#yn<Tb<H>;O~$0uNnU4aNc{v@x)qq|Du~4gz3Ru&_3+8
zzoN`?;e3?`Yu5k$-g52R>WZhg;#paWKau;{zso*JSEYtb&!X+c=zKw)P_%jXOoWRv
zTT)<tegS5zbTw{>@BtDU91iHo+&H5itFXZDGCiz4C7Fmhp_s<DGL~<@^entuz{^>I
zm=Xcuaho-Qg0h)n1!}nEN8X_iYq4pEwxr{yBPAWexk;Izw1&+_-LP7gA|t9<>64P2
za_3cXgxzBlzOvg`bY^z%T~7dRqXpt0x*SQVG<f)cGmg%3E&3Tlwn1n(U8;OhAZ~zm
zl{f<f6|3AI9&{Kswyz!%80QR~1xD|ttjxmHpP1?A!L#D$;thOu?qCF)mJTZip1Ufz
zg7baOU)_loULD5@)Ggew#y`GU(7zIb0XIVmc<EgD^TKFJmV_&P=6-F(<#*__Lw+<?
zI^eD;`tN=~5#)dfxX1|p+nyGHvVz_(w+fjhqmIzx5^E3SnFEt~0lUqMSy_B?P#Nb|
zVJ<S}d3M8G`)lTUU~~~EaDB(<_7sQ(rft!86b^ifZ;#`gVf#&xPLq)+cGAX*_4ZPd
ztI0(rT8gNq!EXalV2OtEh)jtjuN0^(^aCgt^fp<G_PH7tSps_!QSak^_f{XX2U;oU
z%c^poXV%(T4b&<siV4vQwWhwko(;8zf0K$ps)?mx;EHj}&L#UwN@=Z^XkZfJ`A>ed
zF!<c#(s)xadO{Fzrd;&Vcy`2p*csp>JP&jI#jOObg%ICDfU6UUt$GEpsv_hFg-Dsr
z#N?-~h19v0$?!r6Dd$RQc6B{!nocvaO8+r^biX<+`>@%>BNdGCm-)*M@@Exo4j5mj
zlu6O!^gp;B0&8tGfNLqN8JM6K`3B@R!L&ab;jx>%QlQ@>l@b?#Q0V*VECDrWkfm*`
zmzJn_6KoHk4Bq)(W3qHSKEiL7$zLq6*~QOqQn?n@(sEO(#%<u&H)d}B?T5>2R-1zB
zzY+eL05M<w=FVOY>=TyTVmKbd2od7I=32O-@j-a^;}e}@zNg20-!s@nj3Jg#nNHcB
z8qPFW{;~HPl^{0%^;+;sxh=eF2dz&b_;oO~^wjxj_bF1%x|9S{a^Hry3Z%NV1EOx6
zbH<Q4A>u)*X*v*~wf)Xu*YG7%AXnu46o(p|_%PHdr7KLz<_x1s9vq*n+qIMKX)<!z
zSRM4uN>tJNYUs7=txuwcXku#XYm@iSljA{{OEFU@LU7p84V3(VP3nBiP;~Lscb8sG
z@gqpLr%Jn$pGERy`EuC6TyF17H(%LvxP1UoeH+@uSS_!l{_y;z0l?fVpXYB5R*6<x
z!)P$8*cCAyYIv`s8E#_E8?dv9v1N2fUcWpWo{(zdH_dNl=-8{aO<`)i^xVRCx`}Tw
zRT1#zj7C-=TtwQbuTNzgd-JTfo$~aJO7%5iQ0AQH&O|W##5CvcM5q{pO8gFbEZ(jm
zhL=3MVqxE+O`d}%j-g0D;J7Zlen-=EB9@;SdN2$13k$nCaA_@POw{4HwOG^#l@|=%
z!Cb!-{%THYM%#Nkf#7{;x+<I{SXpsFfIlMXA$rUO>rdK~UD-*xPY$6g6Z8|bv|Oem
zZ@BMJmU)RZ_z3b?4JsiDI{ER@y!giZ%W>NtC$<JGTN%GdZ5ek!WpdiO<i6ET{g}Ca
zMSgi^JT@-CIE3yi`xi#OVu7U7Vf{-;y5Ehjt&d}6R>H{5Y@W@+nJ5;YF9VYi?@!yV
zzm9GoOiac=l=y%=oG0u?TxF|NxJJRFK&9Oqn5*UHd5w0$QPrMyak~h*A4x22si4nW
zU8Tyx6XZK~0p9P_iutzroTrW&4l|jHP2{UFA5r~G$PWMMRav9Q1ry(ia!)K+%t|^W
z+;rg1r_~r!cf!{#+$PsqMR1qRT5@kr)%s^Bn=I{(y!C0ja4y)Ec!YWXo-H6o{^^AR
zI&k?+%&gC0DtGPO8PtOqN)4YB7}_;Jc-7V5OiFy5y(&^`Fm%xov$&z++Dx6G{A5Xd
z&d}U)tk74^^vg4q+`B+N9)w4N2&?vM3-jPqkuCpEhM&`~eH@y4q0`OX5kOFr8w{Zk
z#6nr3=|Q9!p?9&mu4?PJz<AQ1$zmGb(?c{%v{tt2k6Icz^A1AV_CAl)+J9fx^JXJY
zH=X}{%7JgKGbO&faY?(J^SPq>N&Or5$HJ_UPv9_?1tlj`aiNAOT38ucKS39NpDTdM
zS6t|ats<EDgp{puW0!9&wMiDI;5_>3?$ep~tPFsKm@3TNSD)FJXkS9j0m&<5rQ!>j
z5U0!vZ{7~gYy;fh@>g1+NvgQ4UUS~FI6um4U2&UdS3XU2k%BP&!k!OSV2E9j?d0Q)
z@WyRRY#u725jka>+AnCKM9T0GSoA5fGXhMQd_g(>Wir43sTvsTFa;e{C8>BA{^Etz
z=C>&MKHsyxBn-oJqn4E6C%UA9R*i8Xc8P*5P1B4mKeOj8Zfn8E@$Vv>ZzWi@bYKT&
zah~+8E?G11%q#ZvjAVF#9tD&=jxv4#NOxM3^&xP*sFbxE?NV#g1Ye2+?EcH~N)GFQ
zQe*WQ7ebZ?u{R&Q=&cXNa$aCJhi}?mojhy=9(kh4B#bH&^q)W{vR@p>Zgi3Ap2gJw
zS(()*6Mr22c23UtlY8_`K28~Jn;+XO#!F)t2r27Ahnp&8&WOLuJl6gtkj_1_%>0Wo
zm<xqI98Iq*-7OGtOB{PTmCC%+ApZ|pUbxNMBRdrst-?jgraOnf+pEngFvD5KB%q5z
zdFBdh?QPX?%qG?{rQqh>Q_B>@A7N615Xl#gZ7NGn+(|Rz*2jy^B+X84h6sdkl&(AO
za^3jPyz{#xh^yDcc+Es=$9KfO>+GU^nM)#z72sRO&W3qmE$T~_&ao4j4eGCA{kyKw
zjT_V()=XCMSn3v29GR({&&+~(DNXj9kdq$o$LUX`{+9Ks+6kq36~5WR9oY#X0m|or
zHd^Mh#`8x98G9P_c<ayduU6n-=qcQ>@K%)v8!*e+zgn9?3&uH}Mmx=uO$KmvjY-pA
z66}JaW+SduQ!-=h7MCtzob$vtX*laq!jGwvcS#!Eoetk|k0<o>@Ud}D*~IpWdYbHG
z&fji3lAG^dU_#zai7H>L+r$beAv7@JIw33q(p(rz@aSF&G+wuyr#O<RG(KWBWU^^R
zD(<3R7QjWmOgw2$+vMj*lgPJOGofBeWu$hEwOan76Zyqpwhj?!;B{$Bv3m*e-)l6B
zTwS2gl>Iq4C>`n6*G1~pRuEgcAZX1Wxt$u7*QTmfl{%A^G{mLcW8|^`(6A^I^1~Hl
zWvhjKPjoapTxxuLseGZjM$p@iq1)W@BMO-<ai1k}*A?!vypR7ITmA>Z{GYbBl$B)y
WOLwA#{(U|O2vig_<SS$?LjMa?H7_Rs

literal 0
HcmV?d00001

diff --git a/frontend/src/pages/GeneralSettings/AudioPreference/index.jsx b/frontend/src/pages/GeneralSettings/AudioPreference/index.jsx
new file mode 100644
index 00000000000..c4abaf5461e
--- /dev/null
+++ b/frontend/src/pages/GeneralSettings/AudioPreference/index.jsx
@@ -0,0 +1,45 @@
+import React, { useEffect, useState, useRef } from "react";
+import { isMobile } from "react-device-detect";
+import Sidebar from "@/components/SettingsSidebar";
+import System from "@/models/system";
+import PreLoader from "@/components/Preloader";
+import SpeechToTextProvider from "./stt";
+import TextToSpeechProvider from "./tts";
+
+export default function AudioPreference() {
+  const [settings, setSettings] = useState(null);
+  const [loading, setLoading] = useState(true);
+
+  useEffect(() => {
+    async function fetchKeys() {
+      const _settings = await System.keys();
+      setSettings(_settings);
+      setLoading(false);
+    }
+    fetchKeys();
+  }, []);
+
+  return (
+    <div className="w-screen h-screen overflow-hidden bg-sidebar flex">
+      <Sidebar />
+      {loading ? (
+        <div
+          style={{ height: isMobile ? "100%" : "calc(100% - 32px)" }}
+          className="relative md:ml-[2px] md:mr-[16px] md:my-[16px] md:rounded-[16px] bg-main-gradient w-full h-full overflow-y-scroll"
+        >
+          <div className="w-full h-full flex justify-center items-center">
+            <PreLoader />
+          </div>
+        </div>
+      ) : (
+        <div
+          style={{ height: isMobile ? "100%" : "calc(100% - 32px)" }}
+          className="relative md:ml-[2px] md:mr-[16px] md:my-[16px] md:rounded-[16px] bg-main-gradient w-full h-full overflow-y-scroll"
+        >
+          <SpeechToTextProvider settings={settings} />
+          <TextToSpeechProvider settings={settings} />
+        </div>
+      )}
+    </div>
+  );
+}
diff --git a/frontend/src/pages/GeneralSettings/AudioPreference/stt.jsx b/frontend/src/pages/GeneralSettings/AudioPreference/stt.jsx
new file mode 100644
index 00000000000..58bb1489be1
--- /dev/null
+++ b/frontend/src/pages/GeneralSettings/AudioPreference/stt.jsx
@@ -0,0 +1,191 @@
+import React, { useEffect, useState, useRef } from "react";
+import System from "@/models/system";
+import showToast from "@/utils/toast";
+import LLMItem from "@/components/LLMSelection/LLMItem";
+import { CaretUpDown, MagnifyingGlass, X } from "@phosphor-icons/react";
+import CTAButton from "@/components/lib/CTAButton";
+import AnythingLLMIcon from "@/media/logo/anything-llm-icon.png";
+import BrowserNative from "@/components/SpeechToText/BrowserNative";
+
+const PROVIDERS = [
+  {
+    name: "System native",
+    value: "native",
+    logo: AnythingLLMIcon,
+    options: (settings) => <BrowserNative settings={settings} />,
+    description: "Uses your browser's built in STT service if supported.",
+  },
+];
+
+export default function SpeechToTextProvider({ settings }) {
+  const [saving, setSaving] = useState(false);
+  const [hasChanges, setHasChanges] = useState(false);
+  const [searchQuery, setSearchQuery] = useState("");
+  const [filteredProviders, setFilteredProviders] = useState([]);
+  const [selectedProvider, setSelectedProvider] = useState(
+    settings?.SpeechToTextProvider || "native"
+  );
+  const [searchMenuOpen, setSearchMenuOpen] = useState(false);
+  const searchInputRef = useRef(null);
+
+  const handleSubmit = async (e) => {
+    e.preventDefault();
+    const form = e.target;
+    const data = { SpeechToTextProvider: selectedProvider };
+    const formData = new FormData(form);
+
+    for (var [key, value] of formData.entries()) data[key] = value;
+    const { error } = await System.updateSystem(data);
+    setSaving(true);
+
+    if (error) {
+      showToast(`Failed to save preferences: ${error}`, "error");
+    } else {
+      showToast("Speech-to-text preferences saved successfully.", "success");
+    }
+    setSaving(false);
+    setHasChanges(!!error);
+  };
+
+  const updateProviderChoice = (selection) => {
+    setSearchQuery("");
+    setSelectedProvider(selection);
+    setSearchMenuOpen(false);
+    setHasChanges(true);
+  };
+
+  const handleXButton = () => {
+    if (searchQuery.length > 0) {
+      setSearchQuery("");
+      if (searchInputRef.current) searchInputRef.current.value = "";
+    } else {
+      setSearchMenuOpen(!searchMenuOpen);
+    }
+  };
+
+  useEffect(() => {
+    const filtered = PROVIDERS.filter((provider) =>
+      provider.name.toLowerCase().includes(searchQuery.toLowerCase())
+    );
+    setFilteredProviders(filtered);
+  }, [searchQuery, selectedProvider]);
+
+  const selectedProviderObject = PROVIDERS.find(
+    (provider) => provider.value === selectedProvider
+  );
+
+  return (
+    <form onSubmit={handleSubmit} className="flex w-full">
+      <div className="flex flex-col w-full px-1 md:pl-6 md:pr-[50px] md:py-6 py-16">
+        <div className="w-full flex flex-col gap-y-1 pb-6 border-white border-b-2 border-opacity-10">
+          <div className="flex gap-x-4 items-center">
+            <p className="text-lg leading-6 font-bold text-white">
+              Speech-to-text Preference
+            </p>
+          </div>
+          <p className="text-xs leading-[18px] font-base text-white text-opacity-60">
+            Here you can specify what kind of text-to-speech and speech-to-text
+            providers you would want to use in your AnythingLLM experience. By
+            default, we use the browser's built in support for these services,
+            but you may want to use others.
+          </p>
+        </div>
+        <div className="w-full justify-end flex">
+          {hasChanges && (
+            <CTAButton
+              onClick={() => handleSubmit()}
+              className="mt-3 mr-0 -mb-14 z-10"
+            >
+              {saving ? "Saving..." : "Save changes"}
+            </CTAButton>
+          )}
+        </div>
+        <div className="text-base font-bold text-white mt-6 mb-4">Provider</div>
+        <div className="relative">
+          {searchMenuOpen && (
+            <div
+              className="fixed top-0 left-0 w-full h-full bg-black bg-opacity-70 backdrop-blur-sm z-10"
+              onClick={() => setSearchMenuOpen(false)}
+            />
+          )}
+          {searchMenuOpen ? (
+            <div className="absolute top-0 left-0 w-full max-w-[640px] max-h-[310px] overflow-auto white-scrollbar min-h-[64px] bg-[#18181B] rounded-lg flex flex-col justify-between cursor-pointer border-2 border-[#46C8FF] z-20">
+              <div className="w-full flex flex-col gap-y-1">
+                <div className="flex items-center sticky top-0 border-b border-[#9CA3AF] mx-4 bg-[#18181B]">
+                  <MagnifyingGlass
+                    size={20}
+                    weight="bold"
+                    className="absolute left-4 z-30 text-white -ml-4 my-2"
+                  />
+                  <input
+                    type="text"
+                    name="stt-provider-search"
+                    autoComplete="off"
+                    placeholder="Search speech to text providers"
+                    className="-ml-4 my-2 bg-transparent z-20 pl-12 h-[38px] w-full px-4 py-1 text-sm outline-none focus:border-white text-white placeholder:text-white placeholder:font-medium"
+                    onChange={(e) => setSearchQuery(e.target.value)}
+                    ref={searchInputRef}
+                    onKeyDown={(e) => {
+                      if (e.key === "Enter") e.preventDefault();
+                    }}
+                  />
+                  <X
+                    size={20}
+                    weight="bold"
+                    className="cursor-pointer text-white hover:text-[#9CA3AF]"
+                    onClick={handleXButton}
+                  />
+                </div>
+                <div className="flex-1 pl-4 pr-2 flex flex-col gap-y-1 overflow-y-auto white-scrollbar pb-4">
+                  {filteredProviders.map((provider) => (
+                    <LLMItem
+                      key={provider.name}
+                      name={provider.name}
+                      value={provider.value}
+                      image={provider.logo}
+                      description={provider.description}
+                      checked={selectedProvider === provider.value}
+                      onClick={() => updateProviderChoice(provider.value)}
+                    />
+                  ))}
+                </div>
+              </div>
+            </div>
+          ) : (
+            <button
+              className="w-full max-w-[640px] h-[64px] bg-[#18181B] rounded-lg flex items-center p-[14px] justify-between cursor-pointer border-2 border-transparent hover:border-[#46C8FF] transition-all duration-300"
+              type="button"
+              onClick={() => setSearchMenuOpen(true)}
+            >
+              <div className="flex gap-x-4 items-center">
+                <img
+                  src={selectedProviderObject.logo}
+                  alt={`${selectedProviderObject.name} logo`}
+                  className="w-10 h-10 rounded-md"
+                />
+                <div className="flex flex-col text-left">
+                  <div className="text-sm font-semibold text-white">
+                    {selectedProviderObject.name}
+                  </div>
+                  <div className="mt-1 text-xs text-[#D2D5DB]">
+                    {selectedProviderObject.description}
+                  </div>
+                </div>
+              </div>
+              <CaretUpDown size={24} weight="bold" className="text-white" />
+            </button>
+          )}
+        </div>
+        <div
+          onChange={() => setHasChanges(true)}
+          className="mt-4 flex flex-col gap-y-1"
+        >
+          {selectedProvider &&
+            PROVIDERS.find(
+              (provider) => provider.value === selectedProvider
+            )?.options(settings)}
+        </div>
+      </div>
+    </form>
+  );
+}
diff --git a/frontend/src/pages/GeneralSettings/AudioPreference/tts.jsx b/frontend/src/pages/GeneralSettings/AudioPreference/tts.jsx
new file mode 100644
index 00000000000..6b11f1a462e
--- /dev/null
+++ b/frontend/src/pages/GeneralSettings/AudioPreference/tts.jsx
@@ -0,0 +1,209 @@
+import React, { useEffect, useState, useRef } from "react";
+import System from "@/models/system";
+import showToast from "@/utils/toast";
+import LLMItem from "@/components/LLMSelection/LLMItem";
+import { CaretUpDown, MagnifyingGlass, X } from "@phosphor-icons/react";
+import CTAButton from "@/components/lib/CTAButton";
+import OpenAiLogo from "@/media/llmprovider/openai.png";
+import AnythingLLMIcon from "@/media/logo/anything-llm-icon.png";
+import ElevenLabsIcon from "@/media/ttsproviders/elevenlabs.png";
+import BrowserNative from "@/components/TextToSpeech/BrowserNative";
+import OpenAiTTSOptions from "@/components/TextToSpeech/OpenAiOptions";
+import ElevenLabsTTSOptions from "@/components/TextToSpeech/ElevenLabsOptions";
+
+const PROVIDERS = [
+  {
+    name: "System native",
+    value: "native",
+    logo: AnythingLLMIcon,
+    options: (settings) => <BrowserNative settings={settings} />,
+    description: "Uses your browser's built in TTS service if supported.",
+  },
+  {
+    name: "OpenAI",
+    value: "openai",
+    logo: OpenAiLogo,
+    options: (settings) => <OpenAiTTSOptions settings={settings} />,
+    description: "Use OpenAI's text to speech voices.",
+  },
+  {
+    name: "ElevenLabs",
+    value: "elevenlabs",
+    logo: ElevenLabsIcon,
+    options: (settings) => <ElevenLabsTTSOptions settings={settings} />,
+    description: "Use ElevenLabs's text to speech voices and technology.",
+  },
+];
+
+export default function TextToSpeechProvider({ settings }) {
+  const [saving, setSaving] = useState(false);
+  const [hasChanges, setHasChanges] = useState(false);
+  const [searchQuery, setSearchQuery] = useState("");
+  const [filteredProviders, setFilteredProviders] = useState([]);
+  const [selectedProvider, setSelectedProvider] = useState(
+    settings?.TextToSpeechProvider || "native"
+  );
+  const [searchMenuOpen, setSearchMenuOpen] = useState(false);
+  const searchInputRef = useRef(null);
+
+  const handleSubmit = async (e) => {
+    e.preventDefault();
+    const form = e.target;
+    const data = { TextToSpeechProvider: selectedProvider };
+    const formData = new FormData(form);
+
+    for (var [key, value] of formData.entries()) data[key] = value;
+    const { error } = await System.updateSystem(data);
+    setSaving(true);
+
+    if (error) {
+      showToast(`Failed to save preferences: ${error}`, "error");
+    } else {
+      showToast("Text-to-speech preferences saved successfully.", "success");
+    }
+    setSaving(false);
+    setHasChanges(!!error);
+  };
+
+  const updateProviderChoice = (selection) => {
+    setSearchQuery("");
+    setSelectedProvider(selection);
+    setSearchMenuOpen(false);
+    setHasChanges(true);
+  };
+
+  const handleXButton = () => {
+    if (searchQuery.length > 0) {
+      setSearchQuery("");
+      if (searchInputRef.current) searchInputRef.current.value = "";
+    } else {
+      setSearchMenuOpen(!searchMenuOpen);
+    }
+  };
+
+  useEffect(() => {
+    const filtered = PROVIDERS.filter((provider) =>
+      provider.name.toLowerCase().includes(searchQuery.toLowerCase())
+    );
+    setFilteredProviders(filtered);
+  }, [searchQuery, selectedProvider]);
+
+  const selectedProviderObject = PROVIDERS.find(
+    (provider) => provider.value === selectedProvider
+  );
+
+  return (
+    <form onSubmit={handleSubmit} className="flex w-full">
+      <div className="flex flex-col w-full px-1 md:pl-6 md:pr-[50px] md:py-6 py-16">
+        <div className="w-full flex flex-col gap-y-1 pb-6 border-white border-b-2 border-opacity-10">
+          <div className="flex gap-x-4 items-center">
+            <p className="text-lg leading-6 font-bold text-white">
+              Text-to-speech Preference
+            </p>
+          </div>
+          <p className="text-xs leading-[18px] font-base text-white text-opacity-60">
+            Here you can specify what kind of text-to-speech providers you would
+            want to use in your AnythingLLM experience. By default, we use the
+            browser's built in support for these services, but you may want to
+            use others.
+          </p>
+        </div>
+        <div className="w-full justify-end flex">
+          {hasChanges && (
+            <CTAButton
+              onClick={() => handleSubmit()}
+              className="mt-3 mr-0 -mb-14 z-10"
+            >
+              {saving ? "Saving..." : "Save changes"}
+            </CTAButton>
+          )}
+        </div>
+        <div className="text-base font-bold text-white mt-6 mb-4">Provider</div>
+        <div className="relative">
+          {searchMenuOpen && (
+            <div
+              className="fixed top-0 left-0 w-full h-full bg-black bg-opacity-70 backdrop-blur-sm z-10"
+              onClick={() => setSearchMenuOpen(false)}
+            />
+          )}
+          {searchMenuOpen ? (
+            <div className="absolute top-0 left-0 w-full max-w-[640px] max-h-[310px] overflow-auto white-scrollbar min-h-[64px] bg-[#18181B] rounded-lg flex flex-col justify-between cursor-pointer border-2 border-[#46C8FF] z-20">
+              <div className="w-full flex flex-col gap-y-1">
+                <div className="flex items-center sticky top-0 border-b border-[#9CA3AF] mx-4 bg-[#18181B]">
+                  <MagnifyingGlass
+                    size={20}
+                    weight="bold"
+                    className="absolute left-4 z-30 text-white -ml-4 my-2"
+                  />
+                  <input
+                    type="text"
+                    name="tts-provider-search"
+                    autoComplete="off"
+                    placeholder="Search text to speech providers"
+                    className="-ml-4 my-2 bg-transparent z-20 pl-12 h-[38px] w-full px-4 py-1 text-sm outline-none focus:border-white text-white placeholder:text-white placeholder:font-medium"
+                    onChange={(e) => setSearchQuery(e.target.value)}
+                    ref={searchInputRef}
+                    onKeyDown={(e) => {
+                      if (e.key === "Enter") e.preventDefault();
+                    }}
+                  />
+                  <X
+                    size={20}
+                    weight="bold"
+                    className="cursor-pointer text-white hover:text-[#9CA3AF]"
+                    onClick={handleXButton}
+                  />
+                </div>
+                <div className="flex-1 pl-4 pr-2 flex flex-col gap-y-1 overflow-y-auto white-scrollbar pb-4">
+                  {filteredProviders.map((provider) => (
+                    <LLMItem
+                      key={provider.name}
+                      name={provider.name}
+                      value={provider.value}
+                      image={provider.logo}
+                      description={provider.description}
+                      checked={selectedProvider === provider.value}
+                      onClick={() => updateProviderChoice(provider.value)}
+                    />
+                  ))}
+                </div>
+              </div>
+            </div>
+          ) : (
+            <button
+              className="w-full max-w-[640px] h-[64px] bg-[#18181B] rounded-lg flex items-center p-[14px] justify-between cursor-pointer border-2 border-transparent hover:border-[#46C8FF] transition-all duration-300"
+              type="button"
+              onClick={() => setSearchMenuOpen(true)}
+            >
+              <div className="flex gap-x-4 items-center">
+                <img
+                  src={selectedProviderObject.logo}
+                  alt={`${selectedProviderObject.name} logo`}
+                  className="w-10 h-10 rounded-md"
+                />
+                <div className="flex flex-col text-left">
+                  <div className="text-sm font-semibold text-white">
+                    {selectedProviderObject.name}
+                  </div>
+                  <div className="mt-1 text-xs text-[#D2D5DB]">
+                    {selectedProviderObject.description}
+                  </div>
+                </div>
+              </div>
+              <CaretUpDown size={24} weight="bold" className="text-white" />
+            </button>
+          )}
+        </div>
+        <div
+          onChange={() => setHasChanges(true)}
+          className="mt-4 flex flex-col gap-y-1"
+        >
+          {selectedProvider &&
+            PROVIDERS.find(
+              (provider) => provider.value === selectedProvider
+            )?.options(settings)}
+        </div>
+      </div>
+    </form>
+  );
+}
diff --git a/server/utils/TextToSpeech/elevenLabs/index.js b/server/utils/TextToSpeech/elevenLabs/index.js
new file mode 100644
index 00000000000..e3d25f3ae9b
--- /dev/null
+++ b/server/utils/TextToSpeech/elevenLabs/index.js
@@ -0,0 +1,54 @@
+const { ElevenLabsClient, stream } = require("elevenlabs");
+
+class ElevenLabsTTS {
+  constructor() {
+    if (!process.env.TTS_ELEVEN_LABS_KEY)
+      throw new Error("No ElevenLabs API key was set.");
+    this.elevenLabs = new ElevenLabsClient({
+      apiKey: process.env.TTS_ELEVEN_LABS_KEY,
+    });
+
+    // Rachel as default voice
+    // https://api.elevenlabs.io/v1/voices
+    this.voiceId =
+      process.env.TTS_ELEVEN_LABS_VOICE_MODEL ?? "21m00Tcm4TlvDq8ikWAM";
+    this.modelId = "eleven_multilingual_v2";
+  }
+
+  static async voices(apiKey = null) {
+    try {
+      const client = new ElevenLabsClient({
+        apiKey: apiKey ?? process.env.TTS_ELEVEN_LABS_KEY ?? null,
+      });
+      return (await client.voices.getAll())?.voices ?? [];
+    } catch {}
+    return [];
+  }
+
+  #stream2buffer(stream) {
+    return new Promise((resolve, reject) => {
+      const _buf = [];
+      stream.on("data", (chunk) => _buf.push(chunk));
+      stream.on("end", () => resolve(Buffer.concat(_buf)));
+      stream.on("error", (err) => reject(err));
+    });
+  }
+
+  async ttsBuffer(textInput) {
+    try {
+      const audio = await this.elevenLabs.generate({
+        voice: this.voiceId,
+        text: textInput,
+        model_id: "eleven_multilingual_v2",
+      });
+      return Buffer.from(await this.#stream2buffer(audio));
+    } catch (e) {
+      console.error(e);
+    }
+    return null;
+  }
+}
+
+module.exports = {
+  ElevenLabsTTS,
+};
diff --git a/server/utils/TextToSpeech/index.js b/server/utils/TextToSpeech/index.js
new file mode 100644
index 00000000000..155fc95405a
--- /dev/null
+++ b/server/utils/TextToSpeech/index.js
@@ -0,0 +1,15 @@
+function getTTSProvider() {
+  const provider = process.env.TTS_PROVIDER || "openai";
+  switch (provider) {
+    case "openai":
+      const { OpenAiTTS } = require("./openAi");
+      return new OpenAiTTS();
+    case "elevenlabs":
+      const { ElevenLabsTTS } = require("./elevenLabs");
+      return new ElevenLabsTTS();
+    default:
+      throw new Error("ENV: No TTS_PROVIDER value found in environment!");
+  }
+}
+
+module.exports = { getTTSProvider };
diff --git a/server/utils/TextToSpeech/openAi/index.js b/server/utils/TextToSpeech/openAi/index.js
new file mode 100644
index 00000000000..3c5b4840d28
--- /dev/null
+++ b/server/utils/TextToSpeech/openAi/index.js
@@ -0,0 +1,29 @@
+class OpenAiTTS {
+  constructor() {
+    if (!process.env.TTS_OPEN_AI_KEY)
+      throw new Error("No OpenAI API key was set.");
+    const { OpenAI: OpenAIApi } = require("openai");
+    this.openai = new OpenAIApi({
+      apiKey: process.env.TTS_OPEN_AI_KEY,
+    });
+    this.voice = process.env.TTS_OPEN_AI_VOICE_MODEL ?? "alloy";
+  }
+
+  async ttsBuffer(textInput) {
+    try {
+      const result = await this.openai.audio.speech.create({
+        model: "tts-1",
+        voice: this.voice,
+        input: textInput,
+      });
+      return Buffer.from(await result.arrayBuffer());
+    } catch (e) {
+      console.error(e);
+    }
+    return null;
+  }
+}
+
+module.exports = {
+  OpenAiTTS,
+};
diff --git a/server/utils/helpers/updateENV.js b/server/utils/helpers/updateENV.js
index beb03a566e7..e2b1d2e1c97 100644
--- a/server/utils/helpers/updateENV.js
+++ b/server/utils/helpers/updateENV.js
@@ -383,7 +383,7 @@ const KEY_MAPPING = {
     checks: [],
   },
 
-  // TTS EleventLabs
+  // TTS ElevenLabs
   TTSElevenLabsKey: {
     envKey: "TTS_ELEVEN_LABS_KEY",
     checks: [isNotEmpty],

From eaf9481e1f6379558d218a93f787f169040d25ca Mon Sep 17 00:00:00 2001
From: timothycarambat <rambat1010@gmail.com>
Date: Mon, 13 May 2024 23:06:58 -0700
Subject: [PATCH 3/4] update comments

---
 .../src/components/TextToSpeech/ElevenLabsOptions/index.jsx   | 4 ++--
 .../HistoricalMessage/Actions/TTSButton/native.jsx            | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/frontend/src/components/TextToSpeech/ElevenLabsOptions/index.jsx b/frontend/src/components/TextToSpeech/ElevenLabsOptions/index.jsx
index 5b4232e8436..ad86caa1cbe 100644
--- a/frontend/src/components/TextToSpeech/ElevenLabsOptions/index.jsx
+++ b/frontend/src/components/TextToSpeech/ElevenLabsOptions/index.jsx
@@ -64,7 +64,7 @@ function ElevenLabsModelSelection({ apiKey, settings }) {
           Chat Model Selection
         </label>
         <select
-          name="OpenAiModelPref"
+          name="TTSElevenLabsVoiceModel"
           disabled={true}
           className="bg-zinc-900 border-gray-500 text-white text-sm rounded-lg block w-full p-2.5"
         >
@@ -82,7 +82,7 @@ function ElevenLabsModelSelection({ apiKey, settings }) {
         Chat Model Selection
       </label>
       <select
-        name="OpenAiModelPref"
+        name="TTSElevenLabsVoiceModel"
         required={true}
         className="bg-zinc-900 border-gray-500 text-white text-sm rounded-lg block w-full p-2.5"
       >
diff --git a/frontend/src/components/WorkspaceChat/ChatContainer/ChatHistory/HistoricalMessage/Actions/TTSButton/native.jsx b/frontend/src/components/WorkspaceChat/ChatContainer/ChatHistory/HistoricalMessage/Actions/TTSButton/native.jsx
index 5d9d4fe5318..5f3bd3f69cd 100644
--- a/frontend/src/components/WorkspaceChat/ChatContainer/ChatHistory/HistoricalMessage/Actions/TTSButton/native.jsx
+++ b/frontend/src/components/WorkspaceChat/ChatContainer/ChatHistory/HistoricalMessage/Actions/TTSButton/native.jsx
@@ -17,7 +17,7 @@ export default function NativeTTSMessage({ message }) {
 
   function speakMessage() {
     // if the user is pausing this particular message
-    // while the synth if speaking we can end it.
+    // while the synth is speaking we can end it.
     // If they are clicking another message's TTS
     // we need to ignore that until they pause the one that is playing.
     if (window.speechSynthesis.speaking && speaking) {

From e4aba9ec082ba493ea74ee16d83542447ea1390c Mon Sep 17 00:00:00 2001
From: timothycarambat <rambat1010@gmail.com>
Date: Tue, 14 May 2024 10:35:48 -0700
Subject: [PATCH 4/4] patch: bad playerRef check

---
 .../HistoricalMessage/Actions/TTSButton/asyncTts.jsx            | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/frontend/src/components/WorkspaceChat/ChatContainer/ChatHistory/HistoricalMessage/Actions/TTSButton/asyncTts.jsx b/frontend/src/components/WorkspaceChat/ChatContainer/ChatHistory/HistoricalMessage/Actions/TTSButton/asyncTts.jsx
index e58d4df3041..1947f0057f4 100644
--- a/frontend/src/components/WorkspaceChat/ChatContainer/ChatHistory/HistoricalMessage/Actions/TTSButton/asyncTts.jsx
+++ b/frontend/src/components/WorkspaceChat/ChatContainer/ChatHistory/HistoricalMessage/Actions/TTSButton/asyncTts.jsx
@@ -39,7 +39,7 @@ export default function AsyncTTSMessage({ slug, chatId }) {
 
   useEffect(() => {
     function setupPlayer() {
-      if (!playerRef) return;
+      if (!playerRef?.current) return;
       playerRef.current.addEventListener("play", () => {
         setSpeaking(true);
       });