From bd8043cf827c76f015881035d85098eae41738ff Mon Sep 17 00:00:00 2001 From: timothycarambat Date: Wed, 1 May 2024 12:47:50 -0700 Subject: [PATCH 1/5] add LMStudio agent support (generic) support "work" with non-tool callable LLMs, highly dependent on system specs --- .vscode/settings.json | 1 + .../AgentConfig/AgentLLMSelection/index.jsx | 2 +- server/endpoints/agentWebsocket.js | 2 +- server/package.json | 1 + server/utils/agents/aibitat/index.js | 2 + .../agents/aibitat/plugins/chat-history.js | 1 + .../agents/aibitat/providers/ai-provider.js | 20 +- .../utils/agents/aibitat/providers/classes.js | 16 ++ .../utils/agents/aibitat/providers/index.js | 2 + .../agents/aibitat/providers/lmstudio.js | 89 ++++++++ .../agents/aibitat/providers/untooled.js | 212 ++++++++++++++++++ server/utils/agents/defaults.js | 5 +- server/utils/agents/index.js | 22 +- server/utils/http/index.js | 11 + server/yarn.lock | 5 + 15 files changed, 384 insertions(+), 7 deletions(-) create mode 100644 server/utils/agents/aibitat/providers/classes.js create mode 100644 server/utils/agents/aibitat/providers/lmstudio.js create mode 100644 server/utils/agents/aibitat/providers/untooled.js diff --git a/.vscode/settings.json b/.vscode/settings.json index b0fccedf04f..f850bbb003f 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -17,6 +17,7 @@ "hljs", "inferencing", "Langchain", + "lmstudio", "mbox", "Milvus", "Mintplex", diff --git a/frontend/src/pages/WorkspaceSettings/AgentConfig/AgentLLMSelection/index.jsx b/frontend/src/pages/WorkspaceSettings/AgentConfig/AgentLLMSelection/index.jsx index f1b99747090..51b384642d6 100644 --- a/frontend/src/pages/WorkspaceSettings/AgentConfig/AgentLLMSelection/index.jsx +++ b/frontend/src/pages/WorkspaceSettings/AgentConfig/AgentLLMSelection/index.jsx @@ -5,7 +5,7 @@ import { AVAILABLE_LLM_PROVIDERS } from "@/pages/GeneralSettings/LLMPreference"; import { CaretUpDown, MagnifyingGlass, X } from "@phosphor-icons/react"; import AgentModelSelection from "../AgentModelSelection"; -const ENABLED_PROVIDERS = ["openai", "anthropic"]; +const ENABLED_PROVIDERS = ["openai", "anthropic", "lmstudio"]; const LLM_DEFAULT = { name: "Please make a selection", diff --git a/server/endpoints/agentWebsocket.js b/server/endpoints/agentWebsocket.js index 9809c60737a..c5fc1475fb7 100644 --- a/server/endpoints/agentWebsocket.js +++ b/server/endpoints/agentWebsocket.js @@ -51,7 +51,7 @@ function agentWebsocket(app) { await agentHandler.createAIbitat({ socket }); await agentHandler.startAgentCluster(); } catch (e) { - console.error(e.message); + console.error(e.message, e); socket?.send(JSON.stringify({ type: "wssFailure", content: e.message })); socket?.close(); } diff --git a/server/package.json b/server/package.json index 5549ba713df..c673d80c214 100644 --- a/server/package.json +++ b/server/package.json @@ -50,6 +50,7 @@ "joi": "^17.11.0", "joi-password-complexity": "^5.2.0", "js-tiktoken": "^1.0.7", + "jsonrepair": "^3.7.0", "jsonwebtoken": "^8.5.1", "langchain": "0.1.36", "mime": "^3.0.0", diff --git a/server/utils/agents/aibitat/index.js b/server/utils/agents/aibitat/index.js index d1e9ae9c8a9..7fa09969a58 100644 --- a/server/utils/agents/aibitat/index.js +++ b/server/utils/agents/aibitat/index.js @@ -727,6 +727,8 @@ ${this.getHistory({ to: route.to }) return new Providers.OpenAIProvider({ model: config.model }); case "anthropic": return new Providers.AnthropicProvider({ model: config.model }); + case "lmstudio": + return new Providers.LMStudioProvider({}); default: throw new Error( diff --git a/server/utils/agents/aibitat/plugins/chat-history.js b/server/utils/agents/aibitat/plugins/chat-history.js index e3123a83bea..4d3f8fd1efe 100644 --- a/server/utils/agents/aibitat/plugins/chat-history.js +++ b/server/utils/agents/aibitat/plugins/chat-history.js @@ -13,6 +13,7 @@ const chatHistory = { name: this.name, setup: function (aibitat) { aibitat.onMessage(async () => { + return; try { const lastResponses = aibitat.chats.slice(-2); if (lastResponses.length !== 2) return; diff --git a/server/utils/agents/aibitat/providers/ai-provider.js b/server/utils/agents/aibitat/providers/ai-provider.js index 5c56cb62c74..0e871b36e1d 100644 --- a/server/utils/agents/aibitat/providers/ai-provider.js +++ b/server/utils/agents/aibitat/providers/ai-provider.js @@ -4,16 +4,25 @@ const { ChatOpenAI } = require("@langchain/openai"); const { ChatAnthropic } = require("@langchain/anthropic"); +const DEFAULT_WORKSPACE_PROMPT = + "You are a helpful ai assistant who can assist the user and use tools available to help answer the users prompts and questions."; class Provider { _client; constructor(client) { if (this.constructor == Provider) { - throw new Error("Class is of abstract type and can't be instantiated"); + return; } this._client = client; } + providerLog(text, ...args) { + console.log( + `\x1b[36m[AgentLLM${this?.model ? ` - ${this.model}` : ""}]\x1b[0m ${text}`, + ...args + ); + } + get client() { return this._client; } @@ -48,6 +57,15 @@ class Provider { return 8_000; } } + + static systemPrompt(provider = null) { + switch (provider) { + case "lmstudio": + return "You are a helpful ai assistant who can assist the user and use tools available to help answer the users prompts and questions. Tools will be handled by another assistant and you will simply receive their responses to help answer the user prompt - always try to answer the user's prompt the best you can with the context available to you and your general knowledge."; + default: + return DEFAULT_WORKSPACE_PROMPT; + } + } } module.exports = Provider; diff --git a/server/utils/agents/aibitat/providers/classes.js b/server/utils/agents/aibitat/providers/classes.js new file mode 100644 index 00000000000..3a6e959b1b7 --- /dev/null +++ b/server/utils/agents/aibitat/providers/classes.js @@ -0,0 +1,16 @@ +function InheritMultiple(bases = []) { + class Bases { + constructor() { + bases.forEach((base) => Object.assign(this, new base())); + } + } + + bases.forEach((base) => { + Object.getOwnPropertyNames(base.prototype) + .filter((prop) => prop != "constructor") + .forEach((prop) => (Bases.prototype[prop] = base.prototype[prop])); + }); + return Bases; +} + +module.exports = InheritMultiple; diff --git a/server/utils/agents/aibitat/providers/index.js b/server/utils/agents/aibitat/providers/index.js index b163b4cd0e7..ebe4de33f06 100644 --- a/server/utils/agents/aibitat/providers/index.js +++ b/server/utils/agents/aibitat/providers/index.js @@ -1,7 +1,9 @@ const OpenAIProvider = require("./openai.js"); const AnthropicProvider = require("./anthropic.js"); +const LMStudioProvider = require("./lmstudio.js"); module.exports = { OpenAIProvider, AnthropicProvider, + LMStudioProvider, }; diff --git a/server/utils/agents/aibitat/providers/lmstudio.js b/server/utils/agents/aibitat/providers/lmstudio.js new file mode 100644 index 00000000000..6fc73d5adb1 --- /dev/null +++ b/server/utils/agents/aibitat/providers/lmstudio.js @@ -0,0 +1,89 @@ +const OpenAI = require("openai"); +const InheritMultiple = require("./classes.js"); +const Provider = require("./ai-provider.js"); +const UnTooled = require("./untooled.js"); + +/** + * The provider for the LMStudio provider. + */ +class LMStudioProvider extends InheritMultiple([Provider, UnTooled]) { + model; + + constructor(_config = {}) { + super(); + const model = process.env.LMSTUDIO_MODEL_PREF || "Loaded from Chat UI"; + const client = new OpenAI({ + baseURL: process.env.LMSTUDIO_BASE_PATH?.replace(/\/+$/, ""), // here is the URL to your LMStudio instance + apiKey: null, + maxRetries: 3, + model, + }); + this._client = client; + this.model = model; + } + + get client() { + return this._client; + } + + /** + * Create a completion based on the received messages. + * + * @param messages A list of messages to send to the API. + * @param functions + * @returns The completion. + */ + async complete(messages, functions = null) { + try { + let completion; + if (functions.length > 0) { + const { toolCall, text } = await this.functionCall(messages, functions); + + if (toolCall !== null) { + this.providerLog(`Valid tool call found - running ${toolCall.name}.`); + this.deduplicator.trackRun(toolCall.name, toolCall.arguments); + return { + result: null, + functionCall: { + name: toolCall.name, + arguments: toolCall.arguments, + }, + cost: 0, + }; + } + completion = { content: text }; + } + + if (!completion?.content) { + this.providerLog( + "Will assume chat completion without tool call inputs." + ); + const response = await this.client.chat.completions.create({ + model: this.model, + messages: this.cleanMsgs(messages), + }); + completion = response.choices[0].message; + } + + return { + result: completion.content, + cost: 0, + }; + } catch (error) { + throw error; + } + } + + /** + * Get the cost of the completion. + * + * @param _usage The completion to get the cost for. + * @returns The cost of the completion. + * Stubbed since LMStudio has no cost basis. + */ + getCost(_usage) { + return 0; + } +} + +module.exports = LMStudioProvider; diff --git a/server/utils/agents/aibitat/providers/untooled.js b/server/utils/agents/aibitat/providers/untooled.js new file mode 100644 index 00000000000..91a515182e4 --- /dev/null +++ b/server/utils/agents/aibitat/providers/untooled.js @@ -0,0 +1,212 @@ +const { safeJsonParse } = require("../../../http"); +const { Deduplicator } = require("../utils/dedupe"); + +class UnTooled { + constructor() { + this.deduplicator = new Deduplicator(); + } + + cleanMsgs(messages) { + const modifiedMessages = []; + messages.forEach((msg) => { + if (msg.role === "function") { + const prevMsg = modifiedMessages[modifiedMessages.length - 1].content; + modifiedMessages[modifiedMessages.length - 1].content = + `${prevMsg}\n${msg.content}`; + return; + } + modifiedMessages.push(msg); + }); + return modifiedMessages; + } + + formatFuncs(functions = []) { + const funcs = []; + functions.forEach((def) => { + funcs.push({ + name: def.name, + description: def.description, + properties: def.properties, + }); + }); + + return JSON.stringify(funcs, null, 2); + } + + /** + * Check if two arrays of strings or numbers have the same values + * @param {string[]|number[]} arr1 + * @param {string[]|number[]} arr2 + * @param {Object} [opts] + * @param {boolean} [opts.enforceOrder] - By default (false), the order of the values in the arrays doesn't matter. + * @return {boolean} + */ + compareArrays(arr1, arr2, opts) { + function vKey(i, v) { + return (opts?.enforceOrder ? `${i}-` : "") + `${typeof v}-${v}`; + } + + if (arr1.length !== arr2.length) return false; + + const d1 = {}; + const d2 = {}; + for (let i = arr1.length - 1; i >= 0; i--) { + d1[vKey(i, arr1[i])] = true; + d2[vKey(i, arr2[i])] = true; + } + + for (let i = arr1.length - 1; i >= 0; i--) { + const v = vKey(i, arr1[i]); + if (d1[v] !== d2[v]) return false; + } + + for (let i = arr2.length - 1; i >= 0; i--) { + const v = vKey(i, arr2[i]); + if (d1[v] !== d2[v]) return false; + } + + return true; + } + + validFuncCall(functionCall = {}, functions = []) { + if ( + !functionCall || + !functionCall?.hasOwnProperty("name") || + !functionCall?.hasOwnProperty("arguments") + ) { + return { + valid: false, + reason: "Missing name or arguments in function call.", + }; + } + + const foundFunc = functions.find((def) => def.name === functionCall.name); + if (!foundFunc) { + return { valid: false, reason: "Function name does not exist." }; + } + + const props = Object.keys(foundFunc.parameters.properties); + const fProps = Object.keys(functionCall.arguments); + if (!this.compareArrays(props, fProps)) { + return { valid: false, reason: "Invalid argument schema match." }; + } + + return { valid: true, reason: null }; + } + + async functionCall(messages, functions) { + const history = [...messages].filter((msg) => + ["user", "assistant"].includes(msg.role) + ); + if (history[history.length - 1].role !== "user") return null; + + const response = await this.client.chat.completions + .create({ + model: this.model, + temperature: 0, + messages: [ + { + content: `You are a program which picks the most optimal function and parameters to call. +DO NOT HAVE TO PICK A FUNCTION IF IT WILL NOT HELP ANSWER OR FULFILL THE USER'S QUERY. +When a function is selection, respond in JSON with no additional text. +When there is no relevant function to call - return with a regular chat text response. +Your task is to pick a **single** function that we will use to call, if any seem useful or relevant for the user query. + +Example of Tool definitions: +[ + { + name: 'rag-memory', + description: 'Search against local documents for context that is relevant to the query or store a snippet of text into memory for retrieval later. Storing information should only be done when the user specifically requests for information to be remembered or saved to long-term memory. You should use this tool before search the internet for information.', + parameters: { + '$schema': 'http://json-schema.org/draft-07/schema#', + type: 'object', + properties: [ + action: { + type: "string", + enum: ["search", "store"], + description: + "The action we want to take to search for existing similar context or storage of new context.", + }, + content: { + type: "string", + description: + "The plain text to search our local documents with or to store in our vector database.", + }, + ], + additionalProperties: false + } + }, + { + name: 'web-scraping', + description: 'Scrapes the content of a webpage or online resource from a URL.', + parameters: { + '$schema': 'http://json-schema.org/draft-07/schema#', + type: 'object', + properties: [ + url: { + type: "string", + format: "uri", + description: "A web URL.", + }, + ], + additionalProperties: false + }, + } +] + +Example Query: +User: Scrape https://example.com +Your response (JSON ONLY): +{ + name: 'web-scraping'. + arguments: { + url: 'https://example.com' + } +} + +Example Query where tool cannot be used: +User: Hello! +Your response (Text Only): Hello, how are you today? + +The available function and their definitions are listed below - respond only with JSON or a regular chat response. +${this.formatFuncs(functions)} + +Now, assess the next function to call: +`, + role: "system", + }, + ...history, + ], + }) + .then((result) => { + if (!result.hasOwnProperty("choices")) + throw new Error("LMStudio chat: No results!"); + if (result.choices.length === 0) + throw new Error("LMStudio chat: No results length!"); + return result.choices[0].message.content; + }) + .catch((_) => { + return null; + }); + + const call = safeJsonParse(response, null); + if (call === null) return { toolCall: null, text: response }; // failed to parse, so must be text. + + const { valid, reason } = this.validFuncCall(call, functions); + if (!valid) { + this.providerLog(`Invalid function tool call: ${reason}.`); + return { toolCall: null, text: null }; + } + + if (this.deduplicator.isDuplicate(call.name, call.arguments)) { + this.providerLog( + `Function tool with exact arguments has already been called this stack.` + ); + return { toolCall: null, text: null }; + } + + return { toolCall: call, text: null }; + } +} + +module.exports = UnTooled; diff --git a/server/utils/agents/defaults.js b/server/utils/agents/defaults.js index a030778f4ba..4e12b90621c 100644 --- a/server/utils/agents/defaults.js +++ b/server/utils/agents/defaults.js @@ -1,6 +1,7 @@ const AgentPlugins = require("./aibitat/plugins"); const { SystemSettings } = require("../../models/systemSettings"); const { safeJsonParse } = require("../http"); +const Provider = require("./aibitat/providers/ai-provider"); const USER_AGENT = { name: "USER", @@ -14,7 +15,7 @@ const USER_AGENT = { const WORKSPACE_AGENT = { name: "@agent", - getDefinition: async () => { + getDefinition: async (provider = null) => { const defaultFunctions = [ AgentPlugins.memory.name, // RAG AgentPlugins.docSummarizer.name, // Doc Summary @@ -30,7 +31,7 @@ const WORKSPACE_AGENT = { }); return { - role: "You are a helpful ai assistant who can assist the user and use tools available to help answer the users prompts and questions.", + role: Provider.systemPrompt(provider), functions: defaultFunctions, }; }, diff --git a/server/utils/agents/index.js b/server/utils/agents/index.js index ce80fff4489..5e54c0b3f67 100644 --- a/server/utils/agents/index.js +++ b/server/utils/agents/index.js @@ -77,14 +77,32 @@ class AgentHandler { if (!process.env.ANTHROPIC_API_KEY) throw new Error("Anthropic API key must be provided to use agents."); break; + case "lmstudio": + if (!process.env.LMSTUDIO_BASE_PATH) + throw new Error("LMStudio bash path must be provided to use agents."); + break; default: throw new Error("No provider found to power agent cluster."); } } + #providerDefault() { + switch (this.provider) { + case "openai": + return "gpt-3.5-turbo"; + case "anthropic": + return "claude-3-sonnet-20240229"; + case "lmstudio": + return "server-default"; + default: + return "unknown"; + } + } + #providerSetupAndCheck() { this.provider = this.invocation.workspace.agentProvider || "openai"; - this.model = this.invocation.workspace.agentModel || "gpt-3.5-turbo"; + this.model = + this.invocation.workspace.agentModel || this.#providerDefault(); this.log(`Start ${this.#invocationUUID}::${this.provider}:${this.model}`); this.#checkSetup(); } @@ -137,7 +155,7 @@ class AgentHandler { this.aibitat.agent(USER_AGENT.name, await USER_AGENT.getDefinition()); this.aibitat.agent( WORKSPACE_AGENT.name, - await WORKSPACE_AGENT.getDefinition() + await WORKSPACE_AGENT.getDefinition(this.provider) ); this.#funcsToLoad = [ diff --git a/server/utils/http/index.js b/server/utils/http/index.js index eedc3315408..1fc8c5b961c 100644 --- a/server/utils/http/index.js +++ b/server/utils/http/index.js @@ -3,6 +3,7 @@ process.env.NODE_ENV === "development" : require("dotenv").config(); const JWT = require("jsonwebtoken"); const { User } = require("../../models/user"); +const { jsonrepair } = require("jsonrepair"); function reqBody(request) { return typeof request.body === "string" @@ -65,6 +66,16 @@ function safeJsonParse(jsonString, fallback = null) { try { return JSON.parse(jsonString); } catch {} + + // If the jsonString does not look like an Obj or Array, dont attempt + // to repair it. + if (jsonString?.startsWith("[") || jsonString?.startsWith("{")) { + try { + const repairedJson = jsonrepair(jsonString); + return JSON.parse(repairedJson); + } catch {} + } + return fallback; } diff --git a/server/yarn.lock b/server/yarn.lock index 49c202af657..34b9faa474c 100644 --- a/server/yarn.lock +++ b/server/yarn.lock @@ -3727,6 +3727,11 @@ jsonpointer@^5.0.1: resolved "https://registry.yarnpkg.com/jsonpointer/-/jsonpointer-5.0.1.tgz#2110e0af0900fd37467b5907ecd13a7884a1b559" integrity sha512-p/nXbhSEcu3pZRdkW1OfJhpsVtW1gd4Wa1fnQc9YLiTfAjn0312eMKimbdIQzuZl9aa9xUGaRlP9T/CJE/ditQ== +jsonrepair@^3.7.0: + version "3.7.0" + resolved "https://registry.yarnpkg.com/jsonrepair/-/jsonrepair-3.7.0.tgz#b4fddb9c8d29dd62263f4f037334099e28feac21" + integrity sha512-TwE50n4P4gdVfMQF2q+X+IGy4ntFfcuHHE8zjRyBcdtrRK0ORZsjOZD6zmdylk4p277nQBAlHgsEPWtMIQk4LQ== + jsonwebtoken@^8.5.1: version "8.5.1" resolved "https://registry.yarnpkg.com/jsonwebtoken/-/jsonwebtoken-8.5.1.tgz#00e71e0b8df54c2121a1f26137df2280673bcc0d" From 0976266b428889e03eff9af83619a287180563b3 Mon Sep 17 00:00:00 2001 From: timothycarambat Date: Wed, 1 May 2024 13:03:12 -0700 Subject: [PATCH 2/5] add comments --- server/utils/agents/aibitat/plugins/chat-history.js | 1 - server/utils/agents/aibitat/providers/untooled.js | 2 ++ 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/server/utils/agents/aibitat/plugins/chat-history.js b/server/utils/agents/aibitat/plugins/chat-history.js index 4d3f8fd1efe..e3123a83bea 100644 --- a/server/utils/agents/aibitat/plugins/chat-history.js +++ b/server/utils/agents/aibitat/plugins/chat-history.js @@ -13,7 +13,6 @@ const chatHistory = { name: this.name, setup: function (aibitat) { aibitat.onMessage(async () => { - return; try { const lastResponses = aibitat.chats.slice(-2); if (lastResponses.length !== 2) return; diff --git a/server/utils/agents/aibitat/providers/untooled.js b/server/utils/agents/aibitat/providers/untooled.js index 91a515182e4..acf71d28317 100644 --- a/server/utils/agents/aibitat/providers/untooled.js +++ b/server/utils/agents/aibitat/providers/untooled.js @@ -1,6 +1,8 @@ const { safeJsonParse } = require("../../../http"); const { Deduplicator } = require("../utils/dedupe"); +// Useful inheritance class for a model which supports OpenAi schema for API requests +// but does not have tool-calling or JSON output support. class UnTooled { constructor() { this.deduplicator = new Deduplicator(); From 7238f1b07f5ca1ace3aca155ca0eac0298257857 Mon Sep 17 00:00:00 2001 From: timothycarambat Date: Thu, 2 May 2024 17:13:59 -0700 Subject: [PATCH 3/5] enable few-shot prompting per function for OSS models --- .../AgentConfig/AgentLLMSelection/index.jsx | 16 +++- server/utils/agents/aibitat/index.js | 12 +++ server/utils/agents/aibitat/plugins/memory.js | 32 ++++++- .../aibitat/plugins/save-file-browser.js | 26 +++++ .../utils/agents/aibitat/plugins/summarize.js | 20 ++++ .../agents/aibitat/plugins/web-browsing.js | 16 +++- .../agents/aibitat/plugins/web-scraping.js | 15 ++- .../providers/{ => helpers}/classes.js | 0 .../providers/{ => helpers}/untooled.js | 94 +++++-------------- .../agents/aibitat/providers/lmstudio.js | 5 +- 10 files changed, 159 insertions(+), 77 deletions(-) rename server/utils/agents/aibitat/providers/{ => helpers}/classes.js (100%) rename server/utils/agents/aibitat/providers/{ => helpers}/untooled.js (66%) diff --git a/frontend/src/pages/WorkspaceSettings/AgentConfig/AgentLLMSelection/index.jsx b/frontend/src/pages/WorkspaceSettings/AgentConfig/AgentLLMSelection/index.jsx index 51b384642d6..408d60a026f 100644 --- a/frontend/src/pages/WorkspaceSettings/AgentConfig/AgentLLMSelection/index.jsx +++ b/frontend/src/pages/WorkspaceSettings/AgentConfig/AgentLLMSelection/index.jsx @@ -2,10 +2,11 @@ import React, { useEffect, useRef, useState } from "react"; import AnythingLLMIcon from "@/media/logo/anything-llm-icon.png"; import AgentLLMItem from "./AgentLLMItem"; import { AVAILABLE_LLM_PROVIDERS } from "@/pages/GeneralSettings/LLMPreference"; -import { CaretUpDown, MagnifyingGlass, X } from "@phosphor-icons/react"; +import { CaretUpDown, Gauge, MagnifyingGlass, X } from "@phosphor-icons/react"; import AgentModelSelection from "../AgentModelSelection"; const ENABLED_PROVIDERS = ["openai", "anthropic", "lmstudio"]; +const WARN_PERFORMANCE = ["lmstudio"]; const LLM_DEFAULT = { name: "Please make a selection", @@ -62,6 +63,19 @@ export default function AgentLLMSelection({ const selectedLLMObject = LLMS.find((llm) => llm.value === selectedLLM); return (
+ {WARN_PERFORMANCE.includes(selectedLLM) && ( +
+
+ +

+ Performance of LLMs that do not explicitly support tool-calling is + highly dependent on the model's capabilities and accuracy. Some + abilities may be limited or non-functional. +

+
+
+ )} +