From 538847602ee4ecca0670ab3631327c5bc51e4d96 Mon Sep 17 00:00:00 2001 From: shatfield4 Date: Thu, 21 Nov 2024 17:24:55 -0800 Subject: [PATCH 1/3] fix apipie streaming/sort by chat models --- server/utils/AiProviders/apipie/index.js | 54 ++++++++++++------------ server/utils/helpers/customModels.js | 21 ++++++--- 2 files changed, 42 insertions(+), 33 deletions(-) diff --git a/server/utils/AiProviders/apipie/index.js b/server/utils/AiProviders/apipie/index.js index acfd2b1e6a0..0e09e64067e 100644 --- a/server/utils/AiProviders/apipie/index.js +++ b/server/utils/AiProviders/apipie/index.js @@ -1,8 +1,4 @@ const { NativeEmbedder } = require("../../EmbeddingEngines/native"); -const { - handleDefaultStreamResponseV2, -} = require("../../helpers/chat/responses"); - const { v4: uuidv4 } = require("uuid"); const { writeResponseChunk, @@ -98,6 +94,18 @@ class ApiPieLLM { ); } + chatModels() { + const allModels = this.models(); + return Object.entries(allModels).reduce((chatModels, [modelId, modelInfo]) => { + // Filter for chat models + if (modelInfo.subtype && + (modelInfo.subtype.includes('chat') || modelInfo.subtype.includes('chatx'))) { + chatModels[modelId] = modelInfo; + } + return chatModels; + }, {}); + } + streamingEnabled() { return "streamGetChatCompletion" in this; } @@ -114,13 +122,13 @@ class ApiPieLLM { } promptWindowLimit() { - const availableModels = this.models(); + const availableModels = this.chatModels(); return availableModels[this.model]?.maxLength || 4096; } async isValidChatCompletionModel(model = "") { await this.#syncModels(); - const availableModels = this.models(); + const availableModels = this.chatModels(); return availableModels.hasOwnProperty(model); } @@ -188,23 +196,20 @@ class ApiPieLLM { return null; return result.choices[0].message.content; } + async streamGetChatCompletion(messages = null, { temperature = 0.7 }) { + if (!(await this.isValidChatCompletionModel(this.model))) + throw new Error( + `ApiPie chat: ${this.model} is not valid for chat completion!` + ); - // APIPie says it supports streaming, but it does not work across all models and providers. - // Notably, it is not working for OpenRouter models at all. - // async streamGetChatCompletion(messages = null, { temperature = 0.7 }) { - // if (!(await this.isValidChatCompletionModel(this.model))) - // throw new Error( - // `ApiPie chat: ${this.model} is not valid for chat completion!` - // ); - - // const streamRequest = await this.openai.chat.completions.create({ - // model: this.model, - // stream: true, - // messages, - // temperature, - // }); - // return streamRequest; - // } + const streamRequest = await this.openai.chat.completions.create({ + model: this.model, + stream: true, + messages, + temperature, + }); + return streamRequest; + } handleStream(response, stream, responseProps) { const { uuid = uuidv4(), sources = [] } = responseProps; @@ -264,10 +269,6 @@ class ApiPieLLM { }); } - // handleStream(response, stream, responseProps) { - // return handleDefaultStreamResponseV2(response, stream, responseProps); - // } - // Simple wrapper for dynamic embedder & normalize interface for all LLM implementations async embedTextInput(textInput) { return await this.embedder.embedTextInput(textInput); @@ -300,6 +301,7 @@ async function fetchApiPieModels(providedApiKey = null) { id: `${model.provider}/${model.model}`, name: `${model.provider}/${model.model}`, organization: model.provider, + subtype: model.subtype, maxLength: model.max_tokens, }; }); diff --git a/server/utils/helpers/customModels.js b/server/utils/helpers/customModels.js index 72882d6d1c1..9ca8d8ef149 100644 --- a/server/utils/helpers/customModels.js +++ b/server/utils/helpers/customModels.js @@ -388,13 +388,20 @@ async function getAPIPieModels(apiKey = null) { if (!Object.keys(knownModels).length === 0) return { models: [], error: null }; - const models = Object.values(knownModels).map((model) => { - return { - id: model.id, - organization: model.organization, - name: model.name, - }; - }); + const models = Object.values(knownModels) + .filter(model => { + // Filter for chat models + return model.subtype && + (model.subtype.includes('chat') || model.subtype.includes('chatx')); + }) + .map((model) => { + return { + id: model.id, + organization: model.organization, + name: model.name, + }; + }); + console.log(models.length); return { models, error: null }; } From f2db25c2d71da7cdbab260c8fdb9837cbb0427f8 Mon Sep 17 00:00:00 2001 From: shatfield4 Date: Thu, 21 Nov 2024 17:28:42 -0800 Subject: [PATCH 2/3] lint --- .../MistralAiOptions/index.jsx | 4 +--- frontend/src/locales/resources.js | 5 ++--- frontend/src/locales/vn/common.js | 3 +-- server/utils/AiProviders/apipie/index.js | 22 ++++++++++++------- .../utils/EmbeddingEngines/mistral/index.js | 2 +- server/utils/helpers/customModels.js | 8 ++++--- 6 files changed, 24 insertions(+), 20 deletions(-) diff --git a/frontend/src/components/EmbeddingSelection/MistralAiOptions/index.jsx b/frontend/src/components/EmbeddingSelection/MistralAiOptions/index.jsx index 6012b319228..77cdd167617 100644 --- a/frontend/src/components/EmbeddingSelection/MistralAiOptions/index.jsx +++ b/frontend/src/components/EmbeddingSelection/MistralAiOptions/index.jsx @@ -28,9 +28,7 @@ export default function MistralAiOptions({ settings }) { className="bg-theme-settings-input-bg border-gray-500 text-white text-sm rounded-lg block w-full p-2.5" > - {[ - "mistral-embed", - ].map((model) => { + {["mistral-embed"].map((model) => { return (