θΏ™ζ˜―indexlocζδΎ›ηš„ζœεŠ‘οΌŒδΈθ¦θΎ“ε…₯任何密码
Skip to content

[FEAT] Implement auto detect URL for KoboldCPP #1860

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
Jul 15, 2024
Merged

[FEAT] Implement auto detect URL for KoboldCPP #1860

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
fa0052f
implement auto detect url for koboldcpp
shatfield4 Jul 12, 2024
e7ec9d9
remove unneeded import
shatfield4 Jul 12, 2024
0d6e59f
border-none inputs and buttons
timothycarambat Jul 15, 2024
91a0fe5
Merge branch 'master' into 1844-feat-auto-detect-url-for-koboldcpp
timothycarambat Jul 15, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
179 changes: 132 additions & 47 deletions frontend/src/components/LLMSelection/KoboldCPPOptions/index.jsx
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,47 +1,116 @@
import { useState, useEffect } from "react";
import { useEffect, useState } from "react";
import System from "@/models/system";
import PreLoader from "@/components/Preloader";
import { KOBOLDCPP_COMMON_URLS } from "@/utils/constants";
import { CaretDown, CaretUp } from "@phosphor-icons/react";
import useProviderEndpointAutoDiscovery from "@/hooks/useProviderEndpointAutoDiscovery";

export default function KoboldCPPOptions({ settings }) {
const [basePathValue, setBasePathValue] = useState(
settings?.KoboldCPPBasePath
const {
autoDetecting: loading,
basePath,
basePathValue,
showAdvancedControls,
setShowAdvancedControls,
handleAutoDetectClick,
} = useProviderEndpointAutoDiscovery({
provider: "koboldcpp",
initialBasePath: settings?.KoboldCPPBasePath,
ENDPOINTS: KOBOLDCPP_COMMON_URLS,
});

const [tokenLimit, setTokenLimit] = useState(
settings?.KoboldCPPTokenLimit || 4096
);
const [basePath, setBasePath] = useState(settings?.KoboldCPPBasePath);

const handleTokenLimitChange = (e) => {
setTokenLimit(Number(e.target.value));
};

return (
<div className="flex gap-[36px] mt-1.5 flex-wrap">
<div className="flex flex-col w-60">
<label className="text-white text-sm font-semibold block mb-3">
Base URL
</label>
<input
type="url"
name="KoboldCPPBasePath"
className="bg-zinc-900 text-white placeholder:text-white/20 text-sm rounded-lg focus:outline-primary-button active:outline-primary-button outline-none block w-full p-2.5"
placeholder="http://127.0.0.1:5000/v1"
defaultValue={settings?.KoboldCPPBasePath}
required={true}
autoComplete="off"
spellCheck={false}
onChange={(e) => setBasePathValue(e.target.value)}
onBlur={() => setBasePath(basePathValue)}
<div className="w-full flex flex-col gap-y-7">
<div className="w-full flex items-start gap-[36px] mt-1.5">
<KoboldCPPModelSelection
settings={settings}
basePath={basePath.value}
/>
<div className="flex flex-col w-60">
<label className="text-white text-sm font-semibold block mb-2">
Token context window
</label>
<input
type="number"
name="KoboldCPPTokenLimit"
className="border-none bg-zinc-900 text-white placeholder:text-white/20 text-sm rounded-lg focus:outline-primary-button active:outline-primary-button outline-none block w-full p-2.5"
placeholder="4096"
min={1}
value={tokenLimit}
onChange={handleTokenLimitChange}
onScroll={(e) => e.target.blur()}
required={true}
autoComplete="off"
/>
<p className="text-xs leading-[18px] font-base text-white text-opacity-60 mt-2">
Maximum number of tokens for context and response.
</p>
</div>
</div>
<KoboldCPPModelSelection settings={settings} basePath={basePath} />
<div className="flex flex-col w-60">
<label className="text-white text-sm font-semibold block mb-3">
Token context window
</label>
<input
type="number"
name="KoboldCPPTokenLimit"
className="bg-zinc-900 text-white placeholder:text-white/20 text-sm rounded-lg focus:outline-primary-button active:outline-primary-button outline-none block w-full p-2.5"
placeholder="4096"
min={1}
onScroll={(e) => e.target.blur()}
defaultValue={settings?.KoboldCPPTokenLimit}
required={true}
autoComplete="off"
/>
<div className="flex justify-start mt-4">
<button
onClick={(e) => {
e.preventDefault();
setShowAdvancedControls(!showAdvancedControls);
}}
className="border-none text-white hover:text-white/70 flex items-center text-sm"
>
{showAdvancedControls ? "Hide" : "Show"} Manual Endpoint Input
{showAdvancedControls ? (
<CaretUp size={14} className="ml-1" />
) : (
<CaretDown size={14} className="ml-1" />
)}
</button>
</div>

<div hidden={!showAdvancedControls}>
<div className="w-full flex items-start gap-4">
<div className="flex flex-col w-60">
<div className="flex justify-between items-center mb-2">
<label className="text-white text-sm font-semibold">
KoboldCPP Base URL
</label>
{loading ? (
<PreLoader size="6" />
) : (
<>
{!basePathValue.value && (
<button
onClick={handleAutoDetectClick}
className="border-none bg-primary-button text-xs font-medium px-2 py-1 rounded-lg hover:bg-secondary hover:text-white shadow-[0_4px_14px_rgba(0,0,0,0.25)]"
>
Auto-Detect
</button>
)}
</>
)}
</div>
<input
type="url"
name="KoboldCPPBasePath"
className="border-none bg-zinc-900 text-white placeholder:text-white/20 text-sm rounded-lg focus:outline-primary-button active:outline-primary-button outline-none block w-full p-2.5"
placeholder="http://127.0.0.1:5000/v1"
value={basePathValue.value}
required={true}
autoComplete="off"
spellCheck={false}
onChange={basePath.onChange}
onBlur={basePath.onBlur}
/>
<p className="text-xs leading-[18px] font-base text-white text-opacity-60 mt-2">
Enter the URL where KoboldCPP is running.
</p>
</div>
</div>
</div>
</div>
);
Expand All @@ -59,8 +128,17 @@ function KoboldCPPModelSelection({ settings, basePath = null }) {
return;
}
setLoading(true);
const { models } = await System.customModels("koboldcpp", null, basePath);
setCustomModels(models || []);
try {
const { models } = await System.customModels(
"koboldcpp",
null,
basePath
);
setCustomModels(models || []);
} catch (error) {
console.error("Failed to fetch custom models:", error);
setCustomModels([]);
}
setLoading(false);
}
findCustomModels();
Expand All @@ -69,44 +147,51 @@ function KoboldCPPModelSelection({ settings, basePath = null }) {
if (loading || customModels.length === 0) {
return (
<div className="flex flex-col w-60">
<label className="text-white text-sm font-semibold block mb-3">
Chat Model Selection
<label className="text-white text-sm font-semibold block mb-2">
KoboldCPP Model
</label>
<select
name="KoboldCPPModelPref"
disabled={true}
className="bg-zinc-900 border-gray-500 text-white text-sm rounded-lg block w-full p-2.5"
className="border-none bg-zinc-900 border-gray-500 text-white text-sm rounded-lg block w-full p-2.5"
>
<option disabled={true} selected={true}>
{basePath?.includes("/v1")
? "-- loading available models --"
: "-- waiting for URL --"}
? "--loading available models--"
: "Enter KoboldCPP URL first"}
</option>
</select>
<p className="text-xs leading-[18px] font-base text-white text-opacity-60 mt-2">
Select the KoboldCPP model you want to use. Models will load after
entering a valid KoboldCPP URL.
</p>
</div>
);
}

return (
<div className="flex flex-col w-60">
<label className="text-white text-sm font-semibold block mb-3">
Chat Model Selection
<label className="text-white text-sm font-semibold block mb-2">
KoboldCPP Model
</label>
<select
name="KoboldCPPModelPref"
required={true}
className="bg-zinc-900 border-gray-500 text-white text-sm rounded-lg block w-full p-2.5"
className="border-none bg-zinc-900 border-gray-500 text-white text-sm rounded-lg block w-full p-2.5"
>
{customModels.map((model) => (
<option
key={model.id}
value={model.id}
selected={settings?.KoboldCPPModelPref === model.id}
selected={settings.KoboldCPPModelPref === model.id}
>
{model.id}
</option>
))}
</select>
<p className="text-xs leading-[18px] font-base text-white text-opacity-60 mt-2">
Choose the KoboldCPP model you want to use for your conversations.
</p>
</div>
);
}
7 changes: 7 additions & 0 deletions frontend/src/utils/constants.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,13 @@ export const LMSTUDIO_COMMON_URLS = [
"http://172.17.0.1:1234/v1",
];

export const KOBOLDCPP_COMMON_URLS = [
"http://127.0.0.1:5000/v1",
"http://localhost:5000/v1",
"http://host.docker.internal:5000/v1",
"http://172.17.0.1:5000/v1",
];

export function fullApiUrl() {
if (API_BASE !== "/api") return API_BASE;
return `${window.location.origin}/api`;
Expand Down