θΏ™ζ˜―indexlocζδΎ›ηš„ζœεŠ‘οΌŒδΈθ¦θΎ“ε…₯任何密码
Skip to content

API request delay for Generic OpenAI embedding engine #4317

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
Sep 18, 2025
Merged

API request delay for Generic OpenAI embedding engine #4317

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
f45ee96
Add ENV to configure api request delay for generic open ai embedding …
chaserhkj Aug 20, 2025
d19048c
yarn lint formatting
chaserhkj Aug 20, 2025
9039843
refactor
timothycarambat Sep 18, 2025
b7d888b
Merge branch 'master' into generic-openai-embedding-delay
timothycarambat Sep 18, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions docker/.env.example
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -195,6 +195,7 @@ GID='1000'
# EMBEDDING_BASE_PATH='http://127.0.0.1:4000'
# GENERIC_OPEN_AI_EMBEDDING_API_KEY='sk-123abc'
# GENERIC_OPEN_AI_EMBEDDING_MAX_CONCURRENT_CHUNKS=500
# GENERIC_OPEN_AI_EMBEDDING_API_DELAY_MS=1000

# EMBEDDING_ENGINE='gemini'
# GEMINI_EMBEDDING_API_KEY=
Expand Down
1 change: 1 addition & 0 deletions server/.env.example
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -194,6 +194,7 @@ SIG_SALT='salt' # Please generate random string at least 32 chars long.
# EMBEDDING_BASE_PATH='http://127.0.0.1:4000'
# GENERIC_OPEN_AI_EMBEDDING_API_KEY='sk-123abc'
# GENERIC_OPEN_AI_EMBEDDING_MAX_CONCURRENT_CHUNKS=500
# GENERIC_OPEN_AI_EMBEDDING_API_DELAY_MS=1000

# EMBEDDING_ENGINE='gemini'
# GEMINI_EMBEDDING_API_KEY=
Expand Down
105 changes: 55 additions & 50 deletions server/utils/EmbeddingEngines/genericOpenAi/index.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,35 @@ class GenericOpenAiEmbedder {
console.log(`\x1b[36m[GenericOpenAiEmbedder]\x1b[0m ${text}`, ...args);
}

/**
* returns the `GENERIC_OPEN_AI_EMBEDDING_API_DELAY_MS` env variable as a number or null if the env variable is not set or is not a number.
* The minimum delay is 500ms.
*
* For some implementation this is necessary to avoid 429 errors due to rate limiting or
* hardware limitations where a single-threaded process is not able to handle the requests fast enough.
* @returns {number}
*/
get apiRequestDelay() {
if (!("GENERIC_OPEN_AI_EMBEDDING_API_DELAY_MS" in process.env)) return null;
if (isNaN(Number(process.env.GENERIC_OPEN_AI_EMBEDDING_API_DELAY_MS)))
return null;
const delayTimeout = Number(
process.env.GENERIC_OPEN_AI_EMBEDDING_API_DELAY_MS
);
if (delayTimeout < 500) return 500; // minimum delay of 500ms
return delayTimeout;
}

/**
* runs the delay if it is set and valid.
* @returns {Promise<void>}
*/
async runDelay() {
if (!this.apiRequestDelay) return;
this.log(`Delaying new batch request for ${this.apiRequestDelay}ms`);
await new Promise((resolve) => setTimeout(resolve, this.apiRequestDelay));
}

/**
* returns the `GENERIC_OPEN_AI_EMBEDDING_MAX_CONCURRENT_CHUNKS` env variable as a number
* or 500 if the env variable is not set or is not a number.
Expand All @@ -52,62 +81,38 @@ class GenericOpenAiEmbedder {

async embedChunks(textChunks = []) {
// Because there is a hard POST limit on how many chunks can be sent at once to OpenAI (~8mb)
// we concurrently execute each max batch of text chunks possible.
// we sequentially execute each max batch of text chunks possible.
// Refer to constructor maxConcurrentChunks for more info.
const embeddingRequests = [];
const allResults = [];
for (const chunk of toChunks(textChunks, this.maxConcurrentChunks)) {
embeddingRequests.push(
new Promise((resolve) => {
this.openai.embeddings
.create({
model: this.model,
input: chunk,
})
.then((result) => {
resolve({ data: result?.data, error: null });
})
.catch((e) => {
e.type =
e?.response?.data?.error?.code ||
e?.response?.status ||
"failed_to_embed";
e.message = e?.response?.data?.error?.message || e.message;
resolve({ data: [], error: e });
});
})
);
}
const { data = [], error = null } = await new Promise((resolve) => {
this.openai.embeddings
.create({
model: this.model,
input: chunk,
})
.then((result) => resolve({ data: result?.data, error: null }))
.catch((e) => {
e.type =
e?.response?.data?.error?.code ||
e?.response?.status ||
"failed_to_embed";
e.message = e?.response?.data?.error?.message || e.message;
resolve({ data: [], error: e });
});
});

const { data = [], error = null } = await Promise.all(
embeddingRequests
).then((results) => {
// If any errors were returned from OpenAI abort the entire sequence because the embeddings
// will be incomplete.
const errors = results
.filter((res) => !!res.error)
.map((res) => res.error)
.flat();
if (errors.length > 0) {
let uniqueErrors = new Set();
errors.map((error) =>
uniqueErrors.add(`[${error.type}]: ${error.message}`)
);

return {
data: [],
error: Array.from(uniqueErrors).join(", "),
};
}
return {
data: results.map((res) => res?.data || []).flat(),
error: null,
};
});
if (error)
throw new Error(`GenericOpenAI Failed to embed: ${error.message}`);
allResults.push(...(data || []));
if (this.apiRequestDelay) await this.runDelay();
}

if (!!error) throw new Error(`GenericOpenAI Failed to embed: ${error}`);
return data.length > 0 &&
data.every((embd) => embd.hasOwnProperty("embedding"))
? data.map((embd) => embd.embedding)
return allResults.length > 0 &&
allResults.every((embd) => embd.hasOwnProperty("embedding"))
? allResults.map((embd) => embd.embedding)
: null;
}
}
Expand Down