diff --git a/frontend/src/pages/Admin/AgentBuilder/index.jsx b/frontend/src/pages/Admin/AgentBuilder/index.jsx
index f658931dad..930a575a7f 100644
--- a/frontend/src/pages/Admin/AgentBuilder/index.jsx
+++ b/frontend/src/pages/Admin/AgentBuilder/index.jsx
@@ -1,5 +1,6 @@
import React, { useState, useEffect, useRef } from "react";
import { useNavigate, useParams } from "react-router-dom";
+import { Tooltip } from "react-tooltip";
import BlockList, { BLOCK_TYPES, BLOCK_INFO } from "./BlockList";
import AddBlockMenu from "./AddBlockMenu";
@@ -288,18 +289,6 @@ export default function AgentBuilder() {
});
};
- // const runFlow = async (uuid) => {
- // try {
- // const { success, error, _results } = await AgentFlows.runFlow(uuid);
- // if (!success) throw new Error(error);
-
- // showToast("Flow executed successfully!", "success", { clear: true });
- // } catch (error) {
- // console.error(error);
- // showToast("Failed to run agent flow", "error", { clear: true });
- // }
- // };
-
const clearFlow = () => {
if (!!flowId) navigate(paths.agents.builder());
setAgentName("");
@@ -356,6 +345,21 @@ export default function AgentBuilder() {
+
+
+ When enabled, long webpage content will be automatically summarized to
+ reduce token usage.
+
+
+ Note: This may affect data quality and remove specific details from
+ the original content.
+
+
);
}
diff --git a/frontend/src/pages/Admin/AgentBuilder/nodes/WebScrapingNode/index.jsx b/frontend/src/pages/Admin/AgentBuilder/nodes/WebScrapingNode/index.jsx
index 76655af74c..f1e48b24de 100644
--- a/frontend/src/pages/Admin/AgentBuilder/nodes/WebScrapingNode/index.jsx
+++ b/frontend/src/pages/Admin/AgentBuilder/nodes/WebScrapingNode/index.jsx
@@ -1,3 +1,4 @@
+import { Info } from "@phosphor-icons/react";
import React from "react";
export default function WebScrapingNode({
@@ -31,7 +32,9 @@ export default function WebScrapingNode({
onConfigChange({ captureAs: e.target.value })}
+ onChange={(e) =>
+ onConfigChange({ ...config, captureAs: e.target.value })
+ }
className="w-full border-none bg-theme-settings-input-bg text-theme-text-primary text-sm rounded-lg focus:outline-primary-button active:outline-primary-button outline-none p-2.5"
>
{[
@@ -60,13 +63,44 @@ export default function WebScrapingNode({
onConfigChange({ querySelector: e.target.value })}
+ onChange={(e) =>
+ onConfigChange({ ...config, querySelector: e.target.value })
+ }
placeholder=".article-content, #content, .main-content, etc."
className="w-full border-none bg-theme-settings-input-bg text-theme-text-primary text-sm rounded-lg focus:outline-primary-button active:outline-primary-button outline-none p-2.5"
/>
)}
+
+
+
+ Content Summarization
+
+
+
+
+
+
+ onConfigChange({
+ ...config,
+ enableSummarization: e.target.checked,
+ })
+ }
+ className="sr-only peer"
+ aria-label="Toggle content summarization"
+ />
+
+
+
+
Result Variable
diff --git a/server/utils/agentFlows/executors/web-scraping.js b/server/utils/agentFlows/executors/web-scraping.js
index 3e9d0462af..e54b95fe6f 100644
--- a/server/utils/agentFlows/executors/web-scraping.js
+++ b/server/utils/agentFlows/executors/web-scraping.js
@@ -10,7 +10,7 @@ const { summarizeContent } = require("../../agents/aibitat/utils/summarize");
* @returns {Promise} Scraped content
*/
async function executeWebScraping(config, context) {
- const { url, captureAs = "text" } = config;
+ const { url, captureAs = "text", enableSummarization = true } = config;
const { introspect, logger, aibitat } = context;
logger(
`\x1b[43m[AgentFlowToolExecutor]\x1b[0m - executing Web Scraping block`
@@ -40,6 +40,11 @@ async function executeWebScraping(config, context) {
throw new Error("There was no content to be collected or read.");
}
+ if (!enableSummarization) {
+ logger(`Returning raw content as summarization is disabled`);
+ return content;
+ }
+
const tokenCount = new TokenManager(
aibitat.defaultProvider.model
).countFromString(content);
@@ -47,7 +52,13 @@ async function executeWebScraping(config, context) {
aibitat.defaultProvider.provider,
aibitat.defaultProvider.model
);
- if (tokenCount < contextLimit) return content;
+
+ if (tokenCount < contextLimit) {
+ logger(
+ `Content within token limit (${tokenCount}/${contextLimit}). Returning raw content.`
+ );
+ return content;
+ }
introspect(
`This page's content is way too long (${tokenCount} tokens). I will summarize it right now.`