diff --git a/server/__tests__/utils/agents/aibitat/providers/helpers/untooled.test.js b/server/__tests__/utils/agents/aibitat/providers/helpers/untooled.test.js new file mode 100644 index 00000000000..a3364f902d3 --- /dev/null +++ b/server/__tests__/utils/agents/aibitat/providers/helpers/untooled.test.js @@ -0,0 +1,87 @@ +const UnTooled = require("../../../../../../utils/agents/aibitat/providers/helpers/untooled"); + +describe("UnTooled: validFuncCall", () => { + const untooled = new UnTooled(); + const validFunc = { + "name": "brave-search-brave_web_search", + "description": "Example function", + "parameters": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "query": { + "type": "string", + "description": "Search query (max 400 chars, 50 words)" + }, + "count": { + "type": "number", + "description": "Number of results (1-20, default 10)", + "default": 10 + }, + "offset": { + "type": "number", + "description": "Pagination offset (max 9, default 0)", + "default": 0 + } + }, + "required": [ + "query" + ] + } + }; + + it("Be truthy if the function call is valid and has all required arguments", () => { + const result = untooled.validFuncCall( + { + name: validFunc.name, + arguments: { query: "test" }, + }, [validFunc]); + expect(result.valid).toBe(true); + expect(result.reason).toBe(null); + }); + + it("Be falsey if the function call has no name or arguments", () => { + const result = untooled.validFuncCall( + { arguments: {} }, [validFunc]); + expect(result.valid).toBe(false); + expect(result.reason).toBe("Missing name or arguments in function call."); + + const result2 = untooled.validFuncCall( + { name: validFunc.name }, [validFunc]); + expect(result2.valid).toBe(false); + expect(result2.reason).toBe("Missing name or arguments in function call."); + }); + + it("Be falsey if the function call references an unknown function definition", () => { + const result = untooled.validFuncCall( + { + name: "unknown-function", + arguments: {}, + }, [validFunc]); + expect(result.valid).toBe(false); + expect(result.reason).toBe("Function name does not exist."); + }); + + it("Be falsey if the function call is valid but missing any required arguments", () => { + const result = untooled.validFuncCall( + { + name: validFunc.name, + arguments: {}, + }, [validFunc]); + expect(result.valid).toBe(false); + expect(result.reason).toBe("Missing required argument: query"); + }); + + it("Be falsey if the function call is valid but has an unknown argument defined (required or not)", () => { + const result = untooled.validFuncCall( + { + name: validFunc.name, + arguments: { + query: "test", + unknown: "unknown", + }, + }, [validFunc]); + expect(result.valid).toBe(false); + expect(result.reason).toBe("Unknown argument: unknown provided but not in schema."); + }); +}); \ No newline at end of file diff --git a/server/utils/agents/aibitat/providers/helpers/untooled.js b/server/utils/agents/aibitat/providers/helpers/untooled.js index 774bb691560..aff7fa70353 100644 --- a/server/utils/agents/aibitat/providers/helpers/untooled.js +++ b/server/utils/agents/aibitat/providers/helpers/untooled.js @@ -45,40 +45,11 @@ ${JSON.stringify(def.parameters.properties, null, 4)}\n`; } /** - * Check if two arrays of strings or numbers have the same values - * @param {string[]|number[]} arr1 - * @param {string[]|number[]} arr2 - * @param {Object} [opts] - * @param {boolean} [opts.enforceOrder] - By default (false), the order of the values in the arrays doesn't matter. - * @return {boolean} + * Validate a function call against a list of functions. + * @param {{name: string, arguments: Object}} functionCall - The function call to validate. + * @param {Object[]} functions - The list of functions definitions to validate against. + * @return {{valid: boolean, reason: string|null}} - The validation result. */ - compareArrays(arr1, arr2, opts) { - function vKey(i, v) { - return (opts?.enforceOrder ? `${i}-` : "") + `${typeof v}-${v}`; - } - - if (arr1.length !== arr2.length) return false; - - const d1 = {}; - const d2 = {}; - for (let i = arr1.length - 1; i >= 0; i--) { - d1[vKey(i, arr1[i])] = true; - d2[vKey(i, arr2[i])] = true; - } - - for (let i = arr1.length - 1; i >= 0; i--) { - const v = vKey(i, arr1[i]); - if (d1[v] !== d2[v]) return false; - } - - for (let i = arr2.length - 1; i >= 0; i--) { - const v = vKey(i, arr2[i]); - if (d1[v] !== d2[v]) return false; - } - - return true; - } - validFuncCall(functionCall = {}, functions = []) { if ( !functionCall || @@ -92,14 +63,31 @@ ${JSON.stringify(def.parameters.properties, null, 4)}\n`; } const foundFunc = functions.find((def) => def.name === functionCall.name); - if (!foundFunc) { + if (!foundFunc) return { valid: false, reason: "Function name does not exist." }; + + const schemaProps = Object.keys(foundFunc?.parameters?.properties || {}); + const requiredProps = foundFunc?.parameters?.required || []; + const providedProps = Object.keys(functionCall.arguments); + + for (const requiredProp of requiredProps) { + if (!providedProps.includes(requiredProp)) { + return { + valid: false, + reason: `Missing required argument: ${requiredProp}`, + }; + } } - const props = Object.keys(foundFunc.parameters.properties); - const fProps = Object.keys(functionCall.arguments); - if (!this.compareArrays(props, fProps)) { - return { valid: false, reason: "Invalid argument schema match." }; + // Ensure all provided arguments are valid for the schema + // This is to prevent the model from hallucinating or providing invalid additional arguments. + for (const providedProp of providedProps) { + if (!schemaProps.includes(providedProp)) { + return { + valid: false, + reason: `Unknown argument: ${providedProp} provided but not in schema.`, + }; + } } return { valid: true, reason: null };