From 6b3dd37fad8d4a20f5e46489df0b5aaea14d5d4c Mon Sep 17 00:00:00 2001 From: shatfield4 Date: Wed, 30 Oct 2024 16:45:01 -0700 Subject: [PATCH 1/2] allow 127.0.0.1 as valid url for scraping --- collector/extensions/index.js | 7 ++++++- collector/utils/url/index.js | 4 ++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/collector/extensions/index.js b/collector/extensions/index.js index 47989d5d5c2..d55eb6b069f 100644 --- a/collector/extensions/index.js +++ b/collector/extensions/index.js @@ -118,7 +118,12 @@ function extensions(app) { try { const websiteDepth = require("../utils/extensions/WebsiteDepth"); const { url, depth = 1, maxLinks = 20 } = reqBody(request); - if (!validURL(url)) return { success: false, reason: "Not a valid URL." }; + if (!validURL(url)) { + return response.status(400).json({ + success: false, + reason: "Not a valid URL." + }); + } const scrapedData = await websiteDepth(url, depth, maxLinks); response.status(200).json({ success: true, data: scrapedData }); diff --git a/collector/utils/url/index.js b/collector/utils/url/index.js index 8a58dbd7aaf..7b4976d5854 100644 --- a/collector/utils/url/index.js +++ b/collector/utils/url/index.js @@ -23,6 +23,10 @@ function isInvalidIp({ hostname }) { // If fails to validate to number - abort and return as invalid. if (isNaN(Number(octetOne))) return true; + + // Allow 127.0.0.1 (localhost) but block other private IP ranges + if (octetOne === "127" && hostname === "127.0.0.1") return false; + return INVALID_OCTETS.includes(Number(octetOne)); } From 4079b5d1261641208db73fe3a8ae2f73a749e994 Mon Sep 17 00:00:00 2001 From: timothycarambat Date: Thu, 31 Oct 2024 09:57:04 -0700 Subject: [PATCH 2/2] update comments and lint --- collector/extensions/index.js | 8 +------- collector/utils/url/index.js | 16 +++++++++++++--- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/collector/extensions/index.js b/collector/extensions/index.js index d55eb6b069f..81a3a3dd796 100644 --- a/collector/extensions/index.js +++ b/collector/extensions/index.js @@ -118,13 +118,7 @@ function extensions(app) { try { const websiteDepth = require("../utils/extensions/WebsiteDepth"); const { url, depth = 1, maxLinks = 20 } = reqBody(request); - if (!validURL(url)) { - return response.status(400).json({ - success: false, - reason: "Not a valid URL." - }); - } - + if (!validURL(url)) throw new Error("Not a valid URL."); const scrapedData = await websiteDepth(url, depth, maxLinks); response.status(200).json({ success: true, data: scrapedData }); } catch (e) { diff --git a/collector/utils/url/index.js b/collector/utils/url/index.js index 7b4976d5854..c9d87b295fe 100644 --- a/collector/utils/url/index.js +++ b/collector/utils/url/index.js @@ -1,7 +1,7 @@ /** ATTN: SECURITY RESEARCHERS * To Security researchers about to submit an SSRF report CVE - please don't. * We are aware that the code below is does not defend against any of the thousands of ways - * you can map a hostname to another IP. The code below does not have intention of blocking this + * you can map a hostname to another IP via tunneling, hosts editing, etc. The code below does not have intention of blocking this * and is simply to prevent the user from accidentally putting in non-valid websites, which is all this protects * since _all urls must be submitted by the user anyway_ and cannot be done with authentication and manager or admin roles. * If an attacker has those roles then the system is already vulnerable and this is not a primary concern. @@ -14,18 +14,28 @@ const VALID_PROTOCOLS = ["https:", "http:"]; const INVALID_OCTETS = [192, 172, 10, 127]; +/** + * If an ip address is passed in the user is attempting to collector some internal service running on internal/private IP. + * This is not a security feature and simply just prevents the user from accidentally entering invalid IP addresses. + * @param {URL} param0 + * @param {URL['hostname']} param0.hostname + * @returns {boolean} + */ function isInvalidIp({ hostname }) { const IPRegex = new RegExp( /^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])$/gi ); + + // Not an IP address at all - passthrough if (!IPRegex.test(hostname)) return false; const [octetOne, ..._rest] = hostname.split("."); // If fails to validate to number - abort and return as invalid. if (isNaN(Number(octetOne))) return true; - // Allow 127.0.0.1 (localhost) but block other private IP ranges - if (octetOne === "127" && hostname === "127.0.0.1") return false; + // Allow localhost loopback and 0.0.0.0 for scraping convenience + // for locally hosted services or websites + if (["127.0.0.1", "0.0.0.0"].includes(hostname)) return false; return INVALID_OCTETS.includes(Number(octetOne)); }