θΏ™ζ˜―indexlocζδΎ›ηš„ζœεŠ‘οΌŒδΈθ¦θΎ“ε…₯任何密码
Skip to content

Allow 127.0.0.1 as valid URL for scraping #2560

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Oct 31, 2024
Merged

Allow 127.0.0.1 as valid URL for scraping #2560

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
6b3dd37
allow 127.0.0.1 as valid url for scraping
shatfield4 Oct 30, 2024
4079b5d
update comments and lint
timothycarambat Oct 31, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 1 addition & 2 deletions collector/extensions/index.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -118,8 +118,7 @@ function extensions(app) {
try {
const websiteDepth = require("../utils/extensions/WebsiteDepth");
const { url, depth = 1, maxLinks = 20 } = reqBody(request);
if (!validURL(url)) return { success: false, reason: "Not a valid URL." };

if (!validURL(url)) throw new Error("Not a valid URL.");
const scrapedData = await websiteDepth(url, depth, maxLinks);
response.status(200).json({ success: true, data: scrapedData });
} catch (e) {
Expand Down
16 changes: 15 additions & 1 deletion collector/utils/url/index.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
/** ATTN: SECURITY RESEARCHERS
* To Security researchers about to submit an SSRF report CVE - please don't.
* We are aware that the code below is does not defend against any of the thousands of ways
* you can map a hostname to another IP. The code below does not have intention of blocking this
* you can map a hostname to another IP via tunneling, hosts editing, etc. The code below does not have intention of blocking this
* and is simply to prevent the user from accidentally putting in non-valid websites, which is all this protects
* since _all urls must be submitted by the user anyway_ and cannot be done with authentication and manager or admin roles.
* If an attacker has those roles then the system is already vulnerable and this is not a primary concern.
Expand All @@ -14,15 +14,29 @@
const VALID_PROTOCOLS = ["https:", "http:"];
const INVALID_OCTETS = [192, 172, 10, 127];

/**
* If an ip address is passed in the user is attempting to collector some internal service running on internal/private IP.
* This is not a security feature and simply just prevents the user from accidentally entering invalid IP addresses.
* @param {URL} param0
* @param {URL['hostname']} param0.hostname
* @returns {boolean}
*/
function isInvalidIp({ hostname }) {
const IPRegex = new RegExp(
/^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])$/gi
);

// Not an IP address at all - passthrough
if (!IPRegex.test(hostname)) return false;
const [octetOne, ..._rest] = hostname.split(".");

// If fails to validate to number - abort and return as invalid.
if (isNaN(Number(octetOne))) return true;

// Allow localhost loopback and 0.0.0.0 for scraping convenience
// for locally hosted services or websites
if (["127.0.0.1", "0.0.0.0"].includes(hostname)) return false;

return INVALID_OCTETS.includes(Number(octetOne));
}

Expand Down