From f18942368313a760d45e4d74167c390072049d03 Mon Sep 17 00:00:00 2001 From: nandini-56 Date: Tue, 15 Jul 2025 20:25:34 +0530 Subject: [PATCH 1/4] made a new source jsmon for finding the subdomains --- v2/pkg/passive/sources.go | 4 +- v2/pkg/runner/options.go | 2 +- v2/pkg/subscraping/sources/jsmon/jsmon.go | 149 ++++++++++++++++++++++ 3 files changed, 152 insertions(+), 3 deletions(-) create mode 100644 v2/pkg/subscraping/sources/jsmon/jsmon.go diff --git a/v2/pkg/passive/sources.go b/v2/pkg/passive/sources.go index 20013042f..01d676464 100644 --- a/v2/pkg/passive/sources.go +++ b/v2/pkg/passive/sources.go @@ -34,8 +34,8 @@ import ( "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/hudsonrock" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/hunter" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/intelx" + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/jsmon" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/leakix" - "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/netlas" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/pugrecon" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/quake" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/rapiddns" @@ -66,6 +66,7 @@ var AllSources = [...]subscraping.Source{ &chinaz.Source{}, &commoncrawl.Source{}, &crtsh.Source{}, + &jsmon.Source{}, &digitorus.Source{}, &dnsdb.Source{}, &dnsdumpster.Source{}, @@ -76,7 +77,6 @@ var AllSources = [...]subscraping.Source{ &hackertarget.Source{}, &hunter.Source{}, &intelx.Source{}, - &netlas.Source{}, &leakix.Source{}, &quake.Source{}, &pugrecon.Source{}, diff --git a/v2/pkg/runner/options.go b/v2/pkg/runner/options.go index 7f39e0315..a46cc20f0 100644 --- a/v2/pkg/runner/options.go +++ b/v2/pkg/runner/options.go @@ -255,7 +255,7 @@ var defaultRateLimits = []string{ "whoisxmlapi=50/s", "securitytrails=2/s", "sitedossier=8/m", - "netlas=1/s", + // "netlas=1/s", // removed to fix validation error // "gitlab=2/s", "github=83/m", "hudsonrock=5/s", diff --git a/v2/pkg/subscraping/sources/jsmon/jsmon.go b/v2/pkg/subscraping/sources/jsmon/jsmon.go new file mode 100644 index 000000000..6bac93817 --- /dev/null +++ b/v2/pkg/subscraping/sources/jsmon/jsmon.go @@ -0,0 +1,149 @@ +// Package jsmon logic +package jsmon + +import ( + "bytes" + "context" + "fmt" + "io" + "time" + + jsoniter "github.com/json-iterator/go" + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" +) + +type subdomainsResponse struct { + Subdomains []string `json:"subdomains"` + Status string `json:"status"` + Message string `json:"message"` +} + +type Source struct { + apiKeys []string + timeTaken time.Duration + errors int + results int + skipped bool +} + +func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { + results := make(chan subscraping.Result) + s.errors = 0 + s.results = 0 + + go func() { + defer func(startTime time.Time) { + s.timeTaken = time.Since(startTime) + close(results) + }(time.Now()) + + if len(s.apiKeys) == 0 { + s.skipped = true + return + } + + // API keys structure: [baseUrl, authToken, workspaceId] + var baseUrl, authToken, wkspId string + if len(s.apiKeys) >= 1 { + baseUrl = s.apiKeys[0] + } + if len(s.apiKeys) >= 2 { + authToken = s.apiKeys[1] + } + if len(s.apiKeys) >= 3 { + wkspId = s.apiKeys[2] + } + + // fmt.Printf("[DEBUG] API Keys parsed - baseUrl: %s, authToken: %s, wkspId: %s\n", baseUrl, authToken, wkspId) + // fmt.Printf("[DEBUG] Total API keys provided: %d\n", len(s.apiKeys)) + + // Use the direct subfinderScan endpoint + subfinderScanURL := fmt.Sprintf("%s/api/v2/subfinderScan?wkspId=%s", baseUrl, wkspId) + + // Prepare the request body with domain + requestBody := fmt.Sprintf(`{"domain":"%s"}`, domain) + + // Prepare headers with Authorization + headers := map[string]string{ + "X-Jsmon-Key": authToken, + "Content-Type": "application/json", + } + + resp, err := session.Post(ctx, subfinderScanURL, "", headers, bytes.NewReader([]byte(requestBody))) + if err != nil { + // fmt.Printf("[DEBUG] Request error: %v\n", err) + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + session.DiscardHTTPResponse(resp) + return + } + + // fmt.Printf("[DEBUG] Response status: %d\n", resp.StatusCode) + + if resp.StatusCode != 200 { + // Read response body for error details + body, _ := io.ReadAll(resp.Body) + // fmt.Printf("[DEBUG] Error response body: %s\n", string(body)) + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("subfinderScan API returned status %d: %s", resp.StatusCode, string(body))} + s.errors++ + session.DiscardHTTPResponse(resp) + return + } + + // Parse the response as a direct array of subdomains + var subdomains []string + err = jsoniter.NewDecoder(resp.Body).Decode(&subdomains) + if err != nil { + // Read response body for debugging + body, _ := io.ReadAll(resp.Body) + fmt.Printf("[DEBUG] Response body: %s\n", string(body)) + fmt.Printf("[DEBUG] JSON decode error: %v\n", err) + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + session.DiscardHTTPResponse(resp) + return + } + + fmt.Printf("[INF] Found %d subdomains\n", len(subdomains)) + + session.DiscardHTTPResponse(resp) + + // Process subdomains + for _, subdomain := range subdomains { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} + s.results++ + } + + }() + + return results +} + +func (s *Source) Name() string { + return "jsmon" +} + +func (s *Source) IsDefault() bool { + return false +} + +func (s *Source) HasRecursiveSupport() bool { + return true +} + +func (s *Source) NeedsKey() bool { + return true +} + +func (s *Source) AddApiKeys(keys []string) { + s.apiKeys = keys +} + +func (s *Source) Statistics() subscraping.Statistics { + return subscraping.Statistics{ + Errors: s.errors, + Results: s.results, + TimeTaken: s.timeTaken, + Skipped: s.skipped, + } +} From f6dc834728c665c9720e8b0a61db6c0b18f51058 Mon Sep 17 00:00:00 2001 From: Nandini Kumari <125210634+nandini-56@users.noreply.github.com> Date: Wed, 16 Jul 2025 12:57:34 +0000 Subject: [PATCH 2/4] added netlas back --- v2/pkg/passive/sources.go | 2 ++ v2/pkg/runner/options.go | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/v2/pkg/passive/sources.go b/v2/pkg/passive/sources.go index 01d676464..0fe6287dc 100644 --- a/v2/pkg/passive/sources.go +++ b/v2/pkg/passive/sources.go @@ -36,6 +36,7 @@ import ( "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/intelx" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/jsmon" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/leakix" + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/netlas" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/pugrecon" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/quake" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/rapiddns" @@ -77,6 +78,7 @@ var AllSources = [...]subscraping.Source{ &hackertarget.Source{}, &hunter.Source{}, &intelx.Source{}, + &netlas.Source{}, &leakix.Source{}, &quake.Source{}, &pugrecon.Source{}, diff --git a/v2/pkg/runner/options.go b/v2/pkg/runner/options.go index a46cc20f0..83cc9c079 100644 --- a/v2/pkg/runner/options.go +++ b/v2/pkg/runner/options.go @@ -255,7 +255,7 @@ var defaultRateLimits = []string{ "whoisxmlapi=50/s", "securitytrails=2/s", "sitedossier=8/m", - // "netlas=1/s", // removed to fix validation error + "netlas=1/s", // "gitlab=2/s", "github=83/m", "hudsonrock=5/s", From 156ab5cc0c06cda80b25e19ada82cad77eaccd1d Mon Sep 17 00:00:00 2001 From: nandini-56 Date: Thu, 7 Aug 2025 14:51:30 +0530 Subject: [PATCH 3/4] added the source as jsmon in subfinder --- v2/pkg/subscraping/sources/jsmon/jsmon.go | 59 ++++++++++++----------- 1 file changed, 30 insertions(+), 29 deletions(-) diff --git a/v2/pkg/subscraping/sources/jsmon/jsmon.go b/v2/pkg/subscraping/sources/jsmon/jsmon.go index 6bac93817..c7ddfc221 100644 --- a/v2/pkg/subscraping/sources/jsmon/jsmon.go +++ b/v2/pkg/subscraping/sources/jsmon/jsmon.go @@ -6,6 +6,7 @@ import ( "context" "fmt" "io" + "strings" "time" jsoniter "github.com/json-iterator/go" @@ -42,28 +43,36 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se return } - // API keys structure: [baseUrl, authToken, workspaceId] + // Parse API key in format: baseUrl:apiKey:wkspId var baseUrl, authToken, wkspId string - if len(s.apiKeys) >= 1 { - baseUrl = s.apiKeys[0] - } - if len(s.apiKeys) >= 2 { - authToken = s.apiKeys[1] - } - if len(s.apiKeys) >= 3 { - wkspId = s.apiKeys[2] - } + if len(s.apiKeys) > 0 { + apiKeyString := s.apiKeys[0] + + // Find the last two colons to split properly + lastColonIndex := strings.LastIndex(apiKeyString, ":") + if lastColonIndex == -1 { + fmt.Printf("[DEBUG] No colons found in API key\n") + s.skipped = true + return + } + + secondLastColonIndex := strings.LastIndex(apiKeyString[:lastColonIndex], ":") + if secondLastColonIndex == -1 { + fmt.Printf("[DEBUG] Only one colon found in API key\n") + s.skipped = true + return + } + + baseUrl = apiKeyString[:secondLastColonIndex] + authToken = apiKeyString[secondLastColonIndex+1 : lastColonIndex] + wkspId = apiKeyString[lastColonIndex+1:] - // fmt.Printf("[DEBUG] API Keys parsed - baseUrl: %s, authToken: %s, wkspId: %s\n", baseUrl, authToken, wkspId) - // fmt.Printf("[DEBUG] Total API keys provided: %d\n", len(s.apiKeys)) + } - // Use the direct subfinderScan endpoint - subfinderScanURL := fmt.Sprintf("%s/api/v2/subfinderScan?wkspId=%s", baseUrl, wkspId) + subfinderScanURL := fmt.Sprintf("%s/api/v2/subfinderScan2?wkspId=%s", baseUrl, wkspId) - // Prepare the request body with domain requestBody := fmt.Sprintf(`{"domain":"%s"}`, domain) - // Prepare headers with Authorization headers := map[string]string{ "X-Jsmon-Key": authToken, "Content-Type": "application/json", @@ -90,26 +99,18 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se return } - // Parse the response as a direct array of subdomains - var subdomains []string - err = jsoniter.NewDecoder(resp.Body).Decode(&subdomains) + var response subdomainsResponse + err = jsoniter.NewDecoder(resp.Body).Decode(&response) if err != nil { - // Read response body for debugging - body, _ := io.ReadAll(resp.Body) - fmt.Printf("[DEBUG] Response body: %s\n", string(body)) - fmt.Printf("[DEBUG] JSON decode error: %v\n", err) results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ session.DiscardHTTPResponse(resp) return } - fmt.Printf("[INF] Found %d subdomains\n", len(subdomains)) - session.DiscardHTTPResponse(resp) - // Process subdomains - for _, subdomain := range subdomains { + for _, subdomain := range response.Subdomains { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} s.results++ } @@ -124,11 +125,11 @@ func (s *Source) Name() string { } func (s *Source) IsDefault() bool { - return false + return true } func (s *Source) HasRecursiveSupport() bool { - return true + return false } func (s *Source) NeedsKey() bool { From 9ac65163b10835dc5429944cb2ba5c2fca01aca3 Mon Sep 17 00:00:00 2001 From: nandini-56 Date: Mon, 18 Aug 2025 13:09:26 +0530 Subject: [PATCH 4/4] changed the baseUrl from variable to a constants --- v2/pkg/subscraping/sources/jsmon/jsmon.go | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/v2/pkg/subscraping/sources/jsmon/jsmon.go b/v2/pkg/subscraping/sources/jsmon/jsmon.go index c7ddfc221..813605f90 100644 --- a/v2/pkg/subscraping/sources/jsmon/jsmon.go +++ b/v2/pkg/subscraping/sources/jsmon/jsmon.go @@ -13,6 +13,10 @@ import ( "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" ) +const ( + baseUrl = "https://api.jsmon.sh" +) + type subdomainsResponse struct { Subdomains []string `json:"subdomains"` Status string `json:"status"` @@ -43,8 +47,8 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se return } - // Parse API key in format: baseUrl:apiKey:wkspId - var baseUrl, authToken, wkspId string + // Parse API key in format: apiKey:wkspId + var authToken, wkspId string if len(s.apiKeys) > 0 { apiKeyString := s.apiKeys[0] @@ -55,16 +59,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se s.skipped = true return } - - secondLastColonIndex := strings.LastIndex(apiKeyString[:lastColonIndex], ":") - if secondLastColonIndex == -1 { - fmt.Printf("[DEBUG] Only one colon found in API key\n") - s.skipped = true - return - } - - baseUrl = apiKeyString[:secondLastColonIndex] - authToken = apiKeyString[secondLastColonIndex+1 : lastColonIndex] + authToken = apiKeyString[:lastColonIndex] wkspId = apiKeyString[lastColonIndex+1:] }