diff --git a/v2/pkg/passive/sources.go b/v2/pkg/passive/sources.go index 7de378976..f6ebc2341 100644 --- a/v2/pkg/passive/sources.go +++ b/v2/pkg/passive/sources.go @@ -20,6 +20,7 @@ import ( "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/chinaz" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/commoncrawl" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/crtsh" + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/digitalyama" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/digitorus" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/dnsdb" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/dnsdumpster" @@ -34,6 +35,7 @@ import ( "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/intelx" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/leakix" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/netlas" + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/pugrecon" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/quake" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/rapiddns" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/redhuntlabs" @@ -47,7 +49,6 @@ import ( "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/waybackarchive" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/whoisxmlapi" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/zoomeyeapi" - "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/digitalyama" mapsutil "github.com/projectdiscovery/utils/maps" ) @@ -77,6 +78,7 @@ var AllSources = [...]subscraping.Source{ &netlas.Source{}, &leakix.Source{}, &quake.Source{}, + &pugrecon.Source{}, &rapiddns.Source{}, &redhuntlabs.Source{}, // &riddler.Source{}, // failing due to cloudfront protection diff --git a/v2/pkg/passive/sources_test.go b/v2/pkg/passive/sources_test.go index 848cc32e7..6b2875c3c 100644 --- a/v2/pkg/passive/sources_test.go +++ b/v2/pkg/passive/sources_test.go @@ -34,6 +34,7 @@ var ( "intelx", "netlas", "quake", + "pugrecon", "rapiddns", "redhuntlabs", // "riddler", // failing due to cloudfront protection diff --git a/v2/pkg/runner/options.go b/v2/pkg/runner/options.go index 5e3d4b172..7f39e0315 100644 --- a/v2/pkg/runner/options.go +++ b/v2/pkg/runner/options.go @@ -244,6 +244,7 @@ func (options *Options) preProcessDomains() { var defaultRateLimits = []string{ "github=30/m", "fullhunt=60/m", + "pugrecon=10/s", fmt.Sprintf("robtex=%d/ms", uint(math.MaxUint)), "securitytrails=1/s", "shodan=1/s", diff --git a/v2/pkg/subscraping/sources/pugrecon/pugrecon.go b/v2/pkg/subscraping/sources/pugrecon/pugrecon.go new file mode 100644 index 000000000..dca7e186d --- /dev/null +++ b/v2/pkg/subscraping/sources/pugrecon/pugrecon.go @@ -0,0 +1,151 @@ +// Package pugrecon logic +package pugrecon + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "net/http" + "time" + + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" +) + +// pugreconResult stores a single result from the pugrecon API +type pugreconResult struct { + Name string `json:"name"` +} + +// pugreconAPIResponse stores the response from the pugrecon API +type pugreconAPIResponse struct { + Results []pugreconResult `json:"results"` + QuotaRemaining int `json:"quota_remaining"` + Limited bool `json:"limited"` + TotalResults int `json:"total_results"` + Message string `json:"message"` +} + +// Source is the passive scraping agent +type Source struct { + apiKeys []string + timeTaken time.Duration + errors int + results int + skipped bool +} + +// Run function returns all subdomains found with the service +func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { + results := make(chan subscraping.Result) + s.errors = 0 + s.results = 0 + + go func() { + defer func(startTime time.Time) { + s.timeTaken = time.Since(startTime) + close(results) + }(time.Now()) + + randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) + if randomApiKey == "" { + s.skipped = true + return + } + + // Prepare POST request data + postData := map[string]string{"domain_name": domain} + bodyBytes, err := json.Marshal(postData) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("failed to marshal request body: %w", err)} + s.errors++ + return + } + bodyReader := bytes.NewReader(bodyBytes) + + // Prepare headers + headers := map[string]string{ + "Authorization": "Bearer " + randomApiKey, + "Content-Type": "application/json", + "Accept": "application/json", + } + + apiURL := "https://pugrecon.com/api/v1/domains" + resp, err := session.HTTPRequest(ctx, http.MethodPost, apiURL, "", headers, bodyReader, subscraping.BasicAuth{}) // Use HTTPRequest for full header control + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + session.DiscardHTTPResponse(resp) + return + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + errorMsg := fmt.Sprintf("received status code %d", resp.StatusCode) + // Attempt to read error message from body if possible + var apiResp pugreconAPIResponse + if json.NewDecoder(resp.Body).Decode(&apiResp) == nil && apiResp.Message != "" { + errorMsg = fmt.Sprintf("%s: %s", errorMsg, apiResp.Message) + } + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf(errorMsg)} + s.errors++ + return + } + + var response pugreconAPIResponse + err = json.NewDecoder(resp.Body).Decode(&response) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + return + } + + if response.Message != "" && !response.Limited { // Handle potential non-error messages, except rate limit info + // Log or handle message if needed, but don't treat as hard error unless necessary + } + + for _, subdomain := range response.Results { + results <- subscraping.Result{ + Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain.Name, + } + s.results++ + } + }() + + return results +} + +// Name returns the name of the source +func (s *Source) Name() string { + return "pugrecon" +} + +// IsDefault returns false as this is not a default source. +func (s *Source) IsDefault() bool { + return false +} + +// HasRecursiveSupport returns false as this source does not support recursive searches. +func (s *Source) HasRecursiveSupport() bool { + return false +} + +// NeedsKey returns true as this source requires an API key. +func (s *Source) NeedsKey() bool { + return true +} + +// AddApiKeys adds the API keys for the source. +func (s *Source) AddApiKeys(keys []string) { + s.apiKeys = keys +} + +// Statistics returns the statistics for the source. +func (s *Source) Statistics() subscraping.Statistics { + return subscraping.Statistics{ + Errors: s.errors, + Results: s.results, + TimeTaken: s.timeTaken, + Skipped: s.skipped, + } +}