From 8773308b854d054f94aa2aff3726a32d55e7bd13 Mon Sep 17 00:00:00 2001 From: Sunghyun Jeon Date: Sat, 25 Jan 2025 04:44:46 +0900 Subject: [PATCH 1/4] add digitalyama source to passive scraping --- v2/pkg/passive/sources.go | 2 + v2/pkg/passive/sources_test.go | 2 + .../sources/digitalyama/digitalyama.go | 128 ++++++++++++++++++ 3 files changed, 132 insertions(+) create mode 100644 v2/pkg/subscraping/sources/digitalyama/digitalyama.go diff --git a/v2/pkg/passive/sources.go b/v2/pkg/passive/sources.go index b8ce15472..5bc1b7e62 100644 --- a/v2/pkg/passive/sources.go +++ b/v2/pkg/passive/sources.go @@ -47,6 +47,7 @@ import ( "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/waybackarchive" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/whoisxmlapi" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/zoomeyeapi" + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/digitalyama" mapsutil "github.com/projectdiscovery/utils/maps" ) @@ -94,6 +95,7 @@ var AllSources = [...]subscraping.Source{ // &reconcloud.Source{}, // failing due to cloudflare bot protection &builtwith.Source{}, &hudsonrock.Source{}, + &digitalyama.Source{}, } var sourceWarnings = mapsutil.NewSyncLockMap[string, string]( diff --git a/v2/pkg/passive/sources_test.go b/v2/pkg/passive/sources_test.go index 3223117e3..d6389acbd 100644 --- a/v2/pkg/passive/sources_test.go +++ b/v2/pkg/passive/sources_test.go @@ -54,6 +54,7 @@ var ( // "reconcloud", "builtwith", "hudsonrock", + "digitalyama" } expectedDefaultSources = []string{ @@ -89,6 +90,7 @@ var ( // "threatminer", // "reconcloud", "builtwith", + "digitalyama" } expectedDefaultRecursiveSources = []string{ diff --git a/v2/pkg/subscraping/sources/digitalyama/digitalyama.go b/v2/pkg/subscraping/sources/digitalyama/digitalyama.go new file mode 100644 index 000000000..aa708ec25 --- /dev/null +++ b/v2/pkg/subscraping/sources/digitalyama/digitalyama.go @@ -0,0 +1,128 @@ +package digitalyama + +import ( + "context" + "fmt" + "time" + + jsoniter "github.com/json-iterator/go" + + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" +) + +// Source is the passive scraping agent +type Source struct { + apiKeys []string + timeTaken time.Duration + errors int + results int + skipped bool +} + +type digitalYamaResponse struct { + Query string `json:"query"` + Count int `json:"count"` + Subdomains []string `json:"subdomains"` + UsageSummary struct { + QueryCost float64 `json:"query_cost"` + CreditsRemaining float64 `json:"credits_remaining"` + } `json:"usage_summary"` +} + +// Run function returns all subdomains found with the service +func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { + results := make(chan subscraping.Result) + s.errors = 0 + s.results = 0 + + go func() { + defer func(startTime time.Time) { + s.timeTaken = time.Since(startTime) + close(results) + }(time.Now()) + + randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) + if randomApiKey == "" { + s.skipped = true + return + } + + searchURL := fmt.Sprintf("https://api.digitalyama.com/subdomain_finder?domain=%s", domain) + resp, err := session.Get(ctx, searchURL, "", map[string]string{"x-api-key": randomApiKey}) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + return + } + defer resp.Body.Close() + + if resp.StatusCode != 200 { + var errResponse struct { + Detail []struct { + Loc []string `json:"loc"` + Msg string `json:"msg"` + Type string `json:"type"` + } `json:"detail"` + } + err = jsoniter.NewDecoder(resp.Body).Decode(&errResponse) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("unexpected status code %d", resp.StatusCode)} + s.errors++ + return + } + if len(errResponse.Detail) > 0 { + errMsg := errResponse.Detail[0].Msg + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("%s (code %d)", errMsg, resp.StatusCode)} + } else { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("unexpected status code %d", resp.StatusCode)} + } + s.errors++ + return + } + + var response digitalYamaResponse + err = jsoniter.NewDecoder(resp.Body).Decode(&response) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + return + } + + for _, subdomain := range response.Subdomains { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} + s.results++ + } + }() + + return results +} + +// Name returns the name of the source +func (s *Source) Name() string { + return "digitalyama" +} + +func (s *Source) IsDefault() bool { + return true +} + +func (s *Source) HasRecursiveSupport() bool { + return false +} + +func (s *Source) NeedsKey() bool { + return true +} + +func (s *Source) AddApiKeys(keys []string) { + s.apiKeys = keys +} + +func (s *Source) Statistics() subscraping.Statistics { + return subscraping.Statistics{ + Errors: s.errors, + Results: s.results, + TimeTaken: s.timeTaken, + Skipped: s.skipped, + } +} \ No newline at end of file From 71b884d2760e5de29e497d3a9fca0f4692ccd4d7 Mon Sep 17 00:00:00 2001 From: Sunghyun Jeon Date: Sat, 25 Jan 2025 22:56:17 +0900 Subject: [PATCH 2/4] fix: correct syntax for digitalyama in sources_test.go --- v2/pkg/passive/sources_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/v2/pkg/passive/sources_test.go b/v2/pkg/passive/sources_test.go index d6389acbd..a648b76c7 100644 --- a/v2/pkg/passive/sources_test.go +++ b/v2/pkg/passive/sources_test.go @@ -54,7 +54,7 @@ var ( // "reconcloud", "builtwith", "hudsonrock", - "digitalyama" + "digitalyama", } expectedDefaultSources = []string{ @@ -90,7 +90,7 @@ var ( // "threatminer", // "reconcloud", "builtwith", - "digitalyama" + "digitalyama", } expectedDefaultRecursiveSources = []string{ From ad8eb0ce7581405119ab57edc843213bef8dc546 Mon Sep 17 00:00:00 2001 From: Sunghyun Jeon Date: Sun, 26 Jan 2025 07:54:40 +0900 Subject: [PATCH 3/4] fix: add missing newline at end of digitalyama.go --- v2/pkg/subscraping/sources/digitalyama/digitalyama.go | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/v2/pkg/subscraping/sources/digitalyama/digitalyama.go b/v2/pkg/subscraping/sources/digitalyama/digitalyama.go index aa708ec25..60027c71f 100644 --- a/v2/pkg/subscraping/sources/digitalyama/digitalyama.go +++ b/v2/pkg/subscraping/sources/digitalyama/digitalyama.go @@ -5,6 +5,7 @@ import ( "fmt" "time" + // jsoniter jsoniter "github.com/json-iterator/go" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" @@ -20,11 +21,11 @@ type Source struct { } type digitalYamaResponse struct { - Query string `json:"query"` - Count int `json:"count"` - Subdomains []string `json:"subdomains"` + Query string `json:"query"` + Count int `json:"count"` + Subdomains []string `json:"subdomains"` UsageSummary struct { - QueryCost float64 `json:"query_cost"` + QueryCost float64 `json:"query_cost"` CreditsRemaining float64 `json:"credits_remaining"` } `json:"usage_summary"` } @@ -125,4 +126,4 @@ func (s *Source) Statistics() subscraping.Statistics { TimeTaken: s.timeTaken, Skipped: s.skipped, } -} \ No newline at end of file +} From a83d28a95883c0d7d1cd9d90ff2f037e6eda1773 Mon Sep 17 00:00:00 2001 From: Sunghyun Jeon Date: Sun, 26 Jan 2025 07:56:35 +0900 Subject: [PATCH 4/4] test for workflow that RapidApi error in buildTest --- v2/pkg/subscraping/sources/digitalyama/digitalyama.go | 1 - 1 file changed, 1 deletion(-) diff --git a/v2/pkg/subscraping/sources/digitalyama/digitalyama.go b/v2/pkg/subscraping/sources/digitalyama/digitalyama.go index 60027c71f..eaa6f8420 100644 --- a/v2/pkg/subscraping/sources/digitalyama/digitalyama.go +++ b/v2/pkg/subscraping/sources/digitalyama/digitalyama.go @@ -5,7 +5,6 @@ import ( "fmt" "time" - // jsoniter jsoniter "github.com/json-iterator/go" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping"