From 2164a450a78610c7173e246a44493cf0b1a31a2d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fan=20Can=20Bak=C4=B1r?= Date: Tue, 4 Jul 2023 12:04:52 +0300 Subject: [PATCH 01/19] add rate limit per source option --- README.md | 1 + v2/go.mod | 2 +- v2/go.sum | 2 + v2/pkg/passive/passive.go | 75 ++++++++++++++++++++++++-- v2/pkg/passive/sources_w_auth_test.go | 19 +++++-- v2/pkg/passive/sources_wo_auth_test.go | 18 ++++++- v2/pkg/runner/enumerate.go | 2 +- v2/pkg/runner/options.go | 6 ++- v2/pkg/subscraping/agent.go | 13 ++--- v2/pkg/subscraping/types.go | 2 +- 10 files changed, 117 insertions(+), 23 deletions(-) diff --git a/README.md b/README.md index 50d010c39..0b0445546 100644 --- a/README.md +++ b/README.md @@ -74,6 +74,7 @@ FILTER: RATE-LIMIT: -rl, -rate-limit int maximum number of http requests to send per second + -rls value maximum number of http requests to send per second four providers in key=value format (-rls hackertarget=10) -t int number of concurrent goroutines for resolving (-active only) (default 10) UPDATE: diff --git a/v2/go.mod b/v2/go.mod index fb17ba61a..64452ff3c 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -11,7 +11,7 @@ require ( github.com/projectdiscovery/dnsx v1.1.4 github.com/projectdiscovery/fdmax v0.0.4 github.com/projectdiscovery/gologger v1.1.10 - github.com/projectdiscovery/ratelimit v0.0.8 + github.com/projectdiscovery/ratelimit v0.0.9 github.com/projectdiscovery/utils v0.0.39 github.com/rs/xid v1.5.0 github.com/stretchr/testify v1.8.4 diff --git a/v2/go.sum b/v2/go.sum index 739895cc3..6309da0bd 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -126,6 +126,8 @@ github.com/projectdiscovery/gologger v1.1.10 h1:XNRdtzLTdxiFGuK9gutoL752mykzXDoi github.com/projectdiscovery/gologger v1.1.10/go.mod h1:VqANHK7qcEq3i6/vV5HNWwdyv2aFPSrlaVDU4Ogrc6U= github.com/projectdiscovery/ratelimit v0.0.8 h1:K6S/DCr48xNxTXHRmU82wl1mj7j0VrXnAKr8sKTacHI= github.com/projectdiscovery/ratelimit v0.0.8/go.mod h1:JJAtj8Rd5DNqN5FgwyMHWIi4BHivOw1+8gDrpsBf8Ic= +github.com/projectdiscovery/ratelimit v0.0.9 h1:28t2xDHUnyss1irzqPG3Oxz5hkRjl+3Q2I/aes7nau8= +github.com/projectdiscovery/ratelimit v0.0.9/go.mod h1:f98UxLsHt0dWrHTbRDxos4+RvOLE0UFpyECfrfKBz1I= github.com/projectdiscovery/retryabledns v1.0.23 h1:8X6HECevl6n7K7kxqfHLTcsCVXDdz9HR+1hS6fto7sw= github.com/projectdiscovery/retryabledns v1.0.23/go.mod h1:tj8BEe0jWCB4m3aAhHq4S2dqfpod0h+BiCZmXkXAGac= github.com/projectdiscovery/retryablehttp-go v1.0.15 h1:kP9x9f++QimRwb8ABqnI1dhEymvnZXS2Wp2Zs4rWk/c= diff --git a/v2/pkg/passive/passive.go b/v2/pkg/passive/passive.go index 826228146..a62cb68eb 100644 --- a/v2/pkg/passive/passive.go +++ b/v2/pkg/passive/passive.go @@ -3,25 +3,37 @@ package passive import ( "context" "fmt" + "math" "sort" + "strconv" + "strings" "sync" "time" + "github.com/projectdiscovery/ratelimit" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" ) // EnumerateSubdomains wraps EnumerateSubdomainsWithCtx with an empty context -func (a *Agent) EnumerateSubdomains(domain string, proxy string, rateLimit, timeout int, maxEnumTime time.Duration) chan subscraping.Result { - return a.EnumerateSubdomainsWithCtx(context.Background(), domain, proxy, rateLimit, timeout, maxEnumTime) +func (a *Agent) EnumerateSubdomains(domain string, proxy string, rateLimit int, rateLimits map[string]interface{}, timeout int, maxEnumTime time.Duration) chan subscraping.Result { + return a.EnumerateSubdomainsWithCtx(context.Background(), domain, proxy, rateLimit, rateLimits, timeout, maxEnumTime) } // EnumerateSubdomainsWithCtx enumerates all the subdomains for a given domain -func (a *Agent) EnumerateSubdomainsWithCtx(ctx context.Context, domain string, proxy string, rateLimit, timeout int, maxEnumTime time.Duration) chan subscraping.Result { +func (a *Agent) EnumerateSubdomainsWithCtx(ctx context.Context, domain string, proxy string, rateLimit int, rateLimits map[string]interface{}, timeout int, maxEnumTime time.Duration) chan subscraping.Result { results := make(chan subscraping.Result) + go func() { defer close(results) - session, err := subscraping.NewSession(domain, proxy, rateLimit, timeout) + multiRateLimiter, err := buildMultiRateLimiter(ctx, a, rateLimit, rateLimits) + if err != nil { + results <- subscraping.Result{ + Type: subscraping.Error, Error: fmt.Errorf("could not init multi rate limiter for %s: %s", domain, err), + } + return + } + session, err := subscraping.NewSession(domain, proxy, multiRateLimiter, timeout) if err != nil { results <- subscraping.Result{ Type: subscraping.Error, Error: fmt.Errorf("could not init passive session for %s: %s", domain, err), @@ -38,7 +50,9 @@ func (a *Agent) EnumerateSubdomainsWithCtx(ctx context.Context, domain string, p wg.Add(1) go func(source subscraping.Source) { - for resp := range source.Run(ctx, domain, session) { + //lint:ignore SA1029 reason + ctxWithValue := context.WithValue(ctx, "source", source.Name()) + for resp := range source.Run(ctxWithValue, domain, session) { results <- resp } wg.Done() @@ -50,6 +64,57 @@ func (a *Agent) EnumerateSubdomainsWithCtx(ctx context.Context, domain string, p return results } +func buildMultiRateLimiter(ctx context.Context, a *Agent, rateLimit int, rateLimits map[string]interface{}) (*ratelimit.MultiLimiter, error) { + var multiRateLimiter *ratelimit.MultiLimiter + var err error + for _, source := range a.sources { + var rl uint + if sourceRateLimit, ok := rateLimits[strings.ToLower(source.Name())]; ok { + rl = sourceRateLimitOrGlobal(rateLimit, sourceRateLimit) + } + + if rl > 0 { + multiRateLimiter, err = addRateLimiter(ctx, multiRateLimiter, source.Name(), rl, time.Second) + } else { + multiRateLimiter, err = addRateLimiter(ctx, multiRateLimiter, source.Name(), math.MaxUint32, time.Millisecond) + } + + if err != nil { + break + } + } + return multiRateLimiter, err +} + +func sourceRateLimitOrGlobal(globalRateLimit int, sourceRateLimit interface{}) uint { + if sourceRateLimitStr, ok := sourceRateLimit.(string); ok { + sourceRateLimitUint, err := strconv.ParseUint(sourceRateLimitStr, 10, 64) + if err == nil { + return uint(sourceRateLimitUint) + } + } + return uint(globalRateLimit) +} + +func addRateLimiter(ctx context.Context, multiRateLimiter *ratelimit.MultiLimiter, key string, maxCount uint, duration time.Duration) (*ratelimit.MultiLimiter, error) { + if multiRateLimiter == nil { + mrl, err := ratelimit.NewMultiLimiter(ctx, &ratelimit.Options{ + Key: key, + IsUnlimited: maxCount == math.MaxUint32, + MaxCount: maxCount, + Duration: duration, + }) + return mrl, err + } + err := multiRateLimiter.Add(&ratelimit.Options{ + Key: key, + IsUnlimited: maxCount == math.MaxUint32, + MaxCount: maxCount, + Duration: duration, + }) + return multiRateLimiter, err +} + func (a *Agent) GetStatistics() map[string]subscraping.Statistics { stats := make(map[string]subscraping.Statistics) sort.Slice(a.sources, func(i, j int) bool { diff --git a/v2/pkg/passive/sources_w_auth_test.go b/v2/pkg/passive/sources_w_auth_test.go index 452bde6f5..ca2a0cc09 100644 --- a/v2/pkg/passive/sources_w_auth_test.go +++ b/v2/pkg/passive/sources_w_auth_test.go @@ -3,15 +3,18 @@ package passive import ( "context" "fmt" + "math" "os" "reflect" "strings" "testing" + "time" "github.com/stretchr/testify/assert" "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/gologger/levels" + "github.com/projectdiscovery/ratelimit" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" ) @@ -21,8 +24,18 @@ func TestSourcesWithKeys(t *testing.T) { gologger.DefaultLogger.SetMaxLevel(levels.LevelDebug) - ctx := context.Background() - session, err := subscraping.NewSession(domain, "", 0, timeout) + ctxParent := context.Background() + //lint:ignore SA1029 reason + ctx := context.WithValue(ctxParent, "default", "default") + multiRateLimiter, err := ratelimit.NewMultiLimiter(ctx, &ratelimit.Options{ + Key: "default", + IsUnlimited: true, + MaxCount: math.MaxUint32, + Duration: time.Millisecond, + }) + assert.Nil(t, err) + + session, err := subscraping.NewSession(domain, "", multiRateLimiter, timeout) assert.Nil(t, err) var expected = subscraping.Result{Type: subscraping.Subdomain, Value: domain, Error: nil} @@ -42,7 +55,7 @@ func TestSourcesWithKeys(t *testing.T) { t.Run(source.Name(), func(t *testing.T) { var results []subscraping.Result - for result := range source.Run(ctx, domain, session) { + for result := range source.Run(ctxParent, domain, session) { results = append(results, result) assert.Equal(t, source.Name(), result.Source, "wrong source name") diff --git a/v2/pkg/passive/sources_wo_auth_test.go b/v2/pkg/passive/sources_wo_auth_test.go index 20b189af1..7212e93c8 100644 --- a/v2/pkg/passive/sources_wo_auth_test.go +++ b/v2/pkg/passive/sources_wo_auth_test.go @@ -3,15 +3,18 @@ package passive import ( "context" "fmt" + "math" "reflect" "strings" "testing" + "time" "github.com/stretchr/testify/assert" "golang.org/x/exp/slices" "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/gologger/levels" + "github.com/projectdiscovery/ratelimit" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" ) @@ -27,8 +30,19 @@ func TestSourcesWithoutKeys(t *testing.T) { gologger.DefaultLogger.SetMaxLevel(levels.LevelDebug) - ctx := context.Background() - session, err := subscraping.NewSession(domain, "", 0, timeout) + ctxParent := context.Background() + + //lint:ignore SA1029 reason + ctx := context.WithValue(ctxParent, "default", "default") + multiRateLimiter, err := ratelimit.NewMultiLimiter(ctx, &ratelimit.Options{ + Key: "default", + IsUnlimited: true, + MaxCount: math.MaxUint32, + Duration: time.Millisecond, + }) + assert.Nil(t, err) + + session, err := subscraping.NewSession(domain, "", multiRateLimiter, timeout) assert.Nil(t, err) var expected = subscraping.Result{Type: subscraping.Subdomain, Value: domain, Error: nil} diff --git a/v2/pkg/runner/enumerate.go b/v2/pkg/runner/enumerate.go index 1b1ab484b..1e56bd8d2 100644 --- a/v2/pkg/runner/enumerate.go +++ b/v2/pkg/runner/enumerate.go @@ -40,7 +40,7 @@ func (r *Runner) EnumerateSingleDomainWithCtx(ctx context.Context, domain string // Run the passive subdomain enumeration now := time.Now() - passiveResults := r.passiveAgent.EnumerateSubdomainsWithCtx(ctx, domain, r.options.Proxy, r.options.RateLimit, r.options.Timeout, time.Duration(r.options.MaxEnumerationTime)*time.Minute) + passiveResults := r.passiveAgent.EnumerateSubdomainsWithCtx(ctx, domain, r.options.Proxy, r.options.RateLimit, r.options.RateLimits.AsMap(), r.options.Timeout, time.Duration(r.options.MaxEnumerationTime)*time.Minute) wg := &sync.WaitGroup{} wg.Add(1) diff --git a/v2/pkg/runner/options.go b/v2/pkg/runner/options.go index 0832b6085..484b7c02a 100644 --- a/v2/pkg/runner/options.go +++ b/v2/pkg/runner/options.go @@ -59,7 +59,8 @@ type Options struct { Config string // Config contains the location of the config file ProviderConfig string // ProviderConfig contains the location of the provider config file Proxy string // HTTP proxy - RateLimit int // Maximum number of HTTP requests to send per second + RateLimit int // Global maximum number of HTTP requests to send per second + RateLimits goflags.RuntimeMap // Maximum number of HTTP requests to send per second ExcludeIps bool Match goflags.StringSlice Filter goflags.StringSlice @@ -114,7 +115,8 @@ func ParseOptions() *Options { ) createGroup(flagSet, "rate-limit", "Rate-limit", - flagSet.IntVarP(&options.RateLimit, "rate-limit", "rl", 0, "maximum number of http requests to send per second"), + flagSet.IntVarP(&options.RateLimit, "rate-limit", "rl", 0, "maximum number of http requests to send per second (global)"), + flagSet.RuntimeMapVar(&options.RateLimits, "rls", nil, "maximum number of http requests to send per second four providers in key=value format (-rls hackertarget=10)"), flagSet.IntVar(&options.Threads, "t", 10, "number of concurrent goroutines for resolving (-active only)"), ) diff --git a/v2/pkg/subscraping/agent.go b/v2/pkg/subscraping/agent.go index dd4e3f0c8..cde09421e 100644 --- a/v2/pkg/subscraping/agent.go +++ b/v2/pkg/subscraping/agent.go @@ -18,7 +18,7 @@ import ( ) // NewSession creates a new session object for a domain -func NewSession(domain string, proxy string, rateLimit, timeout int) (*Session, error) { +func NewSession(domain string, proxy string, multiRateLimiter *ratelimit.MultiLimiter, timeout int) (*Session, error) { Transport := &http.Transport{ MaxIdleConns: 100, MaxIdleConnsPerHost: 100, @@ -49,11 +49,7 @@ func NewSession(domain string, proxy string, rateLimit, timeout int) (*Session, session := &Session{Client: client} // Initiate rate limit instance - if rateLimit > 0 { - session.RateLimiter = ratelimit.New(context.Background(), uint(rateLimit), time.Second) - } else { - session.RateLimiter = ratelimit.NewUnlimited(context.Background()) - } + session.MultiRateLimiter = multiRateLimiter // Create a new extractor object for the current domain extractor, err := NewSubdomainExtractor(domain) @@ -106,7 +102,8 @@ func (s *Session) HTTPRequest(ctx context.Context, method, requestURL, cookies s req.Header.Set(key, value) } - s.RateLimiter.Take() + sourceName := ctx.Value("source").(string) + s.MultiRateLimiter.Take(sourceName) return httpRequestWrapper(s.Client, req) } @@ -125,7 +122,7 @@ func (s *Session) DiscardHTTPResponse(response *http.Response) { // Close the session func (s *Session) Close() { - s.RateLimiter.Stop() + s.MultiRateLimiter.Stop() s.Client.CloseIdleConnections() } diff --git a/v2/pkg/subscraping/types.go b/v2/pkg/subscraping/types.go index ae0d306df..fd79a0cd5 100644 --- a/v2/pkg/subscraping/types.go +++ b/v2/pkg/subscraping/types.go @@ -59,7 +59,7 @@ type Session struct { // Client is the current http client Client *http.Client // Rate limit instance - RateLimiter *ratelimit.Limiter + MultiRateLimiter *ratelimit.MultiLimiter } // Result is a result structure returned by a source From 19b8b3234d70f9a69e71b55727eb1290f8cc9712 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fan=20Can=20Bak=C4=B1r?= Date: Tue, 4 Jul 2023 12:14:57 +0300 Subject: [PATCH 02/19] check for err --- v2/pkg/subscraping/agent.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/v2/pkg/subscraping/agent.go b/v2/pkg/subscraping/agent.go index cde09421e..f4692bd0b 100644 --- a/v2/pkg/subscraping/agent.go +++ b/v2/pkg/subscraping/agent.go @@ -103,7 +103,10 @@ func (s *Session) HTTPRequest(ctx context.Context, method, requestURL, cookies s } sourceName := ctx.Value("source").(string) - s.MultiRateLimiter.Take(sourceName) + mrlErr := s.MultiRateLimiter.Take(sourceName) + if mrlErr != nil { + return nil, mrlErr + } return httpRequestWrapper(s.Client, req) } From 5178c216c93ecbc2165e09e62a727a42c5f419a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fan=20Can=20Bak=C4=B1r?= Date: Tue, 4 Jul 2023 09:31:03 +0000 Subject: [PATCH 03/19] check for lower and uppper bounds --- v2/pkg/passive/passive.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/v2/pkg/passive/passive.go b/v2/pkg/passive/passive.go index a62cb68eb..c34a57038 100644 --- a/v2/pkg/passive/passive.go +++ b/v2/pkg/passive/passive.go @@ -89,7 +89,7 @@ func buildMultiRateLimiter(ctx context.Context, a *Agent, rateLimit int, rateLim func sourceRateLimitOrGlobal(globalRateLimit int, sourceRateLimit interface{}) uint { if sourceRateLimitStr, ok := sourceRateLimit.(string); ok { sourceRateLimitUint, err := strconv.ParseUint(sourceRateLimitStr, 10, 64) - if err == nil { + if err == nil && sourceRateLimitUint > 0 && sourceRateLimitUint <= math.MaxUint32 { return uint(sourceRateLimitUint) } } From 16903b3589ac4f07b6724b6d050f3e34a9f5cd54 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fan=20Can=20Bak=C4=B1r?= Date: Tue, 4 Jul 2023 10:04:52 +0000 Subject: [PATCH 04/19] fix tests --- v2/pkg/passive/passive.go | 2 +- v2/pkg/passive/sources_w_auth_test.go | 20 ++++++++++---------- v2/pkg/passive/sources_wo_auth_test.go | 20 ++++++++++---------- 3 files changed, 21 insertions(+), 21 deletions(-) diff --git a/v2/pkg/passive/passive.go b/v2/pkg/passive/passive.go index c34a57038..18df1f1e7 100644 --- a/v2/pkg/passive/passive.go +++ b/v2/pkg/passive/passive.go @@ -50,7 +50,7 @@ func (a *Agent) EnumerateSubdomainsWithCtx(ctx context.Context, domain string, p wg.Add(1) go func(source subscraping.Source) { - //lint:ignore SA1029 reason + // nolint:staticcheck SA1029 reason ctxWithValue := context.WithValue(ctx, "source", source.Name()) for resp := range source.Run(ctxWithValue, domain, session) { results <- resp diff --git a/v2/pkg/passive/sources_w_auth_test.go b/v2/pkg/passive/sources_w_auth_test.go index ca2a0cc09..1681a1168 100644 --- a/v2/pkg/passive/sources_w_auth_test.go +++ b/v2/pkg/passive/sources_w_auth_test.go @@ -25,15 +25,13 @@ func TestSourcesWithKeys(t *testing.T) { gologger.DefaultLogger.SetMaxLevel(levels.LevelDebug) ctxParent := context.Background() - //lint:ignore SA1029 reason - ctx := context.WithValue(ctxParent, "default", "default") - multiRateLimiter, err := ratelimit.NewMultiLimiter(ctx, &ratelimit.Options{ - Key: "default", - IsUnlimited: true, - MaxCount: math.MaxUint32, - Duration: time.Millisecond, - }) - assert.Nil(t, err) + var multiRateLimiter *ratelimit.MultiLimiter + for _, source := range AllSources { + if !source.NeedsKey() { + continue + } + multiRateLimiter, _ = addRateLimiter(ctxParent, multiRateLimiter, source.Name(), math.MaxInt32, time.Millisecond) + } session, err := subscraping.NewSession(domain, "", multiRateLimiter, timeout) assert.Nil(t, err) @@ -55,7 +53,9 @@ func TestSourcesWithKeys(t *testing.T) { t.Run(source.Name(), func(t *testing.T) { var results []subscraping.Result - for result := range source.Run(ctxParent, domain, session) { + // nolint:staticcheck SA1029 reason + ctxWithValue := context.WithValue(ctxParent, "source", source.Name()) + for result := range source.Run(ctxWithValue, domain, session) { results = append(results, result) assert.Equal(t, source.Name(), result.Source, "wrong source name") diff --git a/v2/pkg/passive/sources_wo_auth_test.go b/v2/pkg/passive/sources_wo_auth_test.go index 7212e93c8..8bbaa7a91 100644 --- a/v2/pkg/passive/sources_wo_auth_test.go +++ b/v2/pkg/passive/sources_wo_auth_test.go @@ -32,15 +32,13 @@ func TestSourcesWithoutKeys(t *testing.T) { ctxParent := context.Background() - //lint:ignore SA1029 reason - ctx := context.WithValue(ctxParent, "default", "default") - multiRateLimiter, err := ratelimit.NewMultiLimiter(ctx, &ratelimit.Options{ - Key: "default", - IsUnlimited: true, - MaxCount: math.MaxUint32, - Duration: time.Millisecond, - }) - assert.Nil(t, err) + var multiRateLimiter *ratelimit.MultiLimiter + for _, source := range AllSources { + if source.NeedsKey() || slices.Contains(ignoredSources, source.Name()) { + continue + } + multiRateLimiter, _ = addRateLimiter(ctxParent, multiRateLimiter, source.Name(), math.MaxInt32, time.Millisecond) + } session, err := subscraping.NewSession(domain, "", multiRateLimiter, timeout) assert.Nil(t, err) @@ -55,7 +53,9 @@ func TestSourcesWithoutKeys(t *testing.T) { t.Run(source.Name(), func(t *testing.T) { var results []subscraping.Result - for result := range source.Run(ctx, domain, session) { + //nolint: SA1029 reason + ctxWithValue := context.WithValue(ctxParent, "source", source.Name()) + for result := range source.Run(ctxWithValue, domain, session) { results = append(results, result) assert.Equal(t, source.Name(), result.Source, "wrong source name") From 345f9cc9e3d4dfbdf97b498c429a1a962223e41a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fan=20Can=20Bak=C4=B1r?= Date: Tue, 4 Jul 2023 10:10:44 +0000 Subject: [PATCH 05/19] redo lint directive --- v2/pkg/passive/passive.go | 2 +- v2/pkg/passive/sources_w_auth_test.go | 2 +- v2/pkg/passive/sources_wo_auth_test.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/v2/pkg/passive/passive.go b/v2/pkg/passive/passive.go index 18df1f1e7..c34a57038 100644 --- a/v2/pkg/passive/passive.go +++ b/v2/pkg/passive/passive.go @@ -50,7 +50,7 @@ func (a *Agent) EnumerateSubdomainsWithCtx(ctx context.Context, domain string, p wg.Add(1) go func(source subscraping.Source) { - // nolint:staticcheck SA1029 reason + //lint:ignore SA1029 reason ctxWithValue := context.WithValue(ctx, "source", source.Name()) for resp := range source.Run(ctxWithValue, domain, session) { results <- resp diff --git a/v2/pkg/passive/sources_w_auth_test.go b/v2/pkg/passive/sources_w_auth_test.go index 1681a1168..2641131ed 100644 --- a/v2/pkg/passive/sources_w_auth_test.go +++ b/v2/pkg/passive/sources_w_auth_test.go @@ -53,7 +53,7 @@ func TestSourcesWithKeys(t *testing.T) { t.Run(source.Name(), func(t *testing.T) { var results []subscraping.Result - // nolint:staticcheck SA1029 reason + //lint:ignore SA1029 reason ctxWithValue := context.WithValue(ctxParent, "source", source.Name()) for result := range source.Run(ctxWithValue, domain, session) { results = append(results, result) diff --git a/v2/pkg/passive/sources_wo_auth_test.go b/v2/pkg/passive/sources_wo_auth_test.go index 8bbaa7a91..185252a37 100644 --- a/v2/pkg/passive/sources_wo_auth_test.go +++ b/v2/pkg/passive/sources_wo_auth_test.go @@ -53,7 +53,7 @@ func TestSourcesWithoutKeys(t *testing.T) { t.Run(source.Name(), func(t *testing.T) { var results []subscraping.Result - //nolint: SA1029 reason + //lint:ignore SA1029 reason ctxWithValue := context.WithValue(ctxParent, "source", source.Name()) for result := range source.Run(ctxWithValue, domain, session) { results = append(results, result) From 09b48824a74216af3bc559f93d2dc4a421e7d203 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fan=20Can=20Bak=C4=B1r?= Date: Thu, 6 Jul 2023 10:01:21 +0000 Subject: [PATCH 06/19] fix lint err --- v2/pkg/passive/passive.go | 10 +++++++--- v2/pkg/passive/sources_w_auth_test.go | 3 +-- v2/pkg/passive/sources_wo_auth_test.go | 3 +-- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/v2/pkg/passive/passive.go b/v2/pkg/passive/passive.go index c34a57038..6b247d80e 100644 --- a/v2/pkg/passive/passive.go +++ b/v2/pkg/passive/passive.go @@ -14,6 +14,12 @@ import ( "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" ) +type ctxarg string + +const ( + ctxSourceArg ctxarg = "source" +) + // EnumerateSubdomains wraps EnumerateSubdomainsWithCtx with an empty context func (a *Agent) EnumerateSubdomains(domain string, proxy string, rateLimit int, rateLimits map[string]interface{}, timeout int, maxEnumTime time.Duration) chan subscraping.Result { return a.EnumerateSubdomainsWithCtx(context.Background(), domain, proxy, rateLimit, rateLimits, timeout, maxEnumTime) @@ -48,10 +54,8 @@ func (a *Agent) EnumerateSubdomainsWithCtx(ctx context.Context, domain string, p // Run each source in parallel on the target domain for _, runner := range a.sources { wg.Add(1) - go func(source subscraping.Source) { - //lint:ignore SA1029 reason - ctxWithValue := context.WithValue(ctx, "source", source.Name()) + ctxWithValue := context.WithValue(ctx, ctxSourceArg, source.Name()) for resp := range source.Run(ctxWithValue, domain, session) { results <- resp } diff --git a/v2/pkg/passive/sources_w_auth_test.go b/v2/pkg/passive/sources_w_auth_test.go index 2641131ed..13e9d542d 100644 --- a/v2/pkg/passive/sources_w_auth_test.go +++ b/v2/pkg/passive/sources_w_auth_test.go @@ -53,8 +53,7 @@ func TestSourcesWithKeys(t *testing.T) { t.Run(source.Name(), func(t *testing.T) { var results []subscraping.Result - //lint:ignore SA1029 reason - ctxWithValue := context.WithValue(ctxParent, "source", source.Name()) + ctxWithValue := context.WithValue(ctxParent, ctxSourceArg, source.Name()) for result := range source.Run(ctxWithValue, domain, session) { results = append(results, result) diff --git a/v2/pkg/passive/sources_wo_auth_test.go b/v2/pkg/passive/sources_wo_auth_test.go index 185252a37..2613e9667 100644 --- a/v2/pkg/passive/sources_wo_auth_test.go +++ b/v2/pkg/passive/sources_wo_auth_test.go @@ -53,8 +53,7 @@ func TestSourcesWithoutKeys(t *testing.T) { t.Run(source.Name(), func(t *testing.T) { var results []subscraping.Result - //lint:ignore SA1029 reason - ctxWithValue := context.WithValue(ctxParent, "source", source.Name()) + ctxWithValue := context.WithValue(ctxParent, ctxSourceArg, source.Name()) for result := range source.Run(ctxWithValue, domain, session) { results = append(results, result) From 842dc62689c2a68d1baf6b6c6ce1729256f6a933 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fan=20Can=20Bak=C4=B1r?= Date: Thu, 6 Jul 2023 10:19:10 +0000 Subject: [PATCH 07/19] fix tests --- v2/pkg/passive/passive.go | 8 +------- v2/pkg/passive/sources_w_auth_test.go | 2 +- v2/pkg/passive/sources_wo_auth_test.go | 2 +- v2/pkg/subscraping/agent.go | 2 +- v2/pkg/subscraping/types.go | 6 ++++++ 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/v2/pkg/passive/passive.go b/v2/pkg/passive/passive.go index 6b247d80e..0e0d56a25 100644 --- a/v2/pkg/passive/passive.go +++ b/v2/pkg/passive/passive.go @@ -14,12 +14,6 @@ import ( "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" ) -type ctxarg string - -const ( - ctxSourceArg ctxarg = "source" -) - // EnumerateSubdomains wraps EnumerateSubdomainsWithCtx with an empty context func (a *Agent) EnumerateSubdomains(domain string, proxy string, rateLimit int, rateLimits map[string]interface{}, timeout int, maxEnumTime time.Duration) chan subscraping.Result { return a.EnumerateSubdomainsWithCtx(context.Background(), domain, proxy, rateLimit, rateLimits, timeout, maxEnumTime) @@ -55,7 +49,7 @@ func (a *Agent) EnumerateSubdomainsWithCtx(ctx context.Context, domain string, p for _, runner := range a.sources { wg.Add(1) go func(source subscraping.Source) { - ctxWithValue := context.WithValue(ctx, ctxSourceArg, source.Name()) + ctxWithValue := context.WithValue(ctx, subscraping.CtxSourceArg, source.Name()) for resp := range source.Run(ctxWithValue, domain, session) { results <- resp } diff --git a/v2/pkg/passive/sources_w_auth_test.go b/v2/pkg/passive/sources_w_auth_test.go index 13e9d542d..ffbe46bfd 100644 --- a/v2/pkg/passive/sources_w_auth_test.go +++ b/v2/pkg/passive/sources_w_auth_test.go @@ -53,7 +53,7 @@ func TestSourcesWithKeys(t *testing.T) { t.Run(source.Name(), func(t *testing.T) { var results []subscraping.Result - ctxWithValue := context.WithValue(ctxParent, ctxSourceArg, source.Name()) + ctxWithValue := context.WithValue(ctxParent, subscraping.CtxSourceArg, source.Name()) for result := range source.Run(ctxWithValue, domain, session) { results = append(results, result) diff --git a/v2/pkg/passive/sources_wo_auth_test.go b/v2/pkg/passive/sources_wo_auth_test.go index 2613e9667..fd7ac40fe 100644 --- a/v2/pkg/passive/sources_wo_auth_test.go +++ b/v2/pkg/passive/sources_wo_auth_test.go @@ -53,7 +53,7 @@ func TestSourcesWithoutKeys(t *testing.T) { t.Run(source.Name(), func(t *testing.T) { var results []subscraping.Result - ctxWithValue := context.WithValue(ctxParent, ctxSourceArg, source.Name()) + ctxWithValue := context.WithValue(ctxParent, subscraping.CtxSourceArg, source.Name()) for result := range source.Run(ctxWithValue, domain, session) { results = append(results, result) diff --git a/v2/pkg/subscraping/agent.go b/v2/pkg/subscraping/agent.go index f4692bd0b..0e2b7f13a 100644 --- a/v2/pkg/subscraping/agent.go +++ b/v2/pkg/subscraping/agent.go @@ -102,7 +102,7 @@ func (s *Session) HTTPRequest(ctx context.Context, method, requestURL, cookies s req.Header.Set(key, value) } - sourceName := ctx.Value("source").(string) + sourceName := ctx.Value(CtxSourceArg).(string) mrlErr := s.MultiRateLimiter.Take(sourceName) if mrlErr != nil { return nil, mrlErr diff --git a/v2/pkg/subscraping/types.go b/v2/pkg/subscraping/types.go index fd79a0cd5..b72f6b117 100644 --- a/v2/pkg/subscraping/types.go +++ b/v2/pkg/subscraping/types.go @@ -9,6 +9,12 @@ import ( "github.com/projectdiscovery/ratelimit" ) +type CtxArg string + +const ( + CtxSourceArg CtxArg = "source" +) + // BasicAuth request's Authorization header type BasicAuth struct { Username string From 1883f9fdec33a881babfc2238ca93e0ef4c0405b Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Thu, 6 Jul 2023 15:26:37 +0200 Subject: [PATCH 08/19] removing deprecated code --- v2/pkg/runner/options.go | 23 ++++++++--------------- 1 file changed, 8 insertions(+), 15 deletions(-) diff --git a/v2/pkg/runner/options.go b/v2/pkg/runner/options.go index 484b7c02a..b1c3530b2 100644 --- a/v2/pkg/runner/options.go +++ b/v2/pkg/runner/options.go @@ -97,24 +97,24 @@ func ParseOptions() *Options { flagSet := goflags.NewFlagSet() flagSet.SetDescription(`Subfinder is a subdomain discovery tool that discovers subdomains for websites by using passive online sources.`) - createGroup(flagSet, "input", "Input", + flagSet.CreateGroup("input", "Input", flagSet.StringSliceVarP(&options.Domain, "domain", "d", []string{}, "domains to find subdomains for", goflags.NormalizedStringSliceOptions), flagSet.StringVarP(&options.DomainsFile, "list", "dL", "", "file containing list of domains for subdomain discovery"), ) - createGroup(flagSet, "source", "Source", + flagSet.CreateGroup("source", "Source", flagSet.StringSliceVarP(&options.Sources, "sources", "s", []string{}, "specific sources to use for discovery (-s crtsh,github). Use -ls to display all available sources.", goflags.NormalizedStringSliceOptions), flagSet.BoolVar(&options.OnlyRecursive, "recursive", false, "use only sources that can handle subdomains recursively (e.g. subdomain.domain.tld vs domain.tld)"), flagSet.BoolVar(&options.All, "all", false, "use all sources for enumeration (slow)"), flagSet.StringSliceVarP(&options.ExcludeSources, "exclude-sources", "es", []string{}, "sources to exclude from enumeration (-es alienvault,zoomeye)", goflags.NormalizedStringSliceOptions), ) - createGroup(flagSet, "filter", "Filter", + flagSet.CreateGroup("filter", "Filter", flagSet.StringSliceVarP(&options.Match, "match", "m", []string{}, "subdomain or list of subdomain to match (file or comma separated)", goflags.FileNormalizedStringSliceOptions), flagSet.StringSliceVarP(&options.Filter, "filter", "f", []string{}, " subdomain or list of subdomain to filter (file or comma separated)", goflags.FileNormalizedStringSliceOptions), ) - createGroup(flagSet, "rate-limit", "Rate-limit", + flagSet.CreateGroup("rate-limit", "Rate-limit", flagSet.IntVarP(&options.RateLimit, "rate-limit", "rl", 0, "maximum number of http requests to send per second (global)"), flagSet.RuntimeMapVar(&options.RateLimits, "rls", nil, "maximum number of http requests to send per second four providers in key=value format (-rls hackertarget=10)"), flagSet.IntVar(&options.Threads, "t", 10, "number of concurrent goroutines for resolving (-active only)"), @@ -125,7 +125,7 @@ func ParseOptions() *Options { flagSet.BoolVarP(&options.DisableUpdateCheck, "disable-update-check", "duc", false, "disable automatic subfinder update check"), ) - createGroup(flagSet, "output", "Output", + flagSet.CreateGroup("output", "Output", flagSet.StringVarP(&options.OutputFile, "output", "o", "", "file to write output to"), flagSet.BoolVarP(&options.JSON, "json", "oJ", false, "write output in JSONL(ines) format"), flagSet.StringVarP(&options.OutputDirectory, "output-dir", "oD", "", "directory to write output (-dL only)"), @@ -133,7 +133,7 @@ func ParseOptions() *Options { flagSet.BoolVarP(&options.HostIP, "ip", "oI", false, "include host IP in output (-active only)"), ) - createGroup(flagSet, "configuration", "Configuration", + flagSet.CreateGroup("configuration", "Configuration", flagSet.StringVar(&options.Config, "config", defaultConfigLocation, "flag config file"), flagSet.StringVarP(&options.ProviderConfig, "provider-config", "pc", defaultProviderConfigLocation, "provider config file"), flagSet.StringSliceVar(&options.Resolvers, "r", []string{}, "comma separated list of resolvers to use", goflags.NormalizedStringSliceOptions), @@ -143,7 +143,7 @@ func ParseOptions() *Options { flagSet.BoolVarP(&options.ExcludeIps, "exclude-ip", "ei", false, "exclude IPs from the list of domains"), ) - createGroup(flagSet, "debug", "Debug", + flagSet.CreateGroup("debug", "Debug", flagSet.BoolVar(&options.Silent, "silent", false, "show only subdomains in output"), flagSet.BoolVar(&options.Version, "version", false, "show version of subfinder"), flagSet.BoolVar(&options.Verbose, "v", false, "show verbose output"), @@ -152,7 +152,7 @@ func ParseOptions() *Options { flagSet.BoolVar(&options.Statistics, "stats", false, "report source statistics"), ) - createGroup(flagSet, "optimization", "Optimization", + flagSet.CreateGroup("optimization", "Optimization", flagSet.IntVar(&options.Timeout, "timeout", 30, "seconds to wait before timing out"), flagSet.IntVar(&options.MaxEnumerationTime, "max-time", 10, "minutes to wait for enumeration results"), ) @@ -287,13 +287,6 @@ func listSources(options *Options) { } } -func createGroup(flagSet *goflags.FlagSet, groupName, description string, flags ...*goflags.FlagData) { - flagSet.SetGroup(groupName, description) - for _, currentFlag := range flags { - currentFlag.Group(groupName) - } -} - func (options *Options) preProcessOptions() { for i, domain := range options.Domain { options.Domain[i], _ = sanitize(domain) From df8fbab326566a183106c8476fa004cedcb6f88d Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Thu, 6 Jul 2023 15:27:47 +0200 Subject: [PATCH 09/19] []string => nil --- v2/pkg/runner/options.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/v2/pkg/runner/options.go b/v2/pkg/runner/options.go index b1c3530b2..8d23c18dc 100644 --- a/v2/pkg/runner/options.go +++ b/v2/pkg/runner/options.go @@ -98,20 +98,20 @@ func ParseOptions() *Options { flagSet.SetDescription(`Subfinder is a subdomain discovery tool that discovers subdomains for websites by using passive online sources.`) flagSet.CreateGroup("input", "Input", - flagSet.StringSliceVarP(&options.Domain, "domain", "d", []string{}, "domains to find subdomains for", goflags.NormalizedStringSliceOptions), + flagSet.StringSliceVarP(&options.Domain, "domain", "d", nil, "domains to find subdomains for", goflags.NormalizedStringSliceOptions), flagSet.StringVarP(&options.DomainsFile, "list", "dL", "", "file containing list of domains for subdomain discovery"), ) flagSet.CreateGroup("source", "Source", - flagSet.StringSliceVarP(&options.Sources, "sources", "s", []string{}, "specific sources to use for discovery (-s crtsh,github). Use -ls to display all available sources.", goflags.NormalizedStringSliceOptions), + flagSet.StringSliceVarP(&options.Sources, "sources", "s", nil, "specific sources to use for discovery (-s crtsh,github). Use -ls to display all available sources.", goflags.NormalizedStringSliceOptions), flagSet.BoolVar(&options.OnlyRecursive, "recursive", false, "use only sources that can handle subdomains recursively (e.g. subdomain.domain.tld vs domain.tld)"), flagSet.BoolVar(&options.All, "all", false, "use all sources for enumeration (slow)"), - flagSet.StringSliceVarP(&options.ExcludeSources, "exclude-sources", "es", []string{}, "sources to exclude from enumeration (-es alienvault,zoomeye)", goflags.NormalizedStringSliceOptions), + flagSet.StringSliceVarP(&options.ExcludeSources, "exclude-sources", "es", nil, "sources to exclude from enumeration (-es alienvault,zoomeye)", goflags.NormalizedStringSliceOptions), ) flagSet.CreateGroup("filter", "Filter", - flagSet.StringSliceVarP(&options.Match, "match", "m", []string{}, "subdomain or list of subdomain to match (file or comma separated)", goflags.FileNormalizedStringSliceOptions), - flagSet.StringSliceVarP(&options.Filter, "filter", "f", []string{}, " subdomain or list of subdomain to filter (file or comma separated)", goflags.FileNormalizedStringSliceOptions), + flagSet.StringSliceVarP(&options.Match, "match", "m", nil, "subdomain or list of subdomain to match (file or comma separated)", goflags.FileNormalizedStringSliceOptions), + flagSet.StringSliceVarP(&options.Filter, "filter", "f", nil, " subdomain or list of subdomain to filter (file or comma separated)", goflags.FileNormalizedStringSliceOptions), ) flagSet.CreateGroup("rate-limit", "Rate-limit", @@ -136,7 +136,7 @@ func ParseOptions() *Options { flagSet.CreateGroup("configuration", "Configuration", flagSet.StringVar(&options.Config, "config", defaultConfigLocation, "flag config file"), flagSet.StringVarP(&options.ProviderConfig, "provider-config", "pc", defaultProviderConfigLocation, "provider config file"), - flagSet.StringSliceVar(&options.Resolvers, "r", []string{}, "comma separated list of resolvers to use", goflags.NormalizedStringSliceOptions), + flagSet.StringSliceVar(&options.Resolvers, "r", nil, "comma separated list of resolvers to use", goflags.NormalizedStringSliceOptions), flagSet.StringVarP(&options.ResolverList, "rlist", "rL", "", "file containing list of resolvers to use"), flagSet.BoolVarP(&options.RemoveWildcard, "active", "nW", false, "display active subdomains only"), flagSet.StringVar(&options.Proxy, "proxy", "", "http proxy to use with subfinder"), From 2f38931d714342ed40c9d9440f59aa8cc30b74a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fan=20Can=20Bak=C4=B1r?= Date: Mon, 10 Jul 2023 08:55:39 +0000 Subject: [PATCH 10/19] add new type for ratelimit cfg --- v2/go.mod | 16 ++++++++-------- v2/go.sum | 16 ++++++++++++++++ v2/pkg/passive/passive.go | 26 +++++++++++--------------- v2/pkg/runner/enumerate.go | 20 +++++++++++++++++++- v2/pkg/subscraping/types.go | 6 ++++++ 5 files changed, 60 insertions(+), 24 deletions(-) diff --git a/v2/go.mod b/v2/go.mod index 64452ff3c..1acfeb3f3 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -12,7 +12,7 @@ require ( github.com/projectdiscovery/fdmax v0.0.4 github.com/projectdiscovery/gologger v1.1.10 github.com/projectdiscovery/ratelimit v0.0.9 - github.com/projectdiscovery/utils v0.0.39 + github.com/projectdiscovery/utils v0.0.41 github.com/rs/xid v1.5.0 github.com/stretchr/testify v1.8.4 github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 @@ -34,7 +34,7 @@ require ( github.com/dlclark/regexp2 v1.8.1 // indirect github.com/dsnet/compress v0.0.1 // indirect github.com/fatih/color v1.14.1 // indirect - github.com/golang/protobuf v1.5.2 // indirect + github.com/golang/protobuf v1.5.3 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/google/go-github/v30 v30.1.0 // indirect github.com/google/go-querystring v1.1.0 // indirect @@ -64,13 +64,13 @@ require ( github.com/yuin/goldmark v1.5.4 // indirect github.com/yuin/goldmark-emoji v1.0.1 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/crypto v0.10.0 // indirect + golang.org/x/crypto v0.11.0 // indirect golang.org/x/mod v0.10.0 // indirect - golang.org/x/oauth2 v0.9.0 // indirect - golang.org/x/text v0.10.0 // indirect + golang.org/x/oauth2 v0.10.0 // indirect + golang.org/x/text v0.11.0 // indirect golang.org/x/tools v0.8.0 // indirect google.golang.org/appengine v1.6.7 // indirect - google.golang.org/protobuf v1.29.1 // indirect + google.golang.org/protobuf v1.31.0 // indirect gopkg.in/djherbis/times.v1 v1.3.0 // indirect ) @@ -85,6 +85,6 @@ require ( github.com/pmezard/go-difflib v1.0.0 // indirect github.com/projectdiscovery/goflags v0.1.10 github.com/projectdiscovery/retryabledns v1.0.23 // indirect - golang.org/x/net v0.11.0 // indirect - golang.org/x/sys v0.9.0 // indirect + golang.org/x/net v0.12.0 // indirect + golang.org/x/sys v0.10.0 // indirect ) diff --git a/v2/go.sum b/v2/go.sum index 6309da0bd..f7ba7690f 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -41,6 +41,8 @@ github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5y github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= @@ -134,6 +136,8 @@ github.com/projectdiscovery/retryablehttp-go v1.0.15 h1:kP9x9f++QimRwb8ABqnI1dhE github.com/projectdiscovery/retryablehttp-go v1.0.15/go.mod h1:+OzSFUv3sQcPt+MgbNx6X/Q3ESxqPUQSphqG5kxoIgI= github.com/projectdiscovery/utils v0.0.39 h1:iyi5qPilENRmFyt16qtd58pb65fUu0wAU2C0Lq5t6zo= github.com/projectdiscovery/utils v0.0.39/go.mod h1:rrd8dTBuKEScNMLgs1Xiu8rPCVeR0QTzmRcQ5iM3ymo= +github.com/projectdiscovery/utils v0.0.41 h1:GY0/EhZR2DYne20eS8KApzcQpYJ5rWw8bSFYBDvJH5Q= +github.com/projectdiscovery/utils v0.0.41/go.mod h1:e+67VwXkS9o+dnRU0jM1BRdjm5GvNwkG/B2yhpZh42s= github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis= @@ -178,6 +182,8 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.0.0-20211209193657-4570a0811e8b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.10.0 h1:LKqV2xt9+kDzSTfOhx4FrkEBcMrAgHSYgzywV9zcGmM= golang.org/x/crypto v0.10.0/go.mod h1:o4eNf7Ede1fv+hwOwZsTHl9EsPFO6q6ZvYR8vYfY45I= +golang.org/x/crypto v0.11.0 h1:6Ewdq3tDic1mg5xRO4milcWCfMVQhI4NkqWWvqejpuA= +golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio= golang.org/x/exp v0.0.0-20230420155640-133eef4313cb h1:rhjz/8Mbfa8xROFiH+MQphmAmgqRM0bOMnytznhWEXk= golang.org/x/exp v0.0.0-20230420155640-133eef4313cb/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= @@ -195,9 +201,13 @@ golang.org/x/net v0.0.0-20221002022538-bcab6841153b/go.mod h1:YDH+HFinaLZZlnHAfS golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.11.0 h1:Gi2tvZIJyBtO9SDr1q9h5hEQCp/4L2RQ+ar0qjx2oNU= golang.org/x/net v0.11.0/go.mod h1:2L/ixqYpgIVXmeoSA/4Lu7BzTG4KIyPIryS4IsOd1oQ= +golang.org/x/net v0.12.0 h1:cfawfvKITfUsFCeJIHJrbSxpeu/E81khclypR0GVT50= +golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.9.0 h1:BPpt2kU7oMRq3kCHAA1tbSEshXRw1LpG2ztgDwrzuAs= golang.org/x/oauth2 v0.9.0/go.mod h1:qYgFZaFiu6Wg24azG8bdV52QJXJGbZzIIsRCdVKzbLw= +golang.org/x/oauth2 v0.10.0 h1:zHCpF2Khkwy4mMB4bv0U37YtJdTGW8jI0glAApi0Kh8= +golang.org/x/oauth2 v0.10.0/go.mod h1:kTpgurOux7LqtuxjuyZa4Gj2gdezIt/jQtGnNFfypQI= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= @@ -214,6 +224,8 @@ golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.9.0 h1:KS/R3tvhPqvJvwcKfnBHJwwthS11LRhmM5D59eEXa0s= golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= +golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -226,6 +238,8 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.10.0 h1:UpjohKhiEgNc0CSauXmwYftY1+LlaC75SJwh0SgCX58= golang.org/x/text v0.10.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4= +golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= @@ -240,6 +254,8 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.29.1 h1:7QBf+IK2gx70Ap/hDsOmam3GE0v9HicjfEdAxE62UoM= google.golang.org/protobuf v1.29.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= +google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/djherbis/times.v1 v1.3.0 h1:uxMS4iMtH6Pwsxog094W0FYldiNnfY/xba00vq6C2+o= diff --git a/v2/pkg/passive/passive.go b/v2/pkg/passive/passive.go index 0e0d56a25..25662ce04 100644 --- a/v2/pkg/passive/passive.go +++ b/v2/pkg/passive/passive.go @@ -5,7 +5,6 @@ import ( "fmt" "math" "sort" - "strconv" "strings" "sync" "time" @@ -15,18 +14,18 @@ import ( ) // EnumerateSubdomains wraps EnumerateSubdomainsWithCtx with an empty context -func (a *Agent) EnumerateSubdomains(domain string, proxy string, rateLimit int, rateLimits map[string]interface{}, timeout int, maxEnumTime time.Duration) chan subscraping.Result { - return a.EnumerateSubdomainsWithCtx(context.Background(), domain, proxy, rateLimit, rateLimits, timeout, maxEnumTime) +func (a *Agent) EnumerateSubdomains(domain string, proxy string, rateLimitJar *subscraping.RateLimitJar, timeout int, maxEnumTime time.Duration) chan subscraping.Result { + return a.EnumerateSubdomainsWithCtx(context.Background(), domain, proxy, rateLimitJar, timeout, maxEnumTime) } // EnumerateSubdomainsWithCtx enumerates all the subdomains for a given domain -func (a *Agent) EnumerateSubdomainsWithCtx(ctx context.Context, domain string, proxy string, rateLimit int, rateLimits map[string]interface{}, timeout int, maxEnumTime time.Duration) chan subscraping.Result { +func (a *Agent) EnumerateSubdomainsWithCtx(ctx context.Context, domain string, proxy string, rateLimitJar *subscraping.RateLimitJar, timeout int, maxEnumTime time.Duration) chan subscraping.Result { results := make(chan subscraping.Result) go func() { defer close(results) - multiRateLimiter, err := buildMultiRateLimiter(ctx, a, rateLimit, rateLimits) + multiRateLimiter, err := buildMultiRateLimiter(ctx, a, rateLimitJar) if err != nil { results <- subscraping.Result{ Type: subscraping.Error, Error: fmt.Errorf("could not init multi rate limiter for %s: %s", domain, err), @@ -62,13 +61,13 @@ func (a *Agent) EnumerateSubdomainsWithCtx(ctx context.Context, domain string, p return results } -func buildMultiRateLimiter(ctx context.Context, a *Agent, rateLimit int, rateLimits map[string]interface{}) (*ratelimit.MultiLimiter, error) { +func buildMultiRateLimiter(ctx context.Context, a *Agent, rateLimitJar *subscraping.RateLimitJar) (*ratelimit.MultiLimiter, error) { var multiRateLimiter *ratelimit.MultiLimiter var err error for _, source := range a.sources { var rl uint - if sourceRateLimit, ok := rateLimits[strings.ToLower(source.Name())]; ok { - rl = sourceRateLimitOrGlobal(rateLimit, sourceRateLimit) + if sourceRateLimit, ok := rateLimitJar.Custom.Get(strings.ToLower(source.Name())); ok { + rl = sourceRateLimitOrGlobal(rateLimitJar.Global, sourceRateLimit) } if rl > 0 { @@ -84,14 +83,11 @@ func buildMultiRateLimiter(ctx context.Context, a *Agent, rateLimit int, rateLim return multiRateLimiter, err } -func sourceRateLimitOrGlobal(globalRateLimit int, sourceRateLimit interface{}) uint { - if sourceRateLimitStr, ok := sourceRateLimit.(string); ok { - sourceRateLimitUint, err := strconv.ParseUint(sourceRateLimitStr, 10, 64) - if err == nil && sourceRateLimitUint > 0 && sourceRateLimitUint <= math.MaxUint32 { - return uint(sourceRateLimitUint) - } +func sourceRateLimitOrGlobal(globalRateLimit uint, sourceRateLimit uint) uint { + if sourceRateLimit > 0 { + return sourceRateLimit } - return uint(globalRateLimit) + return globalRateLimit } func addRateLimiter(ctx context.Context, multiRateLimiter *ratelimit.MultiLimiter, key string, maxCount uint, duration time.Duration) (*ratelimit.MultiLimiter, error) { diff --git a/v2/pkg/runner/enumerate.go b/v2/pkg/runner/enumerate.go index 1e56bd8d2..56a2904f8 100644 --- a/v2/pkg/runner/enumerate.go +++ b/v2/pkg/runner/enumerate.go @@ -3,6 +3,8 @@ package runner import ( "context" "io" + "math" + "strconv" "strings" "sync" "time" @@ -10,6 +12,7 @@ import ( "github.com/hako/durafmt" "github.com/projectdiscovery/gologger" + mapsutil "github.com/projectdiscovery/utils/maps" "github.com/projectdiscovery/subfinder/v2/pkg/resolve" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" @@ -38,9 +41,24 @@ func (r *Runner) EnumerateSingleDomainWithCtx(ctx context.Context, domain string } } + rateLimitJar := &subscraping.RateLimitJar{ + Global: uint(r.options.RateLimit), + Custom: mapsutil.SyncLockMap[string, uint]{ + Map: make(map[string]uint), + }, + } + for source, sourceRateLimit := range r.options.RateLimits.AsMap() { + if sourceRateLimitStr, ok := sourceRateLimit.(string); ok { + sourceRateLimitUint, err := strconv.ParseUint(sourceRateLimitStr, 10, 64) + if err == nil && sourceRateLimitUint > 0 && sourceRateLimitUint <= math.MaxUint32 { + rateLimitJar.Custom.Set(source, uint(sourceRateLimitUint)) + } + } + } + // Run the passive subdomain enumeration now := time.Now() - passiveResults := r.passiveAgent.EnumerateSubdomainsWithCtx(ctx, domain, r.options.Proxy, r.options.RateLimit, r.options.RateLimits.AsMap(), r.options.Timeout, time.Duration(r.options.MaxEnumerationTime)*time.Minute) + passiveResults := r.passiveAgent.EnumerateSubdomainsWithCtx(ctx, domain, r.options.Proxy, rateLimitJar, r.options.Timeout, time.Duration(r.options.MaxEnumerationTime)*time.Minute) wg := &sync.WaitGroup{} wg.Add(1) diff --git a/v2/pkg/subscraping/types.go b/v2/pkg/subscraping/types.go index b72f6b117..5032b14a7 100644 --- a/v2/pkg/subscraping/types.go +++ b/v2/pkg/subscraping/types.go @@ -7,6 +7,7 @@ import ( "time" "github.com/projectdiscovery/ratelimit" + mapsutil "github.com/projectdiscovery/utils/maps" ) type CtxArg string @@ -15,6 +16,11 @@ const ( CtxSourceArg CtxArg = "source" ) +type RateLimitJar struct { + Global uint + Custom mapsutil.SyncLockMap[string, uint] +} + // BasicAuth request's Authorization header type BasicAuth struct { Username string From df293d52e8443b3047a0d7735ec3e74d1b75ffa9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fan=20Can=20Bak=C4=B1r?= Date: Mon, 10 Jul 2023 09:29:33 +0000 Subject: [PATCH 11/19] move rate limit init in NewRunner func --- v2/pkg/runner/enumerate.go | 20 +------------------- v2/pkg/runner/runner.go | 22 ++++++++++++++++++++++ 2 files changed, 23 insertions(+), 19 deletions(-) diff --git a/v2/pkg/runner/enumerate.go b/v2/pkg/runner/enumerate.go index 56a2904f8..94afb01e9 100644 --- a/v2/pkg/runner/enumerate.go +++ b/v2/pkg/runner/enumerate.go @@ -3,8 +3,6 @@ package runner import ( "context" "io" - "math" - "strconv" "strings" "sync" "time" @@ -12,7 +10,6 @@ import ( "github.com/hako/durafmt" "github.com/projectdiscovery/gologger" - mapsutil "github.com/projectdiscovery/utils/maps" "github.com/projectdiscovery/subfinder/v2/pkg/resolve" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" @@ -41,24 +38,9 @@ func (r *Runner) EnumerateSingleDomainWithCtx(ctx context.Context, domain string } } - rateLimitJar := &subscraping.RateLimitJar{ - Global: uint(r.options.RateLimit), - Custom: mapsutil.SyncLockMap[string, uint]{ - Map: make(map[string]uint), - }, - } - for source, sourceRateLimit := range r.options.RateLimits.AsMap() { - if sourceRateLimitStr, ok := sourceRateLimit.(string); ok { - sourceRateLimitUint, err := strconv.ParseUint(sourceRateLimitStr, 10, 64) - if err == nil && sourceRateLimitUint > 0 && sourceRateLimitUint <= math.MaxUint32 { - rateLimitJar.Custom.Set(source, uint(sourceRateLimitUint)) - } - } - } - // Run the passive subdomain enumeration now := time.Now() - passiveResults := r.passiveAgent.EnumerateSubdomainsWithCtx(ctx, domain, r.options.Proxy, rateLimitJar, r.options.Timeout, time.Duration(r.options.MaxEnumerationTime)*time.Minute) + passiveResults := r.passiveAgent.EnumerateSubdomainsWithCtx(ctx, domain, r.options.Proxy, r.ratelimitJar, r.options.Timeout, time.Duration(r.options.MaxEnumerationTime)*time.Minute) wg := &sync.WaitGroup{} wg.Add(1) diff --git a/v2/pkg/runner/runner.go b/v2/pkg/runner/runner.go index 069f9e0e4..3c00c2d80 100644 --- a/v2/pkg/runner/runner.go +++ b/v2/pkg/runner/runner.go @@ -4,18 +4,22 @@ import ( "bufio" "context" "io" + "math" "os" "path" "regexp" + "strconv" "strings" "github.com/pkg/errors" "github.com/projectdiscovery/gologger" fileutil "github.com/projectdiscovery/utils/file" + mapsutil "github.com/projectdiscovery/utils/maps" "github.com/projectdiscovery/subfinder/v2/pkg/passive" "github.com/projectdiscovery/subfinder/v2/pkg/resolve" + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" ) // Runner is an instance of the subdomain enumeration @@ -24,6 +28,7 @@ type Runner struct { options *Options passiveAgent *passive.Agent resolverClient *resolve.Resolver + ratelimitJar *subscraping.RateLimitJar } // NewRunner creates a new runner struct instance by parsing @@ -51,6 +56,23 @@ func NewRunner(options *Options) (*Runner, error) { return nil, err } + // Initialize the rate limit jar + runner.ratelimitJar = &subscraping.RateLimitJar{ + Global: uint(options.RateLimit), + Custom: mapsutil.SyncLockMap[string, uint]{ + Map: make(map[string]uint), + }, + } + + for source, sourceRateLimit := range options.RateLimits.AsMap() { + if sourceRateLimitStr, ok := sourceRateLimit.(string); ok { + sourceRateLimitUint, err := strconv.ParseUint(sourceRateLimitStr, 10, 64) + if err == nil && sourceRateLimitUint > 0 && sourceRateLimitUint <= math.MaxUint32 { + runner.ratelimitJar.Custom.Set(source, uint(sourceRateLimitUint)) + } + } + } + return runner, nil } From 19aa0652aac5bd0745d8618867e08d5aeeaba989 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fan=20Can=20Bak=C4=B1r?= Date: Mon, 10 Jul 2023 09:32:56 +0000 Subject: [PATCH 12/19] commit to last commit --- v2/pkg/runner/runner.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/v2/pkg/runner/runner.go b/v2/pkg/runner/runner.go index 3c00c2d80..b21f16304 100644 --- a/v2/pkg/runner/runner.go +++ b/v2/pkg/runner/runner.go @@ -68,7 +68,7 @@ func NewRunner(options *Options) (*Runner, error) { if sourceRateLimitStr, ok := sourceRateLimit.(string); ok { sourceRateLimitUint, err := strconv.ParseUint(sourceRateLimitStr, 10, 64) if err == nil && sourceRateLimitUint > 0 && sourceRateLimitUint <= math.MaxUint32 { - runner.ratelimitJar.Custom.Set(source, uint(sourceRateLimitUint)) + _ = runner.ratelimitJar.Custom.Set(source, uint(sourceRateLimitUint)) } } } From b1b45fc42dbc9cce371d27da6c8813d20b698400 Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Mon, 10 Jul 2023 16:08:30 +0200 Subject: [PATCH 13/19] little renaming --- v2/pkg/passive/passive.go | 18 +++++++++--------- v2/pkg/runner/enumerate.go | 2 +- v2/pkg/runner/runner.go | 8 ++++---- v2/pkg/subscraping/types.go | 6 +++--- 4 files changed, 17 insertions(+), 17 deletions(-) diff --git a/v2/pkg/passive/passive.go b/v2/pkg/passive/passive.go index 25662ce04..f3c437495 100644 --- a/v2/pkg/passive/passive.go +++ b/v2/pkg/passive/passive.go @@ -14,18 +14,18 @@ import ( ) // EnumerateSubdomains wraps EnumerateSubdomainsWithCtx with an empty context -func (a *Agent) EnumerateSubdomains(domain string, proxy string, rateLimitJar *subscraping.RateLimitJar, timeout int, maxEnumTime time.Duration) chan subscraping.Result { - return a.EnumerateSubdomainsWithCtx(context.Background(), domain, proxy, rateLimitJar, timeout, maxEnumTime) +func (a *Agent) EnumerateSubdomains(domain string, proxy string, rateLimit *subscraping.CustomRateLimit, timeout int, maxEnumTime time.Duration) chan subscraping.Result { + return a.EnumerateSubdomainsWithCtx(context.Background(), domain, proxy, rateLimit, timeout, maxEnumTime) } // EnumerateSubdomainsWithCtx enumerates all the subdomains for a given domain -func (a *Agent) EnumerateSubdomainsWithCtx(ctx context.Context, domain string, proxy string, rateLimitJar *subscraping.RateLimitJar, timeout int, maxEnumTime time.Duration) chan subscraping.Result { +func (a *Agent) EnumerateSubdomainsWithCtx(ctx context.Context, domain string, proxy string, rateLimit *subscraping.CustomRateLimit, timeout int, maxEnumTime time.Duration) chan subscraping.Result { results := make(chan subscraping.Result) go func() { defer close(results) - multiRateLimiter, err := buildMultiRateLimiter(ctx, a, rateLimitJar) + multiRateLimiter, err := a.buildMultiRateLimiter(ctx, rateLimit) if err != nil { results <- subscraping.Result{ Type: subscraping.Error, Error: fmt.Errorf("could not init multi rate limiter for %s: %s", domain, err), @@ -61,13 +61,13 @@ func (a *Agent) EnumerateSubdomainsWithCtx(ctx context.Context, domain string, p return results } -func buildMultiRateLimiter(ctx context.Context, a *Agent, rateLimitJar *subscraping.RateLimitJar) (*ratelimit.MultiLimiter, error) { +func (a *Agent) buildMultiRateLimiter(ctx context.Context, rateLimit *subscraping.CustomRateLimit) (*ratelimit.MultiLimiter, error) { var multiRateLimiter *ratelimit.MultiLimiter var err error for _, source := range a.sources { var rl uint - if sourceRateLimit, ok := rateLimitJar.Custom.Get(strings.ToLower(source.Name())); ok { - rl = sourceRateLimitOrGlobal(rateLimitJar.Global, sourceRateLimit) + if sourceRateLimit, ok := rateLimit.Custom.Get(strings.ToLower(source.Name())); ok { + rl = sourceRateLimitOrDefault(rateLimit.Default, sourceRateLimit) } if rl > 0 { @@ -83,11 +83,11 @@ func buildMultiRateLimiter(ctx context.Context, a *Agent, rateLimitJar *subscrap return multiRateLimiter, err } -func sourceRateLimitOrGlobal(globalRateLimit uint, sourceRateLimit uint) uint { +func sourceRateLimitOrDefault(defaultRateLimit uint, sourceRateLimit uint) uint { if sourceRateLimit > 0 { return sourceRateLimit } - return globalRateLimit + return defaultRateLimit } func addRateLimiter(ctx context.Context, multiRateLimiter *ratelimit.MultiLimiter, key string, maxCount uint, duration time.Duration) (*ratelimit.MultiLimiter, error) { diff --git a/v2/pkg/runner/enumerate.go b/v2/pkg/runner/enumerate.go index 94afb01e9..ee54625d1 100644 --- a/v2/pkg/runner/enumerate.go +++ b/v2/pkg/runner/enumerate.go @@ -40,7 +40,7 @@ func (r *Runner) EnumerateSingleDomainWithCtx(ctx context.Context, domain string // Run the passive subdomain enumeration now := time.Now() - passiveResults := r.passiveAgent.EnumerateSubdomainsWithCtx(ctx, domain, r.options.Proxy, r.ratelimitJar, r.options.Timeout, time.Duration(r.options.MaxEnumerationTime)*time.Minute) + passiveResults := r.passiveAgent.EnumerateSubdomainsWithCtx(ctx, domain, r.options.Proxy, r.rateLimit, r.options.Timeout, time.Duration(r.options.MaxEnumerationTime)*time.Minute) wg := &sync.WaitGroup{} wg.Add(1) diff --git a/v2/pkg/runner/runner.go b/v2/pkg/runner/runner.go index b21f16304..e226aee20 100644 --- a/v2/pkg/runner/runner.go +++ b/v2/pkg/runner/runner.go @@ -28,7 +28,7 @@ type Runner struct { options *Options passiveAgent *passive.Agent resolverClient *resolve.Resolver - ratelimitJar *subscraping.RateLimitJar + rateLimit *subscraping.CustomRateLimit } // NewRunner creates a new runner struct instance by parsing @@ -57,8 +57,8 @@ func NewRunner(options *Options) (*Runner, error) { } // Initialize the rate limit jar - runner.ratelimitJar = &subscraping.RateLimitJar{ - Global: uint(options.RateLimit), + runner.rateLimit = &subscraping.CustomRateLimit{ + Default: uint(options.RateLimit), Custom: mapsutil.SyncLockMap[string, uint]{ Map: make(map[string]uint), }, @@ -68,7 +68,7 @@ func NewRunner(options *Options) (*Runner, error) { if sourceRateLimitStr, ok := sourceRateLimit.(string); ok { sourceRateLimitUint, err := strconv.ParseUint(sourceRateLimitStr, 10, 64) if err == nil && sourceRateLimitUint > 0 && sourceRateLimitUint <= math.MaxUint32 { - _ = runner.ratelimitJar.Custom.Set(source, uint(sourceRateLimitUint)) + _ = runner.rateLimit.Custom.Set(source, uint(sourceRateLimitUint)) } } } diff --git a/v2/pkg/subscraping/types.go b/v2/pkg/subscraping/types.go index 5032b14a7..89f12483c 100644 --- a/v2/pkg/subscraping/types.go +++ b/v2/pkg/subscraping/types.go @@ -16,9 +16,9 @@ const ( CtxSourceArg CtxArg = "source" ) -type RateLimitJar struct { - Global uint - Custom mapsutil.SyncLockMap[string, uint] +type CustomRateLimit struct { + Default uint + Custom mapsutil.SyncLockMap[string, uint] } // BasicAuth request's Authorization header From f92cbc0953a5cb15015ae0878a49c12c31b28dc6 Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Mon, 10 Jul 2023 16:27:16 +0200 Subject: [PATCH 14/19] flagging hackertarget as ignored --- v2/pkg/passive/sources_wo_auth_test.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/v2/pkg/passive/sources_wo_auth_test.go b/v2/pkg/passive/sources_wo_auth_test.go index fd7ac40fe..88f64e04b 100644 --- a/v2/pkg/passive/sources_wo_auth_test.go +++ b/v2/pkg/passive/sources_wo_auth_test.go @@ -20,9 +20,10 @@ import ( func TestSourcesWithoutKeys(t *testing.T) { ignoredSources := []string{ - "commoncrawl", // commoncrawl is under resourced and will likely time-out so step over it for this test https://groups.google.com/u/2/g/common-crawl/c/3QmQjFA_3y4/m/vTbhGqIBBQAJ - "riddler", // Fails with 403: There might be too much traffic or a configuration error - "crtsh", // Fails in GH Action (possibly IP-based ban) causing a timeout. + "commoncrawl", // commoncrawl is under resourced and will likely time-out so step over it for this test https://groups.google.com/u/2/g/common-crawl/c/3QmQjFA_3y4/m/vTbhGqIBBQAJ + "riddler", // Fails with 403: There might be too much traffic or a configuration error + "crtsh", // Fails in GH Action (possibly IP-based ban) causing a timeout. + "hackertarget", // Fails in GH Action (possibly IP-based ban) but works locally } domain := "hackerone.com" From 4e64949dc4e05b3771d078c08d7c05f0b1e15312 Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Mon, 10 Jul 2023 17:55:59 +0200 Subject: [PATCH 15/19] attempting non breaking change --- v2/pkg/passive/passive.go | 29 +++++++++++++++++++++++------ v2/pkg/runner/enumerate.go | 3 ++- v2/pkg/runner/runner.go | 3 +-- v2/pkg/subscraping/types.go | 3 +-- 4 files changed, 27 insertions(+), 11 deletions(-) diff --git a/v2/pkg/passive/passive.go b/v2/pkg/passive/passive.go index f3c437495..199686bd4 100644 --- a/v2/pkg/passive/passive.go +++ b/v2/pkg/passive/passive.go @@ -13,19 +13,36 @@ import ( "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" ) +type EnumerationOptions struct { + customRateLimiter *subscraping.CustomRateLimit +} + +type EnumerateOption func(opts *EnumerationOptions) + +func WithCustomRateLimit(crl *subscraping.CustomRateLimit) EnumerateOption { + return func(opts *EnumerationOptions) { + opts.customRateLimiter = crl + } +} + // EnumerateSubdomains wraps EnumerateSubdomainsWithCtx with an empty context -func (a *Agent) EnumerateSubdomains(domain string, proxy string, rateLimit *subscraping.CustomRateLimit, timeout int, maxEnumTime time.Duration) chan subscraping.Result { - return a.EnumerateSubdomainsWithCtx(context.Background(), domain, proxy, rateLimit, timeout, maxEnumTime) +func (a *Agent) EnumerateSubdomains(domain string, proxy string, rateLimit int, timeout int, maxEnumTime time.Duration, options ...EnumerateOption) chan subscraping.Result { + return a.EnumerateSubdomainsWithCtx(context.Background(), domain, proxy, rateLimit, timeout, maxEnumTime, options...) } // EnumerateSubdomainsWithCtx enumerates all the subdomains for a given domain -func (a *Agent) EnumerateSubdomainsWithCtx(ctx context.Context, domain string, proxy string, rateLimit *subscraping.CustomRateLimit, timeout int, maxEnumTime time.Duration) chan subscraping.Result { +func (a *Agent) EnumerateSubdomainsWithCtx(ctx context.Context, domain string, proxy string, rateLimit int, timeout int, maxEnumTime time.Duration, options ...EnumerateOption) chan subscraping.Result { results := make(chan subscraping.Result) go func() { defer close(results) - multiRateLimiter, err := a.buildMultiRateLimiter(ctx, rateLimit) + var enumerateOptions EnumerationOptions + for _, enumerateOption := range options { + enumerateOption(&enumerateOptions) + } + + multiRateLimiter, err := a.buildMultiRateLimiter(ctx, rateLimit, enumerateOptions.customRateLimiter) if err != nil { results <- subscraping.Result{ Type: subscraping.Error, Error: fmt.Errorf("could not init multi rate limiter for %s: %s", domain, err), @@ -61,13 +78,13 @@ func (a *Agent) EnumerateSubdomainsWithCtx(ctx context.Context, domain string, p return results } -func (a *Agent) buildMultiRateLimiter(ctx context.Context, rateLimit *subscraping.CustomRateLimit) (*ratelimit.MultiLimiter, error) { +func (a *Agent) buildMultiRateLimiter(ctx context.Context, globalRateLimit int, rateLimit *subscraping.CustomRateLimit) (*ratelimit.MultiLimiter, error) { var multiRateLimiter *ratelimit.MultiLimiter var err error for _, source := range a.sources { var rl uint if sourceRateLimit, ok := rateLimit.Custom.Get(strings.ToLower(source.Name())); ok { - rl = sourceRateLimitOrDefault(rateLimit.Default, sourceRateLimit) + rl = sourceRateLimitOrDefault(uint(globalRateLimit), sourceRateLimit) } if rl > 0 { diff --git a/v2/pkg/runner/enumerate.go b/v2/pkg/runner/enumerate.go index ee54625d1..5ec1e0f0d 100644 --- a/v2/pkg/runner/enumerate.go +++ b/v2/pkg/runner/enumerate.go @@ -11,6 +11,7 @@ import ( "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/subfinder/v2/pkg/passive" "github.com/projectdiscovery/subfinder/v2/pkg/resolve" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" ) @@ -40,7 +41,7 @@ func (r *Runner) EnumerateSingleDomainWithCtx(ctx context.Context, domain string // Run the passive subdomain enumeration now := time.Now() - passiveResults := r.passiveAgent.EnumerateSubdomainsWithCtx(ctx, domain, r.options.Proxy, r.rateLimit, r.options.Timeout, time.Duration(r.options.MaxEnumerationTime)*time.Minute) + passiveResults := r.passiveAgent.EnumerateSubdomainsWithCtx(ctx, domain, r.options.Proxy, r.options.RateLimit, r.options.Timeout, time.Duration(r.options.MaxEnumerationTime)*time.Minute, passive.WithCustomRateLimit(r.rateLimit)) wg := &sync.WaitGroup{} wg.Add(1) diff --git a/v2/pkg/runner/runner.go b/v2/pkg/runner/runner.go index e226aee20..f5defdeec 100644 --- a/v2/pkg/runner/runner.go +++ b/v2/pkg/runner/runner.go @@ -56,9 +56,8 @@ func NewRunner(options *Options) (*Runner, error) { return nil, err } - // Initialize the rate limit jar + // Initialize the custom rate limit runner.rateLimit = &subscraping.CustomRateLimit{ - Default: uint(options.RateLimit), Custom: mapsutil.SyncLockMap[string, uint]{ Map: make(map[string]uint), }, diff --git a/v2/pkg/subscraping/types.go b/v2/pkg/subscraping/types.go index 89f12483c..3d54473b4 100644 --- a/v2/pkg/subscraping/types.go +++ b/v2/pkg/subscraping/types.go @@ -17,8 +17,7 @@ const ( ) type CustomRateLimit struct { - Default uint - Custom mapsutil.SyncLockMap[string, uint] + Custom mapsutil.SyncLockMap[string, uint] } // BasicAuth request's Authorization header From 3b165c66dc0ba30d021cdb4a12f71170a831d8f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fan=20Can=20Bak=C4=B1r?= Date: Tue, 25 Jul 2023 09:03:52 +0000 Subject: [PATCH 16/19] use ratelimit flag --- v2/go.mod | 2 +- v2/go.sum | 2 ++ v2/pkg/runner/options.go | 42 ++++++++++++++++++++++++++++------------ v2/pkg/runner/runner.go | 8 ++------ 4 files changed, 35 insertions(+), 19 deletions(-) diff --git a/v2/go.mod b/v2/go.mod index 3fe55ba39..502855e9a 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -83,7 +83,7 @@ require ( github.com/modern-go/reflect2 v1.0.2 // indirect github.com/pkg/errors v0.9.1 github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/projectdiscovery/goflags v0.1.11 + github.com/projectdiscovery/goflags v0.1.12 github.com/projectdiscovery/retryabledns v1.0.23 // indirect golang.org/x/net v0.12.0 // indirect golang.org/x/sys v0.10.0 // indirect diff --git a/v2/go.sum b/v2/go.sum index 801776459..e06058244 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -122,6 +122,8 @@ github.com/projectdiscovery/fdmax v0.0.4 h1:K9tIl5MUZrEMzjvwn/G4drsHms2aufTn1xUd github.com/projectdiscovery/fdmax v0.0.4/go.mod h1:oZLqbhMuJ5FmcoaalOm31B1P4Vka/CqP50nWjgtSz+I= github.com/projectdiscovery/goflags v0.1.11 h1:C4UTO3SM5Vfy1J2sdhukm7wONW/tljMpUMNKue5ie00= github.com/projectdiscovery/goflags v0.1.11/go.mod h1:wC5uJonjddDcCqDNfPq+03nRessSB/LLaaIea4w47ws= +github.com/projectdiscovery/goflags v0.1.12 h1:NucjSqw7reczmon2vQq9KyOrvOmlnznECeifHI2gOW0= +github.com/projectdiscovery/goflags v0.1.12/go.mod h1:wC5uJonjddDcCqDNfPq+03nRessSB/LLaaIea4w47ws= github.com/projectdiscovery/gologger v1.1.11 h1:8vsz9oJlDT9euw6xlj7F7dZ6RWItVIqVwn4Mr6uzky8= github.com/projectdiscovery/gologger v1.1.11/go.mod h1:UR2bgXl7zraOxYGnUwuO917hifWrwMJ0feKnVqMQkzY= github.com/projectdiscovery/ratelimit v0.0.9 h1:28t2xDHUnyss1irzqPG3Oxz5hkRjl+3Q2I/aes7nau8= diff --git a/v2/pkg/runner/options.go b/v2/pkg/runner/options.go index 8d23c18dc..8372327ab 100644 --- a/v2/pkg/runner/options.go +++ b/v2/pkg/runner/options.go @@ -4,6 +4,7 @@ import ( "errors" "fmt" "io" + "math" "math/rand" "os" "os/user" @@ -50,17 +51,17 @@ type Options struct { Domain goflags.StringSlice // Domain is the domain to find subdomains for DomainsFile string // DomainsFile is the file containing list of domains to find subdomains for Output io.Writer - OutputFile string // Output is the file to write found subdomains to. - OutputDirectory string // OutputDirectory is the directory to write results to in case list of domains is given - Sources goflags.StringSlice `yaml:"sources,omitempty"` // Sources contains a comma-separated list of sources to use for enumeration - ExcludeSources goflags.StringSlice `yaml:"exclude-sources,omitempty"` // ExcludeSources contains the comma-separated sources to not include in the enumeration process - Resolvers goflags.StringSlice `yaml:"resolvers,omitempty"` // Resolvers is the comma-separated resolvers to use for enumeration - ResolverList string // ResolverList is a text file containing list of resolvers to use for enumeration - Config string // Config contains the location of the config file - ProviderConfig string // ProviderConfig contains the location of the provider config file - Proxy string // HTTP proxy - RateLimit int // Global maximum number of HTTP requests to send per second - RateLimits goflags.RuntimeMap // Maximum number of HTTP requests to send per second + OutputFile string // Output is the file to write found subdomains to. + OutputDirectory string // OutputDirectory is the directory to write results to in case list of domains is given + Sources goflags.StringSlice `yaml:"sources,omitempty"` // Sources contains a comma-separated list of sources to use for enumeration + ExcludeSources goflags.StringSlice `yaml:"exclude-sources,omitempty"` // ExcludeSources contains the comma-separated sources to not include in the enumeration process + Resolvers goflags.StringSlice `yaml:"resolvers,omitempty"` // Resolvers is the comma-separated resolvers to use for enumeration + ResolverList string // ResolverList is a text file containing list of resolvers to use for enumeration + Config string // Config contains the location of the config file + ProviderConfig string // ProviderConfig contains the location of the provider config file + Proxy string // HTTP proxy + RateLimit int // Global maximum number of HTTP requests to send per second + RateLimits goflags.RateLimitMap // Maximum number of HTTP requests to send per second ExcludeIps bool Match goflags.StringSlice Filter goflags.StringSlice @@ -116,7 +117,7 @@ func ParseOptions() *Options { flagSet.CreateGroup("rate-limit", "Rate-limit", flagSet.IntVarP(&options.RateLimit, "rate-limit", "rl", 0, "maximum number of http requests to send per second (global)"), - flagSet.RuntimeMapVar(&options.RateLimits, "rls", nil, "maximum number of http requests to send per second four providers in key=value format (-rls hackertarget=10)"), + flagSet.RateLimitMapVarP(&options.RateLimits, "rate-limits", "rls", defaultRateLimits(), "maximum number of http requests to send per second four providers in key=value format (-rls hackertarget=10/m)", goflags.NormalizedStringSliceOptions), flagSet.IntVar(&options.Threads, "t", 10, "number of concurrent goroutines for resolving (-active only)"), ) @@ -300,3 +301,20 @@ func userHomeDir() string { } return usr.HomeDir } + +func defaultRateLimits() []string { + return []string{ + "github=30/m", + "gitlab=2000/m", + "fullhunt=60/m", + fmt.Sprintf("robotex=%d/ms", uint(math.MaxUint)), + "securitytrails=1/s", + "shodan=1/s", + "virustotal=4/m", + "hackertarget=2/s", + "threatminer=10/m", + "waybackarchive=15/m", + "whoisxmlapi=50/s", + } + +} diff --git a/v2/pkg/runner/runner.go b/v2/pkg/runner/runner.go index f5defdeec..d9a6fe28e 100644 --- a/v2/pkg/runner/runner.go +++ b/v2/pkg/runner/runner.go @@ -8,7 +8,6 @@ import ( "os" "path" "regexp" - "strconv" "strings" "github.com/pkg/errors" @@ -64,11 +63,8 @@ func NewRunner(options *Options) (*Runner, error) { } for source, sourceRateLimit := range options.RateLimits.AsMap() { - if sourceRateLimitStr, ok := sourceRateLimit.(string); ok { - sourceRateLimitUint, err := strconv.ParseUint(sourceRateLimitStr, 10, 64) - if err == nil && sourceRateLimitUint > 0 && sourceRateLimitUint <= math.MaxUint32 { - _ = runner.rateLimit.Custom.Set(source, uint(sourceRateLimitUint)) - } + if sourceRateLimit.MaxCount > 0 && sourceRateLimit.MaxCount <= math.MaxUint { + _ = runner.rateLimit.Custom.Set(source, sourceRateLimit.MaxCount) } } From d161f3fb6a25a26ce3d3b2c629ac53253b2446d7 Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Fri, 28 Jul 2023 17:58:18 +0200 Subject: [PATCH 17/19] some code cleanup --- .github/workflows/build-test.yml | 2 +- .github/workflows/lint-test.yml | 2 +- .github/workflows/release-binary.yml | 2 +- README.md | 2 +- v2/go.mod | 4 ++-- v2/pkg/runner/options.go | 33 +++++++++++----------------- v2/pkg/subscraping/utils.go | 10 --------- 7 files changed, 19 insertions(+), 36 deletions(-) diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index 1cfdd475b..aa01cc015 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -18,7 +18,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v4 with: - go-version: 1.19 + go-version: 1.20.x - name: Check out code uses: actions/checkout@v3 diff --git a/.github/workflows/lint-test.yml b/.github/workflows/lint-test.yml index 111be711c..dbe1ce1ad 100644 --- a/.github/workflows/lint-test.yml +++ b/.github/workflows/lint-test.yml @@ -15,7 +15,7 @@ jobs: - uses: actions/checkout@v3 - uses: actions/setup-go@v4 with: - go-version: 1.19 + go-version: 1.20.x - name: Run golangci-lint uses: golangci/golangci-lint-action@v3.6.0 with: diff --git a/.github/workflows/release-binary.yml b/.github/workflows/release-binary.yml index fad8e5d76..f0936c4dd 100644 --- a/.github/workflows/release-binary.yml +++ b/.github/workflows/release-binary.yml @@ -18,7 +18,7 @@ jobs: - name: "Set up Go" uses: actions/setup-go@v4 with: - go-version: 1.19 + go-version: 1.20.x - name: "Create release on GitHub" uses: goreleaser/goreleaser-action@v3 diff --git a/README.md b/README.md index a9ca01916..94e96f73d 100644 --- a/README.md +++ b/README.md @@ -111,7 +111,7 @@ OPTIMIZATION: # Installation -`subfinder` requires **go1.19** to install successfully. Run the following command to install the latest version: +`subfinder` requires **go1.20** to install successfully. Run the following command to install the latest version: ```sh go install -v github.com/projectdiscovery/subfinder/v2/cmd/subfinder@latest diff --git a/v2/go.mod b/v2/go.mod index 467560fb0..918e7e2a8 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -1,6 +1,6 @@ module github.com/projectdiscovery/subfinder/v2 -go 1.18 +go 1.20 require ( github.com/corpix/uarand v0.2.0 @@ -12,6 +12,7 @@ require ( github.com/projectdiscovery/fdmax v0.0.4 github.com/projectdiscovery/gologger v1.1.11 github.com/projectdiscovery/ratelimit v0.0.9 + github.com/projectdiscovery/retryablehttp-go v1.0.15 github.com/projectdiscovery/utils v0.0.44 github.com/rs/xid v1.5.0 github.com/stretchr/testify v1.8.4 @@ -54,7 +55,6 @@ require ( github.com/pierrec/lz4 v2.6.1+incompatible // indirect github.com/projectdiscovery/blackrock v0.0.1 // indirect github.com/projectdiscovery/cdncheck v1.0.1 // indirect - github.com/projectdiscovery/retryablehttp-go v1.0.15 // indirect github.com/rivo/uniseg v0.4.4 // indirect github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d // indirect github.com/ulikunitz/xz v0.5.11 // indirect diff --git a/v2/pkg/runner/options.go b/v2/pkg/runner/options.go index 8372327ab..5f83189f2 100644 --- a/v2/pkg/runner/options.go +++ b/v2/pkg/runner/options.go @@ -5,13 +5,11 @@ import ( "fmt" "io" "math" - "math/rand" "os" "os/user" "path/filepath" "regexp" "strings" - "time" "gopkg.in/yaml.v3" @@ -77,8 +75,6 @@ type OnResultCallback func(result *resolve.HostEntry) // ParseOptions parses the command line flags provided by a user func ParseOptions() *Options { logutil.DisableDefaultLogger() - // Seed default random number generator - rand.Seed(time.Now().UnixNano()) // Migrate config to provider config if fileutil.FileExists(defaultConfigLocation) && !fileutil.FileExists(defaultProviderConfigLocation) { @@ -117,7 +113,7 @@ func ParseOptions() *Options { flagSet.CreateGroup("rate-limit", "Rate-limit", flagSet.IntVarP(&options.RateLimit, "rate-limit", "rl", 0, "maximum number of http requests to send per second (global)"), - flagSet.RateLimitMapVarP(&options.RateLimits, "rate-limits", "rls", defaultRateLimits(), "maximum number of http requests to send per second four providers in key=value format (-rls hackertarget=10/m)", goflags.NormalizedStringSliceOptions), + flagSet.RateLimitMapVarP(&options.RateLimits, "rate-limits", "rls", defaultRateLimits, "maximum number of http requests to send per second four providers in key=value format (-rls hackertarget=10/m)", goflags.NormalizedStringSliceOptions), flagSet.IntVar(&options.Threads, "t", 10, "number of concurrent goroutines for resolving (-active only)"), ) @@ -302,19 +298,16 @@ func userHomeDir() string { return usr.HomeDir } -func defaultRateLimits() []string { - return []string{ - "github=30/m", - "gitlab=2000/m", - "fullhunt=60/m", - fmt.Sprintf("robotex=%d/ms", uint(math.MaxUint)), - "securitytrails=1/s", - "shodan=1/s", - "virustotal=4/m", - "hackertarget=2/s", - "threatminer=10/m", - "waybackarchive=15/m", - "whoisxmlapi=50/s", - } - +var defaultRateLimits = []string{ + "github=30/m", + "gitlab=2000/m", + "fullhunt=60/m", + fmt.Sprintf("robotex=%d/ms", uint(math.MaxUint)), + "securitytrails=1/s", + "shodan=1/s", + "virustotal=4/m", + "hackertarget=2/s", + "threatminer=10/m", + "waybackarchive=15/m", + "whoisxmlapi=50/s", } diff --git a/v2/pkg/subscraping/utils.go b/v2/pkg/subscraping/utils.go index 7b2b6e4c6..cd119a8c6 100644 --- a/v2/pkg/subscraping/utils.go +++ b/v2/pkg/subscraping/utils.go @@ -4,25 +4,15 @@ import ( "math/rand" "regexp" "strings" - "sync" - "time" "github.com/projectdiscovery/gologger" ) const MultipleKeyPartsLength = 2 -var subdomainExtractorMutex = &sync.Mutex{} - -func init() { - rand.Seed(time.Now().UnixNano()) -} - // NewSubdomainExtractor creates a new regular expression to extract // subdomains from text based on the given domain. func NewSubdomainExtractor(domain string) (*regexp.Regexp, error) { - subdomainExtractorMutex.Lock() - defer subdomainExtractorMutex.Unlock() extractor, err := regexp.Compile(`(?i)[a-zA-Z0-9\*_.-]+\.` + domain) if err != nil { return nil, err From e95078daf8a14ee48bce14c8998f57446eb14d5a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fan=20Can=20Bak=C4=B1r?= Date: Mon, 31 Jul 2023 08:38:34 +0000 Subject: [PATCH 18/19] validate keys --- v2/pkg/runner/options.go | 6 +++--- v2/pkg/runner/validate.go | 10 ++++++++++ 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/v2/pkg/runner/options.go b/v2/pkg/runner/options.go index 5f83189f2..570b218b5 100644 --- a/v2/pkg/runner/options.go +++ b/v2/pkg/runner/options.go @@ -300,14 +300,14 @@ func userHomeDir() string { var defaultRateLimits = []string{ "github=30/m", - "gitlab=2000/m", + // "gitlab=2000/m", "fullhunt=60/m", - fmt.Sprintf("robotex=%d/ms", uint(math.MaxUint)), + fmt.Sprintf("robtex=%d/ms", uint(math.MaxUint)), "securitytrails=1/s", "shodan=1/s", "virustotal=4/m", "hackertarget=2/s", - "threatminer=10/m", + // "threatminer=10/m", "waybackarchive=15/m", "whoisxmlapi=50/s", } diff --git a/v2/pkg/runner/validate.go b/v2/pkg/runner/validate.go index e8010ccb6..20189f18d 100644 --- a/v2/pkg/runner/validate.go +++ b/v2/pkg/runner/validate.go @@ -9,6 +9,9 @@ import ( "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/gologger/formatter" "github.com/projectdiscovery/gologger/levels" + "github.com/projectdiscovery/subfinder/v2/pkg/passive" + mapsutil "github.com/projectdiscovery/utils/maps" + sliceutil "github.com/projectdiscovery/utils/slice" ) // validateOptions validates the configuration options passed @@ -55,6 +58,13 @@ func (options *Options) validateOptions() error { } } } + + sources := mapsutil.GetKeys(passive.NameSourceMap) + for source := range options.RateLimits.AsMap() { + if !sliceutil.Contains(sources, source) { + return fmt.Errorf("invalid source %s specified in -rls flag", source) + } + } return nil } func stripRegexString(val string) string { From 3a2ce218c2d066c84d7b3b84ca0dd63f802a2cf5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fan=20Can=20Bak=C4=B1r?= Date: Mon, 31 Jul 2023 08:39:56 +0000 Subject: [PATCH 19/19] update README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 94e96f73d..88bebf31e 100644 --- a/README.md +++ b/README.md @@ -74,7 +74,7 @@ FILTER: RATE-LIMIT: -rl, -rate-limit int maximum number of http requests to send per second - -rls value maximum number of http requests to send per second four providers in key=value format (-rls hackertarget=10) + -rls value maximum number of http requests to send per second four providers in key=value format (-rls "hackertarget=10/s,shodan=15/s") -t int number of concurrent goroutines for resolving (-active only) (default 10) UPDATE: