diff --git a/v2/go.mod b/v2/go.mod index d42488f9d..a423bfce7 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -120,7 +120,7 @@ require ( github.com/miekg/dns v1.1.56 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect - github.com/pkg/errors v0.9.1 + github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/projectdiscovery/goflags v0.1.63 github.com/projectdiscovery/retryabledns v1.0.74 // indirect diff --git a/v2/pkg/passive/sources.go b/v2/pkg/passive/sources.go index a72122292..db569fdf6 100644 --- a/v2/pkg/passive/sources.go +++ b/v2/pkg/passive/sources.go @@ -1,7 +1,6 @@ package passive import ( - "fmt" "strings" "golang.org/x/exp/maps" @@ -157,7 +156,7 @@ func New(sourceNames, excludedSourceNames []string, useAllSources, useSourcesSup } } - gologger.Debug().Msgf(fmt.Sprintf("Selected source(s) for this search: %s", strings.Join(maps.Keys(sources), ", "))) + gologger.Debug().Msgf("Selected source(s) for this search: %s", strings.Join(maps.Keys(sources), ", ")) for _, currentSource := range sources { if warning, ok := sourceWarnings.Get(strings.ToLower(currentSource.Name())); ok { diff --git a/v2/pkg/runner/options.go b/v2/pkg/runner/options.go index 676d78e7b..32562cf17 100644 --- a/v2/pkg/runner/options.go +++ b/v2/pkg/runner/options.go @@ -215,7 +215,7 @@ func (options *Options) loadProvidersFrom(location string) { // We skip bailing out if file doesn't exist because we'll create it // at the end of options parsing from default via goflags. - if err := UnmarshalFrom(location); err != nil && (!strings.Contains(err.Error(), "file doesn't exist") || errors.Is(os.ErrNotExist, err)) { + if err := UnmarshalFrom(location); err != nil && (!strings.Contains(err.Error(), "file doesn't exist") || errors.Is(err, os.ErrNotExist)) { gologger.Error().Msgf("Could not read providers from %s: %s\n", location, err) } } @@ -237,7 +237,7 @@ func listSources(options *Options) { func (options *Options) preProcessDomains() { for i, domain := range options.Domain { - options.Domain[i], _ = sanitize(domain) + options.Domain[i] = preprocessDomain(domain) } } diff --git a/v2/pkg/runner/runner.go b/v2/pkg/runner/runner.go index b66b68537..e5cb3740b 100644 --- a/v2/pkg/runner/runner.go +++ b/v2/pkg/runner/runner.go @@ -11,8 +11,6 @@ import ( "strconv" "strings" - "github.com/pkg/errors" - "github.com/projectdiscovery/gologger" contextutil "github.com/projectdiscovery/utils/context" fileutil "github.com/projectdiscovery/utils/file" @@ -116,12 +114,13 @@ func (r *Runner) EnumerateMultipleDomains(reader io.Reader, writers []io.Writer) // EnumerateMultipleDomainsWithCtx enumerates subdomains for multiple domains // We keep enumerating subdomains for a given domain until we reach an error func (r *Runner) EnumerateMultipleDomainsWithCtx(ctx context.Context, reader io.Reader, writers []io.Writer) error { + var err error scanner := bufio.NewScanner(reader) ip, _ := regexp.Compile(`^([0-9\.]+$)`) for scanner.Scan() { - domain, err := normalizeLowercase(scanner.Text()) - isIp := ip.MatchString(domain) - if errors.Is(err, ErrEmptyInput) || (r.options.ExcludeIps && isIp) { + domain := preprocessDomain(scanner.Text()) + + if domain == "" || (r.options.ExcludeIps && ip.MatchString(domain)) { continue } diff --git a/v2/pkg/runner/stats.go b/v2/pkg/runner/stats.go index 4d9ee8f23..55f497a0a 100644 --- a/v2/pkg/runner/stats.go +++ b/v2/pkg/runner/stats.go @@ -30,14 +30,12 @@ func printStatistics(stats map[string]subscraping.Statistics) { if len(lines) > 0 { gologger.Print().Msgf("\n Source Duration Results Errors\n%s\n", strings.Repeat("─", 56)) - gologger.Print().Msgf(strings.Join(lines, "\n")) - gologger.Print().Msgf("\n") + gologger.Print().Msgf("%s\n", strings.Join(lines, "\n")) } if len(skipped) > 0 { gologger.Print().Msgf("\n The following sources were included but skipped...\n\n") - gologger.Print().Msgf(strings.Join(skipped, "\n")) - gologger.Print().Msgf("\n\n") + gologger.Print().Msgf("%s\n\n", strings.Join(skipped, "\n")) } } diff --git a/v2/pkg/runner/util.go b/v2/pkg/runner/util.go index cc883cc82..f0458c738 100644 --- a/v2/pkg/runner/util.go +++ b/v2/pkg/runner/util.go @@ -1,15 +1,8 @@ package runner import ( - "strings" - - "github.com/pkg/errors" - fileutil "github.com/projectdiscovery/utils/file" -) - -var ( - ErrEmptyInput = errors.New("empty data") + stringsutil "github.com/projectdiscovery/utils/strings" ) func loadFromFile(file string) ([]string, error) { @@ -19,9 +12,8 @@ func loadFromFile(file string) ([]string, error) { } var items []string for item := range chanItems { - var err error - item, err = sanitize(item) - if errors.Is(err, ErrEmptyInput) { + item = preprocessDomain(item) + if item == "" { continue } items = append(items, item) @@ -29,15 +21,12 @@ func loadFromFile(file string) ([]string, error) { return items, nil } -func sanitize(data string) (string, error) { - data = strings.Trim(data, "\n\t\"'` ") - if data == "" { - return "", ErrEmptyInput - } - return data, nil -} - -func normalizeLowercase(s string) (string, error) { - data, err := sanitize(s) - return strings.ToLower(data), err +func preprocessDomain(s string) string { + return stringsutil.NormalizeWithOptions(s, + stringsutil.NormalizeOptions{ + StripComments: true, + TrimCutset: "\n\t\"'` ", + Lowercase: true, + }, + ) } diff --git a/v2/pkg/subscraping/sources/binaryedge/binaryedge.go b/v2/pkg/subscraping/sources/binaryedge/binaryedge.go index cc6b569a8..7684ff602 100644 --- a/v2/pkg/subscraping/sources/binaryedge/binaryedge.go +++ b/v2/pkg/subscraping/sources/binaryedge/binaryedge.go @@ -119,7 +119,7 @@ func (s *Source) enumerate(ctx context.Context, session *subscraping.Session, ba // Check error messages if response.Message != "" && response.Status != nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf(response.Message)} + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("%s", response.Message)} s.errors++ return }