projectdiscovery · ehsandeep · Apr 4, 2022 · Mar 6, 2022 · Mar 6, 2022 · Mar 6, 2022
diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml
@@ -18,7 +18,7 @@ jobs:
           go-version: 1.17
 
       - name: Check out code
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
 
       - name: Build
         run: go build ./...

diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml
@@ -23,7 +23,7 @@ jobs:
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v2
+      uses: actions/checkout@v3
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL

diff --git a/.github/workflows/dockerhub-push.yml b/.github/workflows/dockerhub-push.yml
@@ -13,7 +13,7 @@ jobs:
     steps:
 
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
 
       - name: Get Github tag
         id: meta

diff --git a/.github/workflows/lint-test.yml b/.github/workflows/lint-test.yml
@@ -9,7 +9,7 @@ jobs:
     name: Lint Test
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - uses: actions/setup-go@v2
         with: 
           go-version: 1.17

diff --git a/.github/workflows/release-binary.yml b/.github/workflows/release-binary.yml
@@ -9,7 +9,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: "Check out code"
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with: 
           fetch-depth: 0
 

diff --git a/.github/workflows/sonarcloud.yml b/.github/workflows/sonarcloud.yml
@@ -13,7 +13,7 @@ jobs:
     name: SonarCloud
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
         with:
           fetch-depth: 0  # Shallow clones should be disabled for a better relevancy of analysis
 

diff --git a/Dockerfile b/Dockerfile
@@ -1,9 +1,9 @@
 # Build
-FROM golang:1.17-alpine AS build-env
+FROM golang:1.18.0-alpine AS build-env
 RUN go install -v github.com/projectdiscovery/subfinder/v2/cmd/subfinder@latest
 
 # Release
-FROM alpine:3.15.0
+FROM alpine:3.15.3
 RUN apk -U upgrade --no-cache \
     && apk add --no-cache bind-tools ca-certificates
 COPY --from=build-env /go/bin/subfinder /usr/local/bin/subfinder

diff --git a/README.md b/README.md
@@ -242,23 +242,16 @@ import (
 )
 
 func main() {
-	config := runner.ConfigFile{
-		// Use the default list of resolvers by marshaling it to the config
-		Resolvers: resolve.DefaultResolvers,
-		// Use the default list of passive sources
-		Sources: passive.DefaultSources,
-		// Use the default list of all passive sources
-		AllSources: passive.DefaultAllSources,
-		// Use the default list of recursive sources
-		Recursive: passive.DefaultRecursiveSources,
-	}
-
 	runnerInstance, err := runner.NewRunner(&runner.Options{
 		Threads:            10, // Thread controls the number of threads to use for active enumerations
 		Timeout:            30, // Timeout is the seconds to wait for sources to respond
 		MaxEnumerationTime: 10, // MaxEnumerationTime is the maximum amount of time in mins to wait for enumeration
-		YAMLConfig:         config,
-	})
+		Resolvers:          resolve.DefaultResolvers, // Use the default list of resolvers by marshaling it to the config
+		Sources:            passive.DefaultSources, // Use the default list of passive sources
+		AllSources:         passive.DefaultAllSources, // Use the default list of all passive sources
+		Recursive:          passive.DefaultRecursiveSources,	// Use the default list of recursive sources
+		Providers:          &runner.Providers{}, // Use empty api keys for all providers
+  })
 
 	buf := bytes.Buffer{}
 	err = runnerInstance.EnumerateSingleDomain(context.Background(), "projectdiscovery.io", []io.Writer{&buf})

diff --git a/v2/go.mod b/v2/go.mod
@@ -33,7 +33,7 @@ require (
 	github.com/modern-go/reflect2 v1.0.1 // indirect
 	github.com/pkg/errors v0.9.1
 	github.com/pmezard/go-difflib v1.0.0 // indirect
-	github.com/projectdiscovery/goflags v0.0.8-0.20220208063718-9bbeacc2fb8f
+	github.com/projectdiscovery/goflags v0.0.8-0.20220328195035-cc76049ee216
 	github.com/projectdiscovery/retryabledns v1.0.12-0.20210419174848-eec3ac17d61e // indirect
 	github.com/projectdiscovery/stringsutil v0.0.0-20210804142656-fd3c28dbaafe // indirect
 	golang.org/x/net v0.0.0-20210415231046-e915ea6b2b7d // indirect

diff --git a/v2/go.sum b/v2/go.sum
@@ -97,6 +97,8 @@ github.com/projectdiscovery/fileutil v0.0.0-20210928100737-cab279c5d4b5 h1:2dbm7
 github.com/projectdiscovery/fileutil v0.0.0-20210928100737-cab279c5d4b5/go.mod h1:U+QCpQnX8o2N2w0VUGyAzjM3yBAe4BKedVElxiImsx0=
 github.com/projectdiscovery/goflags v0.0.8-0.20220208063718-9bbeacc2fb8f h1:FKTkdM1pPIL0gQRRQDoWjd/mZz+4DZ2Bk1l+ZbOJmIQ=
 github.com/projectdiscovery/goflags v0.0.8-0.20220208063718-9bbeacc2fb8f/go.mod h1:Jjwsf4eEBPXDSQI2Y+6fd3dBumJv/J1U0nmpM+hy2YY=
+github.com/projectdiscovery/goflags v0.0.8-0.20220328195035-cc76049ee216 h1:Th8GrVWt6LJQAwneHikni4BpoLEPa69DPEnYCxAINmo=
+github.com/projectdiscovery/goflags v0.0.8-0.20220328195035-cc76049ee216/go.mod h1:37KhVbVLllyuIAgpXGqcvE/hsFEwJ+ctEUSHawjhsBY=
 github.com/projectdiscovery/gologger v1.0.0/go.mod h1:Ok+axMqK53bWNwDSU1nTNwITLYMXMdZtRc8/y1c7sWE=
 github.com/projectdiscovery/gologger v1.1.4 h1:qWxGUq7ukHWT849uGPkagPKF3yBPYAsTtMKunQ8O2VI=
 github.com/projectdiscovery/gologger v1.1.4/go.mod h1:Bhb6Bdx2PV1nMaFLoXNBmHIU85iROS9y1tBuv7T5pMY=

diff --git a/v2/pkg/resolve/resolve.go b/v2/pkg/resolve/resolve.go
@@ -25,8 +25,8 @@ type ResolutionPool struct {
 
 // HostEntry defines a host with the source
 type HostEntry struct {
-	Host   string `json:"host"`
-	Source string `json:"source"`
+	Host   string
+	Source string
 }
 
 // Result contains the result for a host resolution

diff --git a/v2/pkg/runner/banners.go b/v2/pkg/runner/banners.go
@@ -1,6 +1,9 @@
 package runner
 
 import (
+	"errors"
+	"os"
+
 	"github.com/projectdiscovery/gologger"
 	"github.com/projectdiscovery/subfinder/v2/pkg/passive"
 	"github.com/projectdiscovery/subfinder/v2/pkg/resolve"
@@ -11,11 +14,11 @@ const banner = `
    _______  __/ /_  / __(_)___  ____/ /__  _____
   / ___/ / / / __ \/ /_/ / __ \/ __  / _ \/ ___/
  (__  ) /_/ / /_/ / __/ / / / / /_/ /  __/ /    
-/____/\__,_/_.___/_/ /_/_/ /_/\__,_/\___/_/ v2.5.0
+/____/\__,_/_.___/_/ /_/_/ /_/\__,_/\___/_/ v2.5.1
 `
 
 // Version is the current version of subfinder
-const Version = `v2.5.0`
+const Version = `v2.5.1`
 
 // showBanner is used to show the banner to the user
 func showBanner() {
@@ -44,7 +47,9 @@ func (options *Options) loadProvidersFrom(location string) {
 	}
 
 	options.Providers = &Providers{}
-	if err := options.Providers.UnmarshalFrom(location); isFatalErr(err) {
+	// We skip bailing out if file doesn't exist because we'll create it
+	// at the end of options parsing from default via goflags.
+	if err := options.Providers.UnmarshalFrom(location); isFatalErr(err) && !errors.Is(err, os.ErrNotExist) {
 		gologger.Fatal().Msgf("Could not read providers from %s: %s\n", location, err)
 	}
 }
diff --git a/v2/pkg/runner/enumerate.go b/v2/pkg/runner/enumerate.go
@@ -119,15 +119,15 @@ func (r *Runner) EnumerateSingleDomain(ctx context.Context, domain string, outpu
 	var err error
 	for _, w := range outputs {
 		if r.options.HostIP {
-			err = outputter.WriteHostIP(foundResults, w)
+			err = outputter.WriteHostIP(domain, foundResults, w)
 		} else {
 			if r.options.RemoveWildcard {
-				err = outputter.WriteHostNoWildcard(foundResults, w)
+				err = outputter.WriteHostNoWildcard(domain, foundResults, w)
 			} else {
 				if r.options.CaptureSources {
-					err = outputter.WriteSourceHost(sourceMap, w)
+					err = outputter.WriteSourceHost(domain, sourceMap, w)
 				} else {
-					err = outputter.WriteHost(uniqueMap, w)
+					err = outputter.WriteHost(domain, uniqueMap, w)
 				}
 			}
 		}

diff --git a/v2/pkg/runner/options.go b/v2/pkg/runner/options.go
@@ -60,18 +60,17 @@ type Options struct {
 	Proxy          string                        // HTTP proxy
 	RateLimit      int                           // Maximum number of HTTP requests to send per second
 	// YAMLConfig contains the unmarshalled yaml config file
-	Providers *Providers
+	Providers  *Providers
+	ExcludeIps bool
 }
 
 // ParseOptions parses the command line flags provided by a user
 func ParseOptions() *Options {
-	showBanner()
-
 	// Migrate config to provider config
 	if fileutil.FileExists(defaultConfigLocation) && !fileutil.FileExists(defaultProviderConfigLocation) {
 		gologger.Info().Msgf("Detected old %s config file, trying to migrate providers to %s\n", defaultConfigLocation, defaultProviderConfigLocation)
 		if err := migrateToProviderConfig(defaultConfigLocation, defaultProviderConfigLocation); err != nil {
-			gologger.Fatal().Msgf("Could not migrate providers from existing config (%s) to provider config (%s): %s\n", defaultConfigLocation, defaultProviderConfigLocation, err)
+			gologger.Warning().Msgf("Could not migrate providers from existing config (%s) to provider config (%s): %s\n", defaultConfigLocation, defaultProviderConfigLocation, err)
 		} else {
 			//cleanup the existing config file post migration
 			os.Remove(defaultConfigLocation)
@@ -80,6 +79,7 @@ func ParseOptions() *Options {
 	}
 
 	options := &Options{}
+
 	var err error
 	flagSet := goflags.NewFlagSet()
 	flagSet.SetDescription(`Subfinder is a subdomain discovery tool that discovers subdomains for websites by using passive online sources.`)
@@ -116,6 +116,7 @@ func ParseOptions() *Options {
 		flagSet.StringVarP(&options.ResolverList, "rlist", "rL", "", "file containing list of resolvers to use"),
 		flagSet.BoolVarP(&options.RemoveWildcard, "active", "nW", false, "display active subdomains only"),
 		flagSet.StringVar(&options.Proxy, "proxy", "", "http proxy to use with subfinder"),
+		flagSet.BoolVarP(&options.ExcludeIps, "exclude-ip", "ei", false, "Exclude ips from the list of domains"),
 	)
 
 	createGroup(flagSet, "debug", "Debug",
@@ -137,7 +138,8 @@ func ParseOptions() *Options {
 	}
 
 	if options.Config != defaultConfigLocation {
-		if err := flagSet.MergeConfigFile(options.Config); err != nil {
+		// An empty source file is not a fatal error
+		if err := flagSet.MergeConfigFile(options.Config); err != nil && !errors.Is(err, io.EOF) {
 			gologger.Fatal().Msgf("Could not read config: %s\n", err)
 		}
 	}
@@ -172,6 +174,10 @@ func ParseOptions() *Options {
 
 	options.preProcessOptions()
 
+	if !options.Silent {
+		showBanner()
+	}
+
 	// Validate the options passed by the user and if any
 	// invalid options have been used, exit.
 	err = options.validateOptions()

diff --git a/v2/pkg/runner/outputter.go b/v2/pkg/runner/outputter.go
@@ -17,14 +17,22 @@ type OutPutter struct {
 	JSON bool
 }
 
-type jsonResult struct {
+type jsonSourceResult struct {
+	Host   string `json:"host"`
+	Input  string `json:"input"`
+	Source string `json:"source"`
+}
+
+type jsonSourceIPResult struct {
 	Host   string `json:"host"`
 	IP     string `json:"ip"`
+	Input  string `json:"input"`
 	Source string `json:"source"`
 }
 
-type jsonSourceResult struct {
+type jsonSourcesResult struct {
 	Host    string   `json:"host"`
+	Input   string   `json:"input"`
 	Sources []string `json:"sources"`
 }
 
@@ -64,17 +72,17 @@ func (o *OutPutter) createFile(filename string, appendtoFile bool) (*os.File, er
 }
 
 // WriteHostIP writes the output list of subdomain to an io.Writer
-func (o *OutPutter) WriteHostIP(results map[string]resolve.Result, writer io.Writer) error {
+func (o *OutPutter) WriteHostIP(input string, results map[string]resolve.Result, writer io.Writer) error {
 	var err error
 	if o.JSON {
-		err = writeJSONHostIP(results, writer)
+		err = writeJSONHostIP(input, results, writer)
 	} else {
-		err = writePlainHostIP(results, writer)
+		err = writePlainHostIP(input, results, writer)
 	}
 	return err
 }
 
-func writePlainHostIP(results map[string]resolve.Result, writer io.Writer) error {
+func writePlainHostIP(_ string, results map[string]resolve.Result, writer io.Writer) error {
 	bufwriter := bufio.NewWriter(writer)
 	sb := &strings.Builder{}
 
@@ -96,14 +104,15 @@ func writePlainHostIP(results map[string]resolve.Result, writer io.Writer) error
 	return bufwriter.Flush()
 }
 
-func writeJSONHostIP(results map[string]resolve.Result, writer io.Writer) error {
+func writeJSONHostIP(input string, results map[string]resolve.Result, writer io.Writer) error {
 	encoder := jsoniter.NewEncoder(writer)
 
-	var data jsonResult
+	var data jsonSourceIPResult
 
 	for _, result := range results {
 		data.Host = result.Host
 		data.IP = result.IP
+		data.Input = input
 		data.Source = result.Source
 
 		err := encoder.Encode(&data)
@@ -115,27 +124,27 @@ func writeJSONHostIP(results map[string]resolve.Result, writer io.Writer) error
 }
 
 // WriteHostNoWildcard writes the output list of subdomain with nW flag to an io.Writer
-func (o *OutPutter) WriteHostNoWildcard(results map[string]resolve.Result, writer io.Writer) error {
+func (o *OutPutter) WriteHostNoWildcard(input string, results map[string]resolve.Result, writer io.Writer) error {
 	hosts := make(map[string]resolve.HostEntry)
 	for host, result := range results {
 		hosts[host] = resolve.HostEntry{Host: result.Host, Source: result.Source}
 	}
 
-	return o.WriteHost(hosts, writer)
+	return o.WriteHost(input, hosts, writer)
 }
 
 // WriteHost writes the output list of subdomain to an io.Writer
-func (o *OutPutter) WriteHost(results map[string]resolve.HostEntry, writer io.Writer) error {
+func (o *OutPutter) WriteHost(input string, results map[string]resolve.HostEntry, writer io.Writer) error {
 	var err error
 	if o.JSON {
-		err = writeJSONHost(results, writer)
+		err = writeJSONHost(input, results, writer)
 	} else {
-		err = writePlainHost(results, writer)
+		err = writePlainHost(input, results, writer)
 	}
 	return err
 }
 
-func writePlainHost(results map[string]resolve.HostEntry, writer io.Writer) error {
+func writePlainHost(_ string, results map[string]resolve.HostEntry, writer io.Writer) error {
 	bufwriter := bufio.NewWriter(writer)
 	sb := &strings.Builder{}
 
@@ -153,11 +162,15 @@ func writePlainHost(results map[string]resolve.HostEntry, writer io.Writer) erro
 	return bufwriter.Flush()
 }
 
-func writeJSONHost(results map[string]resolve.HostEntry, writer io.Writer) error {
+func writeJSONHost(input string, results map[string]resolve.HostEntry, writer io.Writer) error {
 	encoder := jsoniter.NewEncoder(writer)
 
+	var data jsonSourceResult
 	for _, result := range results {
-		err := encoder.Encode(result)
+		data.Host = result.Host
+		data.Input = input
+		data.Source = result.Source
+		err := encoder.Encode(data)
 		if err != nil {
 			return err
 		}
@@ -166,23 +179,24 @@ func writeJSONHost(results map[string]resolve.HostEntry, writer io.Writer) error
 }
 
 // WriteSourceHost writes the output list of subdomain to an io.Writer
-func (o *OutPutter) WriteSourceHost(sourceMap map[string]map[string]struct{}, writer io.Writer) error {
+func (o *OutPutter) WriteSourceHost(input string, sourceMap map[string]map[string]struct{}, writer io.Writer) error {
 	var err error
 	if o.JSON {
-		err = writeSourceJSONHost(sourceMap, writer)
+		err = writeSourceJSONHost(input, sourceMap, writer)
 	} else {
-		err = writeSourcePlainHost(sourceMap, writer)
+		err = writeSourcePlainHost(input, sourceMap, writer)
 	}
 	return err
 }
 
-func writeSourceJSONHost(sourceMap map[string]map[string]struct{}, writer io.Writer) error {
+func writeSourceJSONHost(input string, sourceMap map[string]map[string]struct{}, writer io.Writer) error {
 	encoder := jsoniter.NewEncoder(writer)
 
-	var data jsonSourceResult
+	var data jsonSourcesResult
 
 	for host, sources := range sourceMap {
 		data.Host = host
+		data.Input = input
 		keys := make([]string, 0, len(sources))
 		for source := range sources {
 			keys = append(keys, source)
@@ -197,7 +211,7 @@ func writeSourceJSONHost(sourceMap map[string]map[string]struct{}, writer io.Wri
 	return nil
 }
 
-func writeSourcePlainHost(sourceMap map[string]map[string]struct{}, writer io.Writer) error {
+func writeSourcePlainHost(_ string, sourceMap map[string]map[string]struct{}, writer io.Writer) error {
 	bufwriter := bufio.NewWriter(writer)
 	sb := &strings.Builder{}