From f311374d7e7b3373b16e512b078775ec8bfb1793 Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Fri, 6 Dec 2019 17:27:26 +0530 Subject: [PATCH 1/2] Close the response bodies --- pkg/subscraping/sources/archiveis/archiveis.go | 1 + pkg/subscraping/sources/binaryedge/binaryedge.go | 2 ++ pkg/subscraping/sources/bufferover/bufferover.go | 1 + pkg/subscraping/sources/censys/censys.go | 1 + pkg/subscraping/sources/certspotter/certspotter.go | 2 ++ pkg/subscraping/sources/certspotterold/certspotterold.go | 1 + pkg/subscraping/sources/commoncrawl/commoncrawl.go | 5 +++++ pkg/subscraping/sources/crtsh/crtsh.go | 1 + pkg/subscraping/sources/digicert/digicert.go | 1 + pkg/subscraping/sources/dnsdumpster/dnsdumpster.go | 3 ++- pkg/subscraping/sources/entrust/entrust.go | 1 + pkg/subscraping/sources/googleter/googleter.go | 1 + pkg/subscraping/sources/hackertarget/hackertarget.go | 1 + pkg/subscraping/sources/ipv4info/ipv4info.go | 5 +++++ pkg/subscraping/sources/passivetotal/passivetotal.go | 1 + pkg/subscraping/sources/securitytrails/securitytrails.go | 1 + pkg/subscraping/sources/shodan/shodan.go | 1 + pkg/subscraping/sources/sitedossier/sitedossier.go | 1 + pkg/subscraping/sources/threatcrowd/threatcrowd.go | 1 + pkg/subscraping/sources/threatminer/threatminer.go | 1 + pkg/subscraping/sources/virustotal/virustotal.go | 1 + pkg/subscraping/sources/waybackarchive/waybackarchive.go | 1 + 22 files changed, 33 insertions(+), 1 deletion(-) diff --git a/pkg/subscraping/sources/archiveis/archiveis.go b/pkg/subscraping/sources/archiveis/archiveis.go index 8322bedb5..ea56e6e3e 100755 --- a/pkg/subscraping/sources/archiveis/archiveis.go +++ b/pkg/subscraping/sources/archiveis/archiveis.go @@ -37,6 +37,7 @@ func (a *ArchiveIs) enumerate(ctx context.Context, baseURL string) { body, err := ioutil.ReadAll(resp.Body) if err != nil { a.Results <- subscraping.Result{Source: "archiveis", Type: subscraping.Error, Error: err} + resp.Body.Close() close(a.Results) return } diff --git a/pkg/subscraping/sources/binaryedge/binaryedge.go b/pkg/subscraping/sources/binaryedge/binaryedge.go index 1b1ba3145..d71b17c18 100755 --- a/pkg/subscraping/sources/binaryedge/binaryedge.go +++ b/pkg/subscraping/sources/binaryedge/binaryedge.go @@ -37,6 +37,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se err = jsoniter.NewDecoder(resp.Body).Decode(&response) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + resp.Body.Close() close(results) return } @@ -82,6 +83,7 @@ func (s *Source) getSubdomains(ctx context.Context, domain string, remaining, cu err = jsoniter.NewDecoder(resp.Body).Decode(&response) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + resp.Body.Close() return false } resp.Body.Close() diff --git a/pkg/subscraping/sources/bufferover/bufferover.go b/pkg/subscraping/sources/bufferover/bufferover.go index 11689e0fe..822c27440 100755 --- a/pkg/subscraping/sources/bufferover/bufferover.go +++ b/pkg/subscraping/sources/bufferover/bufferover.go @@ -37,6 +37,7 @@ func (s *Source) getData(URL string, session *subscraping.Session, results chan body, err := ioutil.ReadAll(resp.Body) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + resp.Body.Close() return } resp.Body.Close() diff --git a/pkg/subscraping/sources/censys/censys.go b/pkg/subscraping/sources/censys/censys.go index 7d8d90568..b02c23eb2 100644 --- a/pkg/subscraping/sources/censys/censys.go +++ b/pkg/subscraping/sources/censys/censys.go @@ -64,6 +64,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se err = jsoniter.NewDecoder(resp.Body).Decode(&response) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + resp.Body.Close() close(results) return } diff --git a/pkg/subscraping/sources/certspotter/certspotter.go b/pkg/subscraping/sources/certspotter/certspotter.go index 581a68a7f..15c4ae74b 100755 --- a/pkg/subscraping/sources/certspotter/certspotter.go +++ b/pkg/subscraping/sources/certspotter/certspotter.go @@ -37,6 +37,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se err = jsoniter.NewDecoder(resp.Body).Decode(&response) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + resp.Body.Close() close(results) return } @@ -63,6 +64,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se err = jsoniter.NewDecoder(resp.Body).Decode(&response) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + resp.Body.Close() close(results) return } diff --git a/pkg/subscraping/sources/certspotterold/certspotterold.go b/pkg/subscraping/sources/certspotterold/certspotterold.go index 0da2a6a58..5f3115d8f 100755 --- a/pkg/subscraping/sources/certspotterold/certspotterold.go +++ b/pkg/subscraping/sources/certspotterold/certspotterold.go @@ -26,6 +26,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se body, err := ioutil.ReadAll(resp.Body) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + resp.Body.Close() close(results) return } diff --git a/pkg/subscraping/sources/commoncrawl/commoncrawl.go b/pkg/subscraping/sources/commoncrawl/commoncrawl.go index bbaeebb23..9d5b2f538 100755 --- a/pkg/subscraping/sources/commoncrawl/commoncrawl.go +++ b/pkg/subscraping/sources/commoncrawl/commoncrawl.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "io" "io/ioutil" "strings" @@ -37,6 +38,8 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se if resp.StatusCode == 500 { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: errors.New("internal server error")} + io.Copy(ioutil.Discard, resp.Body) + resp.Body.Close() close(results) return } @@ -45,6 +48,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se err = jsoniter.NewDecoder(resp.Body).Decode(&indexes) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + resp.Body.Close() close(results) return } @@ -94,6 +98,7 @@ func (s *Source) getSubdomains(ctx context.Context, url string, domain string, s body, err := ioutil.ReadAll(resp.Body) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + resp.Body.Close() return false } resp.Body.Close() diff --git a/pkg/subscraping/sources/crtsh/crtsh.go b/pkg/subscraping/sources/crtsh/crtsh.go index bec2b0c75..8e3f0c5ce 100755 --- a/pkg/subscraping/sources/crtsh/crtsh.go +++ b/pkg/subscraping/sources/crtsh/crtsh.go @@ -26,6 +26,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se body, err := ioutil.ReadAll(resp.Body) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + resp.Body.Close() close(results) return } diff --git a/pkg/subscraping/sources/digicert/digicert.go b/pkg/subscraping/sources/digicert/digicert.go index eb74658ed..e731250ea 100755 --- a/pkg/subscraping/sources/digicert/digicert.go +++ b/pkg/subscraping/sources/digicert/digicert.go @@ -26,6 +26,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se body, err := ioutil.ReadAll(resp.Body) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + resp.Body.Close() close(results) return } diff --git a/pkg/subscraping/sources/dnsdumpster/dnsdumpster.go b/pkg/subscraping/sources/dnsdumpster/dnsdumpster.go index f77569a05..fcc7b6b32 100755 --- a/pkg/subscraping/sources/dnsdumpster/dnsdumpster.go +++ b/pkg/subscraping/sources/dnsdumpster/dnsdumpster.go @@ -62,7 +62,7 @@ func postForm(token, domain string) (string, error) { // Now, grab the entire page in, err := ioutil.ReadAll(resp.Body) resp.Body.Close() - return string(in), nil + return string(in), err } // Source is the passive scraping agent @@ -83,6 +83,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se body, err := ioutil.ReadAll(resp.Body) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + resp.Body.Close() close(results) return } diff --git a/pkg/subscraping/sources/entrust/entrust.go b/pkg/subscraping/sources/entrust/entrust.go index 12aad1eea..e042e5475 100755 --- a/pkg/subscraping/sources/entrust/entrust.go +++ b/pkg/subscraping/sources/entrust/entrust.go @@ -27,6 +27,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se body, err := ioutil.ReadAll(resp.Body) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + resp.Body.Close() close(results) return } diff --git a/pkg/subscraping/sources/googleter/googleter.go b/pkg/subscraping/sources/googleter/googleter.go index 02e4ae8bb..93f042ca9 100755 --- a/pkg/subscraping/sources/googleter/googleter.go +++ b/pkg/subscraping/sources/googleter/googleter.go @@ -38,6 +38,7 @@ func (a *agent) makeRequest(token string, domain string) (string, error) { } body, err := ioutil.ReadAll(resp.Body) if err != nil { + resp.Body.Close() return "", err } resp.Body.Close() diff --git a/pkg/subscraping/sources/hackertarget/hackertarget.go b/pkg/subscraping/sources/hackertarget/hackertarget.go index 325b6bcc5..9e6f99cd4 100755 --- a/pkg/subscraping/sources/hackertarget/hackertarget.go +++ b/pkg/subscraping/sources/hackertarget/hackertarget.go @@ -27,6 +27,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se body, err := ioutil.ReadAll(resp.Body) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + resp.Body.Close() close(results) return } diff --git a/pkg/subscraping/sources/ipv4info/ipv4info.go b/pkg/subscraping/sources/ipv4info/ipv4info.go index 8a5f5c2de..bd840f0aa 100755 --- a/pkg/subscraping/sources/ipv4info/ipv4info.go +++ b/pkg/subscraping/sources/ipv4info/ipv4info.go @@ -28,6 +28,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se body, err := ioutil.ReadAll(resp.Body) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + resp.Body.Close() close(results) return } @@ -54,6 +55,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se body, err = ioutil.ReadAll(resp.Body) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + resp.Body.Close() close(results) return } @@ -79,6 +81,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se body, err = ioutil.ReadAll(resp.Body) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + resp.Body.Close() close(results) return } @@ -104,6 +107,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se body, err = ioutil.ReadAll(resp.Body) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + resp.Body.Close() close(results) return } @@ -153,6 +157,7 @@ func (s *Source) getSubdomains(ctx context.Context, domain string, nextPage *int body, err := ioutil.ReadAll(resp.Body) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + resp.Body.Close() return false } resp.Body.Close() diff --git a/pkg/subscraping/sources/passivetotal/passivetotal.go b/pkg/subscraping/sources/passivetotal/passivetotal.go index 1960bc44f..47e2bc63c 100755 --- a/pkg/subscraping/sources/passivetotal/passivetotal.go +++ b/pkg/subscraping/sources/passivetotal/passivetotal.go @@ -50,6 +50,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se err = jsoniter.NewDecoder(resp.Body).Decode(&data) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + resp.Body.Close() close(results) return } diff --git a/pkg/subscraping/sources/securitytrails/securitytrails.go b/pkg/subscraping/sources/securitytrails/securitytrails.go index 3ce1c285d..8653ee136 100755 --- a/pkg/subscraping/sources/securitytrails/securitytrails.go +++ b/pkg/subscraping/sources/securitytrails/securitytrails.go @@ -37,6 +37,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se err = jsoniter.NewDecoder(resp.Body).Decode(&response) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + resp.Body.Close() close(results) return } diff --git a/pkg/subscraping/sources/shodan/shodan.go b/pkg/subscraping/sources/shodan/shodan.go index 2c31d4a19..0d32277ab 100644 --- a/pkg/subscraping/sources/shodan/shodan.go +++ b/pkg/subscraping/sources/shodan/shodan.go @@ -43,6 +43,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se err = jsoniter.NewDecoder(resp.Body).Decode(&response) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + resp.Body.Close() close(results) return } diff --git a/pkg/subscraping/sources/sitedossier/sitedossier.go b/pkg/subscraping/sources/sitedossier/sitedossier.go index 6c5117504..d24d82c38 100755 --- a/pkg/subscraping/sources/sitedossier/sitedossier.go +++ b/pkg/subscraping/sources/sitedossier/sitedossier.go @@ -34,6 +34,7 @@ func (a *agent) enumerate(ctx context.Context, baseURL string) error { body, err := ioutil.ReadAll(resp.Body) if err != nil { a.results <- subscraping.Result{Source: "sitedossier", Type: subscraping.Error, Error: err} + resp.Body.Close() close(a.results) return err } diff --git a/pkg/subscraping/sources/threatcrowd/threatcrowd.go b/pkg/subscraping/sources/threatcrowd/threatcrowd.go index 22e6ffd24..f0fed815d 100755 --- a/pkg/subscraping/sources/threatcrowd/threatcrowd.go +++ b/pkg/subscraping/sources/threatcrowd/threatcrowd.go @@ -27,6 +27,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se body, err := ioutil.ReadAll(resp.Body) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + resp.Body.Close() close(results) return } diff --git a/pkg/subscraping/sources/threatminer/threatminer.go b/pkg/subscraping/sources/threatminer/threatminer.go index 538fc563e..252093974 100755 --- a/pkg/subscraping/sources/threatminer/threatminer.go +++ b/pkg/subscraping/sources/threatminer/threatminer.go @@ -27,6 +27,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se body, err := ioutil.ReadAll(resp.Body) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + resp.Body.Close() close(results) return } diff --git a/pkg/subscraping/sources/virustotal/virustotal.go b/pkg/subscraping/sources/virustotal/virustotal.go index 8950f1a1d..fb6e834ea 100755 --- a/pkg/subscraping/sources/virustotal/virustotal.go +++ b/pkg/subscraping/sources/virustotal/virustotal.go @@ -36,6 +36,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se err = jsoniter.NewDecoder(resp.Body).Decode(&data) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + resp.Body.Close() close(results) return } diff --git a/pkg/subscraping/sources/waybackarchive/waybackarchive.go b/pkg/subscraping/sources/waybackarchive/waybackarchive.go index d34c68e6f..f97d74422 100755 --- a/pkg/subscraping/sources/waybackarchive/waybackarchive.go +++ b/pkg/subscraping/sources/waybackarchive/waybackarchive.go @@ -27,6 +27,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se body, err := ioutil.ReadAll(pagesResp.Body) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + pagesResp.Body.Close() close(results) return } From 443181d53921be57138d88bd01b324a63e0a8a5c Mon Sep 17 00:00:00 2001 From: Ice3man543 Date: Fri, 6 Dec 2019 18:02:16 +0530 Subject: [PATCH 2/2] Added release scripts to subfinder --- .github/dockerhub-push-on-release.yml | 17 ++++++++ .github/push-binaries-on-release.yml | 56 +++++++++++++++++++++++++++ 2 files changed, 73 insertions(+) create mode 100644 .github/dockerhub-push-on-release.yml create mode 100644 .github/push-binaries-on-release.yml diff --git a/.github/dockerhub-push-on-release.yml b/.github/dockerhub-push-on-release.yml new file mode 100644 index 000000000..4e6a5dc08 --- /dev/null +++ b/.github/dockerhub-push-on-release.yml @@ -0,0 +1,17 @@ +# dockerhub-push pushes docker build to dockerhub automatically +# on the creation of a new release +name: Publish to Dockerhub on creation of a new release +on: + release: + types: [created] +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@master + - name: Publish to Dockerhub Registry + uses: elgohr/Publish-Docker-Github-Action@master + with: + name: ice3man/subfinder + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} \ No newline at end of file diff --git a/.github/push-binaries-on-release.yml b/.github/push-binaries-on-release.yml new file mode 100644 index 000000000..8d468d6ab --- /dev/null +++ b/.github/push-binaries-on-release.yml @@ -0,0 +1,56 @@ +# push-binaries-onrelease cross compiles the subfinder binary, +# tars the files and uploads them to the release. +name: Cross compile binaries and attach to release +on: + release: + types: [created] +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: Set up Go 1.13.5 + uses: actions/setup-go@v1 + with: + go-version: 1.13.5 + id: go + + - name: Check out code into the Go module directory + uses: actions/checkout@v1 + + - name: Get dependencies + run: | + go get -v -t -d ./... + if [ -f Gopkg.toml ]; then + curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh + dep ensure + fi + + - name: Build the tar archives + run: | + platforms=("windows/amd64" "linux/amd64" "darwin/amd64") + + for platform in "${platforms[@]}" + do + platform_split=(${platform//\// }) + GOOS=${platform_split[0]} + GOARCH=${platform_split[1]} + output_name=subfinder'-'$GOOS'-'$GOARCH + if [ $GOOS = "windows" ]; then + output_name+='.exe' + fi + + env GOOS=$GOOS GOARCH=$GOARCH go build -o $output_name github.com/projectdiscovery/subfinder/cmd/subfinder + if [ $? -ne 0 ]; then + echo 'An error has occurred! Aborting the script execution...' + exit 1 + fi + tar -czvf $output_name.tar $output_name + rm $output_name + done + + - name: Publish the binaries to the release + uses: skx/github-action-publish-binaries@master + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + args: 'subfinder-*'