diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index 0c9916b54..54bac718b 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -3,12 +3,12 @@ name: 🔨 Build Test on: pull_request: paths: - - '**.go' - - '**.mod' + - "**.go" + - "**.mod" workflow_dispatch: inputs: short: - description: 'Use -short flag for tests' + description: "Use -short flag for tests" required: false type: boolean default: false @@ -21,8 +21,10 @@ jobs: steps: - uses: actions/checkout@v4 - uses: projectdiscovery/actions/setup/go@v1 + with: + go-version-file: v2/go.mod - name: Run golangci-lint - uses: projectdiscovery/actions/golangci-lint/v2@v1 + uses: golangci/golangci-lint-action@v8 with: version: latest args: --timeout 5m @@ -38,6 +40,8 @@ jobs: steps: - uses: actions/checkout@v4 - uses: projectdiscovery/actions/setup/go@v1 + with: + go-version-file: v2/go.mod - run: go build ./... working-directory: v2/ @@ -79,4 +83,4 @@ jobs: - name: Run Example run: go run . - working-directory: v2/examples \ No newline at end of file + working-directory: v2/examples diff --git a/Dockerfile b/Dockerfile index eadf4f7e1..7d2291080 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # Build -FROM golang:1.21-alpine AS build-env +FROM golang:1.24-alpine AS build-env RUN apk add build-base WORKDIR /app COPY . /app @@ -8,7 +8,7 @@ RUN go mod download RUN go build ./cmd/subfinder # Release -FROM alpine:3.18.6 +FROM alpine:latest RUN apk upgrade --no-cache \ && apk add --no-cache bind-tools ca-certificates COPY --from=build-env /app/v2/subfinder /usr/local/bin/ diff --git a/README.md b/README.md index 97e0b09fa..4a3ce144e 100644 --- a/README.md +++ b/README.md @@ -111,7 +111,7 @@ OPTIMIZATION: # Installation -`subfinder` requires **go1.21** to install successfully. Run the following command to install the latest version: +`subfinder` requires **go1.24** to install successfully. Run the following command to install the latest version: ```sh go install -v github.com/projectdiscovery/subfinder/v2/cmd/subfinder@latest diff --git a/v2/.goreleaser.yml b/v2/.goreleaser.yml index 6e97216d6..867267d2e 100644 --- a/v2/.goreleaser.yml +++ b/v2/.goreleaser.yml @@ -1,3 +1,5 @@ +version: 2 + before: hooks: - go mod tidy @@ -25,7 +27,8 @@ builds: main: cmd/subfinder/main.go archives: -- format: zip +- formats: + - zip name_template: '{{ .ProjectName }}_{{ .Version }}_{{ if eq .Os "darwin" }}macOS{{ else }}{{ .Os }}{{ end }}_{{ .Arch }}' checksum: @@ -40,4 +43,4 @@ announce: discord: enabled: true - message_template: '**New Release: {{ .ProjectName }} {{.Tag}}** is published! Check it out at {{ .ReleaseURL }}' \ No newline at end of file + message_template: '**New Release: {{ .ProjectName }} {{.Tag}}** is published! Check it out at {{ .ReleaseURL }}' diff --git a/v2/go.mod b/v2/go.mod index cde73b596..fdb011821 100644 --- a/v2/go.mod +++ b/v2/go.mod @@ -13,9 +13,9 @@ require ( github.com/projectdiscovery/dnsx v1.2.2 github.com/projectdiscovery/fdmax v0.0.4 github.com/projectdiscovery/gologger v1.1.54 - github.com/projectdiscovery/ratelimit v0.0.79 - github.com/projectdiscovery/retryablehttp-go v1.0.109 - github.com/projectdiscovery/utils v0.4.18 + github.com/projectdiscovery/ratelimit v0.0.81 + github.com/projectdiscovery/retryablehttp-go v1.0.114 + github.com/projectdiscovery/utils v0.4.20 github.com/rs/xid v1.5.0 github.com/stretchr/testify v1.10.0 github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 @@ -43,13 +43,13 @@ require ( github.com/charmbracelet/lipgloss v0.13.0 // indirect github.com/charmbracelet/x/ansi v0.3.2 // indirect github.com/cheggaaa/pb/v3 v3.1.4 // indirect - github.com/cloudflare/circl v1.5.0 // indirect + github.com/cloudflare/circl v1.6.1 // indirect github.com/dimchansky/utfbom v1.1.1 // indirect github.com/dlclark/regexp2 v1.11.4 // indirect github.com/docker/go-units v0.5.0 // indirect github.com/dsnet/compress v0.0.2-0.20230904184137-39efe44ab707 // indirect github.com/fatih/color v1.15.0 // indirect - github.com/gaissmai/bart v0.17.10 // indirect + github.com/gaissmai/bart v0.20.4 // indirect github.com/go-ole/go-ole v1.2.6 // indirect github.com/golang/protobuf v1.5.3 // indirect github.com/golang/snappy v0.0.4 // indirect @@ -78,11 +78,11 @@ require ( github.com/pierrec/lz4/v4 v4.1.21 // indirect github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect github.com/projectdiscovery/blackrock v0.0.1 // indirect - github.com/projectdiscovery/cdncheck v1.1.15 // indirect - github.com/projectdiscovery/fastdialer v0.4.0 // indirect - github.com/projectdiscovery/hmap v0.0.87 // indirect + github.com/projectdiscovery/cdncheck v1.1.23 // indirect + github.com/projectdiscovery/fastdialer v0.4.1 // indirect + github.com/projectdiscovery/hmap v0.0.89 // indirect github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983 // indirect - github.com/projectdiscovery/networkpolicy v0.1.12 // indirect + github.com/projectdiscovery/networkpolicy v0.1.15 // indirect github.com/refraction-networking/utls v1.7.0 // indirect github.com/rivo/uniseg v0.4.7 // indirect github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d // indirect @@ -135,7 +135,7 @@ require ( github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/projectdiscovery/goflags v0.1.74 - github.com/projectdiscovery/retryabledns v1.0.98 // indirect + github.com/projectdiscovery/retryabledns v1.0.101 // indirect golang.org/x/net v0.38.0 // indirect golang.org/x/sys v0.31.0 // indirect ) diff --git a/v2/go.sum b/v2/go.sum index 834f80d0f..c0de7afbd 100644 --- a/v2/go.sum +++ b/v2/go.sum @@ -76,8 +76,8 @@ github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5P github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cloudflare/circl v1.1.0/go.mod h1:prBCrKB9DV4poKZY1l9zBXg2QJY7mvgRvtMxxK7fi4I= -github.com/cloudflare/circl v1.5.0 h1:hxIWksrX6XN5a1L2TI/h53AGPhNHoUBo+TD1ms9+pys= -github.com/cloudflare/circl v1.5.0/go.mod h1:uddAzsPgqdMAYatqJ0lsjX1oECcQLIlRpzZh3pJrofs= +github.com/cloudflare/circl v1.6.1 h1:zqIqSPIndyBh1bjLVVDHMPpVKqp8Su/V+6MeDzzQBQ0= +github.com/cloudflare/circl v1.6.1/go.mod h1:uddAzsPgqdMAYatqJ0lsjX1oECcQLIlRpzZh3pJrofs= github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08 h1:ox2F0PSMlrAAiAdknSRMDrAr8mfxPCfSZolH+/qQnyQ= github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08/go.mod h1:pCxVEbcm3AMg7ejXyorUXi6HQCzOIBf7zEDVPtw0/U4= github.com/corpix/uarand v0.2.0 h1:U98xXwud/AVuCpkpgfPF7J5TQgr7R5tqT8VZP5KWbzE= @@ -102,8 +102,8 @@ github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBD github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= -github.com/gaissmai/bart v0.17.10 h1:TY1y++A6N/ESrwRLTRWrnVOrQpZqpOYSVnKMu/FYW6o= -github.com/gaissmai/bart v0.17.10/go.mod h1:JCPkH/Xt5bSPCKDc6OpzkhSCeib8BIxu3kthzZwcl6w= +github.com/gaissmai/bart v0.20.4 h1:Ik47r1fy3jRVU+1eYzKSW3ho2UgBVTVnUS8O993584U= +github.com/gaissmai/bart v0.20.4/go.mod h1:cEed+ge8dalcbpi8wtS9x9m2hn/fNJH5suhdGQOHnYk= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= @@ -255,34 +255,34 @@ github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/projectdiscovery/blackrock v0.0.1 h1:lHQqhaaEFjgf5WkuItbpeCZv2DUIE45k0VbGJyft6LQ= github.com/projectdiscovery/blackrock v0.0.1/go.mod h1:ANUtjDfaVrqB453bzToU+YB4cUbvBRpLvEwoWIwlTss= -github.com/projectdiscovery/cdncheck v1.1.15 h1:rRs3LW2MP7V8QeONVRYce6RhDcWp83O+AWmt+QQ4mBM= -github.com/projectdiscovery/cdncheck v1.1.15/go.mod h1:dFEGsG0qAJY0AaRr2N1BY0OtZiTxS4kYeT5+OkF8t1U= +github.com/projectdiscovery/cdncheck v1.1.23 h1:LOd6Y7hnV6sXFBs4qGDM0N9xfheAmqLhsfH2cog+M2c= +github.com/projectdiscovery/cdncheck v1.1.23/go.mod h1:dFEGsG0qAJY0AaRr2N1BY0OtZiTxS4kYeT5+OkF8t1U= github.com/projectdiscovery/chaos-client v0.5.2 h1:dN+7GXEypsJAbCD//dBcUxzAEAEH1fjc/7Rf4F/RiNU= github.com/projectdiscovery/chaos-client v0.5.2/go.mod h1:KnoJ/NJPhll42uaqlDga6oafFfNw5l2XI2ajRijtDuU= github.com/projectdiscovery/dnsx v1.2.2 h1:ZjUov0GOyrS8ERlKAAhk+AOkqzaYHBzCP0qZfO+6Ihg= github.com/projectdiscovery/dnsx v1.2.2/go.mod h1:3iYm86OEqo0WxeGDkVl5WZNmG0qYE5TYNx8fBg6wX1I= -github.com/projectdiscovery/fastdialer v0.4.0 h1:licZKyq+Shd5lLDb8uPd60Jp43K4NFE8cr67XD2eg7w= -github.com/projectdiscovery/fastdialer v0.4.0/go.mod h1:Q0YLArvpx9GAfY/NcTPMCA9qZuVOGnuVoNYWzKBwxdQ= +github.com/projectdiscovery/fastdialer v0.4.1 h1:kp6Q0odo0VZ0vZIGOn+q9aLgBSk6uYoD1MsjCAH8+h4= +github.com/projectdiscovery/fastdialer v0.4.1/go.mod h1:875Wlggf0JAz+fDIPwUQeeBqEF6nJA71XVrjuTZCV7I= github.com/projectdiscovery/fdmax v0.0.4 h1:K9tIl5MUZrEMzjvwn/G4drsHms2aufTn1xUdeVcmhmc= github.com/projectdiscovery/fdmax v0.0.4/go.mod h1:oZLqbhMuJ5FmcoaalOm31B1P4Vka/CqP50nWjgtSz+I= github.com/projectdiscovery/goflags v0.1.74 h1:n85uTRj5qMosm0PFBfsvOL24I7TdWRcWq/1GynhXS7c= github.com/projectdiscovery/goflags v0.1.74/go.mod h1:UMc9/7dFz2oln+10tv6cy+7WZKTHf9UGhaNkF95emh4= github.com/projectdiscovery/gologger v1.1.54 h1:WMzvJ8j/4gGfPKpCttSTaYCVDU1MWQSJnk3wU8/U6Ws= github.com/projectdiscovery/gologger v1.1.54/go.mod h1:vza/8pe2OKOt+ujFWncngknad1XWr8EnLKlbcejOyUE= -github.com/projectdiscovery/hmap v0.0.87 h1:bSIqggL878qmmMG67rNgmEa314GB1o2rFM9wjJbsJHA= -github.com/projectdiscovery/hmap v0.0.87/go.mod h1:Je1MuSMaP1gM2toj/t3YQhGQpSJGYjQuQwHJpPyJT6g= +github.com/projectdiscovery/hmap v0.0.89 h1:H+XIzk2YcE/9PpW/1N9NdQSrJWm2vthGPNIxSM+WHNU= +github.com/projectdiscovery/hmap v0.0.89/go.mod h1:N3gXFDLN6GqkYsk+2ZkReVOo32OBUV+PNiYyWhWG4ZE= github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983 h1:ZScLodGSezQVwsQDtBSMFp72WDq0nNN+KE/5DHKY5QE= github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983/go.mod h1:3G3BRKui7nMuDFAZKR/M2hiOLtaOmyukT20g88qRQjI= -github.com/projectdiscovery/networkpolicy v0.1.12 h1:SwfCOm772jmkLQNKWKZHIhjJK3eYz4RVzMHZJfwtti8= -github.com/projectdiscovery/networkpolicy v0.1.12/go.mod h1:8fm26WaxgfNY3CGQWzohQy95oSzZlgikU9Oxd1Pq5mk= -github.com/projectdiscovery/ratelimit v0.0.79 h1:9Kzff7K5ZyAX0IWspx5X3fHtff0/TzFq7jDxEScO1Qw= -github.com/projectdiscovery/ratelimit v0.0.79/go.mod h1:z+hNaODlTmdajGj7V2yIqcQhB7fovdMSK2PNwpbrlHY= -github.com/projectdiscovery/retryabledns v1.0.98 h1:2rz0dExX6pJlp8BrF0ZwwimO+Y6T7KCDsstmUioF8cA= -github.com/projectdiscovery/retryabledns v1.0.98/go.mod h1:AeFHeqjpm375uKHKf9dn4+EvwsE/xXGGDU5cT5EEiqQ= -github.com/projectdiscovery/retryablehttp-go v1.0.109 h1:z7USVuroBrJJH/ozGS4m+evwukFyJvIvjmvaTNXrhr8= -github.com/projectdiscovery/retryablehttp-go v1.0.109/go.mod h1:0A6WpqP585LzGIFHQButwfQin4746gvNK2BrGpmRoXI= -github.com/projectdiscovery/utils v0.4.18 h1:cSjMOLXI5gAajfA6KV+0iQG4dGx2IHWLQyND/Snvw7k= -github.com/projectdiscovery/utils v0.4.18/go.mod h1:y5gnpQn802iEWqf0djTRNskJlS62P5eqe1VS1+ah0tk= +github.com/projectdiscovery/networkpolicy v0.1.15 h1:jHHPo43s/TSiWmm6T8kJuMqTwL3ukU92iQhxq0K0jg0= +github.com/projectdiscovery/networkpolicy v0.1.15/go.mod h1:GWMDGJmgJ9qGoVTUOxbq1oLIbEx0pPsL0VKlriCkn2g= +github.com/projectdiscovery/ratelimit v0.0.81 h1:u6lW+rAhS/UO0amHTYmYLipPK8NEotA9521hdojBtgI= +github.com/projectdiscovery/ratelimit v0.0.81/go.mod h1:tK04WXHuC4i6AsFkByInODSNf45gd9sfaMHzmy2bAsA= +github.com/projectdiscovery/retryabledns v1.0.101 h1:8DIVD8CL34Lc9h6KeOopPUfsPlcFxMlrnKOaI9VeOMk= +github.com/projectdiscovery/retryabledns v1.0.101/go.mod h1:fQI91PKUyTZYL2pYloyA9Bh3Bq8IgOB6X+bN+8Xm14I= +github.com/projectdiscovery/retryablehttp-go v1.0.114 h1:JFvk7RJ2AUrHV9dScHcnyaBpQRGq1d8/QfrpccCT0xc= +github.com/projectdiscovery/retryablehttp-go v1.0.114/go.mod h1:ZXHlpbSw9w3nZqe1LH0GPX2UDAmv2QpUOoafy+xydYs= +github.com/projectdiscovery/utils v0.4.20 h1:7Fmjb+4YZJSzn7bL21sjF3wAR53eSi7VdAfDkDBUUwY= +github.com/projectdiscovery/utils v0.4.20/go.mod h1:RnC23+hI8j4drZFHQpMX92hV9++9d/yBeNr1pzcbF7Y= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/refraction-networking/utls v1.7.0 h1:9JTnze/Md74uS3ZWiRAabityY0un69rOLXsBf8LGgTs= github.com/refraction-networking/utls v1.7.0/go.mod h1:lV0Gwc1/Fi+HYH8hOtgFRdHfKo4FKSn6+FdyOz9hRms= diff --git a/v2/pkg/passive/sources.go b/v2/pkg/passive/sources.go index 7de378976..20013042f 100644 --- a/v2/pkg/passive/sources.go +++ b/v2/pkg/passive/sources.go @@ -1,6 +1,8 @@ package passive import ( + "fmt" + "os" "strings" "golang.org/x/exp/maps" @@ -10,7 +12,6 @@ import ( "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/alienvault" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/anubis" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/bevigil" - "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/binaryedge" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/bufferover" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/builtwith" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/c99" @@ -20,6 +21,7 @@ import ( "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/chinaz" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/commoncrawl" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/crtsh" + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/digitalyama" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/digitorus" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/dnsdb" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/dnsdumpster" @@ -34,10 +36,12 @@ import ( "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/intelx" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/leakix" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/netlas" + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/pugrecon" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/quake" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/rapiddns" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/redhuntlabs" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/robtex" + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/rsecloud" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/securitytrails" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/shodan" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/sitedossier" @@ -47,7 +51,6 @@ import ( "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/waybackarchive" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/whoisxmlapi" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/zoomeyeapi" - "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/digitalyama" mapsutil "github.com/projectdiscovery/utils/maps" ) @@ -55,7 +58,6 @@ var AllSources = [...]subscraping.Source{ &alienvault.Source{}, &anubis.Source{}, &bevigil.Source{}, - &binaryedge.Source{}, &bufferover.Source{}, &c99.Source{}, &censys.Source{}, @@ -77,10 +79,12 @@ var AllSources = [...]subscraping.Source{ &netlas.Source{}, &leakix.Source{}, &quake.Source{}, + &pugrecon.Source{}, &rapiddns.Source{}, &redhuntlabs.Source{}, // &riddler.Source{}, // failing due to cloudfront protection &robtex.Source{}, + &rsecloud.Source{}, &securitytrails.Source{}, &shodan.Source{}, &sitedossier.Source{}, @@ -166,6 +170,15 @@ func New(sourceNames, excludedSourceNames []string, useAllSources, useSourcesSup } } + // TODO: Consider refactoring this to avoid potential duplication issues + for _, source := range sources { + if source.NeedsKey() { + if apiKey := os.Getenv(fmt.Sprintf("%s_API_KEY", strings.ToUpper(source.Name()))); apiKey != "" { + source.AddApiKeys([]string{apiKey}) + } + } + } + // Create the agent, insert the sources and remove the excluded sources agent := &Agent{sources: maps.Values(sources)} diff --git a/v2/pkg/passive/sources_test.go b/v2/pkg/passive/sources_test.go index 848cc32e7..a63760296 100644 --- a/v2/pkg/passive/sources_test.go +++ b/v2/pkg/passive/sources_test.go @@ -14,7 +14,6 @@ var ( "alienvault", "anubis", "bevigil", - "binaryedge", "bufferover", "c99", "censys", @@ -34,10 +33,12 @@ var ( "intelx", "netlas", "quake", + "pugrecon", "rapiddns", "redhuntlabs", // "riddler", // failing due to cloudfront protection "robtex", + "rsecloud", "securitytrails", "shodan", "sitedossier", @@ -79,6 +80,7 @@ var ( "redhuntlabs", "robtex", // "riddler", // failing due to cloudfront protection + "rsecloud", "securitytrails", "shodan", "virustotal", @@ -94,7 +96,6 @@ var ( expectedDefaultRecursiveSources = []string{ "alienvault", - "binaryedge", "bufferover", "certspotter", "crtsh", diff --git a/v2/pkg/passive/sources_wo_auth_test.go b/v2/pkg/passive/sources_wo_auth_test.go index d5aef6530..036588cc3 100644 --- a/v2/pkg/passive/sources_wo_auth_test.go +++ b/v2/pkg/passive/sources_wo_auth_test.go @@ -24,14 +24,15 @@ func TestSourcesWithoutKeys(t *testing.T) { } ignoredSources := []string{ - "commoncrawl", // commoncrawl is under resourced and will likely time-out so step over it for this test https://groups.google.com/u/2/g/common-crawl/c/3QmQjFA_3y4/m/vTbhGqIBBQAJ - "riddler", // failing due to cloudfront protection - "crtsh", // Fails in GH Action (possibly IP-based ban) causing a timeout. - "hackertarget", // Fails in GH Action (possibly IP-based ban) but works locally - "waybackarchive", // Fails randomly - "alienvault", // 503 Service Temporarily Unavailable - "digitorus", // failing with "Failed to retrieve certificate" - "dnsdumpster", // failing with "unexpected status code 403 received" + "commoncrawl", // commoncrawl is under resourced and will likely time-out so step over it for this test https://groups.google.com/u/2/g/common-crawl/c/3QmQjFA_3y4/m/vTbhGqIBBQAJ + "riddler", // failing due to cloudfront protection + "crtsh", // Fails in GH Action (possibly IP-based ban) causing a timeout. + "hackertarget", // Fails in GH Action (possibly IP-based ban) but works locally + "waybackarchive", // Fails randomly + "alienvault", // 503 Service Temporarily Unavailable + "digitorus", // failing with "Failed to retrieve certificate" + "dnsdumpster", // failing with "unexpected status code 403 received" + "anubis", // failing with "too many redirects" } domain := "hackerone.com" diff --git a/v2/pkg/resolve/resolve.go b/v2/pkg/resolve/resolve.go index 271d699fb..d7e4d887c 100644 --- a/v2/pkg/resolve/resolve.go +++ b/v2/pkg/resolve/resolve.go @@ -60,7 +60,7 @@ func (r *Resolver) NewResolutionPool(workers int, removeWildcard bool) *Resoluti } go func() { - for i := 0; i < workers; i++ { + for range workers { resolutionPool.wg.Add(1) go resolutionPool.resolveWorker() } @@ -73,7 +73,7 @@ func (r *Resolver) NewResolutionPool(workers int, removeWildcard bool) *Resoluti // InitWildcards inits the wildcard ips array func (r *ResolutionPool) InitWildcards(domain string) error { - for i := 0; i < maxWildcardChecks; i++ { + for range maxWildcardChecks { uid := xid.New().String() hosts, _ := r.DNSClient.Lookup(uid + "." + domain) diff --git a/v2/pkg/runner/banners.go b/v2/pkg/runner/banners.go index 2bb921d1b..a0d3a7d1c 100644 --- a/v2/pkg/runner/banners.go +++ b/v2/pkg/runner/banners.go @@ -17,7 +17,7 @@ const banner = ` const ToolName = `subfinder` // Version is the current version of subfinder -const version = `v2.7.1` +const version = `v2.8.0` // showBanner is used to show the banner to the user func showBanner() { diff --git a/v2/pkg/runner/config.go b/v2/pkg/runner/config.go index 6c8ed2c6f..1751c9054 100644 --- a/v2/pkg/runner/config.go +++ b/v2/pkg/runner/config.go @@ -17,7 +17,11 @@ func createProviderConfigYAML(configFilePath string) error { if err != nil { return err } - defer configFile.Close() + defer func() { + if err := configFile.Close(); err != nil { + gologger.Error().Msgf("Error closing config file: %s", err) + } + }() sourcesRequiringApiKeysMap := make(map[string][]string) for _, source := range passive.AllSources { diff --git a/v2/pkg/runner/options.go b/v2/pkg/runner/options.go index 5e3d4b172..7f39e0315 100644 --- a/v2/pkg/runner/options.go +++ b/v2/pkg/runner/options.go @@ -244,6 +244,7 @@ func (options *Options) preProcessDomains() { var defaultRateLimits = []string{ "github=30/m", "fullhunt=60/m", + "pugrecon=10/s", fmt.Sprintf("robtex=%d/ms", uint(math.MaxUint)), "securitytrails=1/s", "shodan=1/s", diff --git a/v2/pkg/runner/outputter.go b/v2/pkg/runner/outputter.go index faef0cae0..cfa4fc1a5 100644 --- a/v2/pkg/runner/outputter.go +++ b/v2/pkg/runner/outputter.go @@ -97,7 +97,9 @@ func writePlainHostIP(_ string, results map[string]resolve.Result, writer io.Wri _, err := bufwriter.WriteString(sb.String()) if err != nil { - bufwriter.Flush() + if flushErr := bufwriter.Flush(); flushErr != nil { + return errors.Join(err, flushErr) + } return err } sb.Reset() @@ -155,7 +157,9 @@ func writePlainHost(_ string, results map[string]resolve.HostEntry, writer io.Wr _, err := bufwriter.WriteString(sb.String()) if err != nil { - bufwriter.Flush() + if flushErr := bufwriter.Flush(); flushErr != nil { + return errors.Join(err, flushErr) + } return err } sb.Reset() @@ -228,7 +232,9 @@ func writeSourcePlainHost(_ string, sourceMap map[string]map[string]struct{}, wr _, err := bufwriter.WriteString(sb.String()) if err != nil { - bufwriter.Flush() + if flushErr := bufwriter.Flush(); flushErr != nil { + return errors.Join(err, flushErr) + } return err } sb.Reset() diff --git a/v2/pkg/runner/runner.go b/v2/pkg/runner/runner.go index ce9350419..f00ce6418 100644 --- a/v2/pkg/runner/runner.go +++ b/v2/pkg/runner/runner.go @@ -94,7 +94,9 @@ func (r *Runner) RunEnumerationWithCtx(ctx context.Context) error { return err } err = r.EnumerateMultipleDomainsWithCtx(ctx, f, outputs) - f.Close() + if closeErr := f.Close(); closeErr != nil { + gologger.Error().Msgf("Error closing file %s: %s", r.options.DomainsFile, closeErr) + } return err } @@ -139,7 +141,9 @@ func (r *Runner) EnumerateMultipleDomainsWithCtx(ctx context.Context, reader io. _, err = r.EnumerateSingleDomainWithCtx(ctx, domain, append(writers, file)) - file.Close() + if closeErr := file.Close(); closeErr != nil { + gologger.Error().Msgf("Error closing file %s: %s", r.options.OutputFile, closeErr) + } } else if r.options.OutputDirectory != "" { outputFile := path.Join(r.options.OutputDirectory, domain) if r.options.JSON { @@ -157,7 +161,9 @@ func (r *Runner) EnumerateMultipleDomainsWithCtx(ctx context.Context, reader io. _, err = r.EnumerateSingleDomainWithCtx(ctx, domain, append(writers, file)) - file.Close() + if closeErr := file.Close(); closeErr != nil { + gologger.Error().Msgf("Error closing file %s: %s", outputFile, closeErr) + } } else { _, err = r.EnumerateSingleDomainWithCtx(ctx, domain, writers) } diff --git a/v2/pkg/subscraping/agent.go b/v2/pkg/subscraping/agent.go index 0e2b7f13a..4f2cab8c6 100644 --- a/v2/pkg/subscraping/agent.go +++ b/v2/pkg/subscraping/agent.go @@ -119,7 +119,9 @@ func (s *Session) DiscardHTTPResponse(response *http.Response) { gologger.Warning().Msgf("Could not discard response body: %s\n", err) return } - response.Body.Close() + if closeErr := response.Body.Close(); closeErr != nil { + gologger.Warning().Msgf("Could not close response body: %s\n", closeErr) + } } } diff --git a/v2/pkg/subscraping/sources/alienvault/alienvault.go b/v2/pkg/subscraping/sources/alienvault/alienvault.go index 857b2c78a..27e1f57b4 100644 --- a/v2/pkg/subscraping/sources/alienvault/alienvault.go +++ b/v2/pkg/subscraping/sources/alienvault/alienvault.go @@ -51,10 +51,10 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ - resp.Body.Close() + session.DiscardHTTPResponse(resp) return } - resp.Body.Close() + session.DiscardHTTPResponse(resp) if response.Error != "" { results <- subscraping.Result{ diff --git a/v2/pkg/subscraping/sources/anubis/anubis.go b/v2/pkg/subscraping/sources/anubis/anubis.go index 8ab84a3d0..6efbd5343 100644 --- a/v2/pkg/subscraping/sources/anubis/anubis.go +++ b/v2/pkg/subscraping/sources/anubis/anubis.go @@ -40,7 +40,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se } if resp.StatusCode != http.StatusOK { - resp.Body.Close() + session.DiscardHTTPResponse(resp) return } @@ -49,11 +49,11 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ - resp.Body.Close() + session.DiscardHTTPResponse(resp) return } - resp.Body.Close() + session.DiscardHTTPResponse(resp) for _, record := range subdomains { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: record} diff --git a/v2/pkg/subscraping/sources/bevigil/bevigil.go b/v2/pkg/subscraping/sources/bevigil/bevigil.go index 893db4913..2021e4c64 100644 --- a/v2/pkg/subscraping/sources/bevigil/bevigil.go +++ b/v2/pkg/subscraping/sources/bevigil/bevigil.go @@ -59,11 +59,11 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ - resp.Body.Close() + session.DiscardHTTPResponse(resp) return } - resp.Body.Close() + session.DiscardHTTPResponse(resp) if len(response.Subdomains) > 0 { subdomains = response.Subdomains diff --git a/v2/pkg/subscraping/sources/binaryedge/binaryedge.go b/v2/pkg/subscraping/sources/binaryedge/binaryedge.go deleted file mode 100644 index d798fe702..000000000 --- a/v2/pkg/subscraping/sources/binaryedge/binaryedge.go +++ /dev/null @@ -1,194 +0,0 @@ -// Package binaryedge logic -package binaryedge - -import ( - "context" - "errors" - "fmt" - "math" - "net/url" - "strconv" - "time" - - jsoniter "github.com/json-iterator/go" - - "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" -) - -const ( - v1 = "v1" - v2 = "v2" - baseAPIURLFmt = "https://api.binaryedge.io/%s/query/domains/subdomain/%s" - v2SubscriptionURL = "https://api.binaryedge.io/v2/user/subscription" - v1PageSizeParam = "pagesize" - pageParam = "page" - firstPage = 1 - maxV1PageSize = 10000 -) - -type subdomainsResponse struct { - Message string `json:"message"` - Title string `json:"title"` - Status interface{} `json:"status"` // string for v1, int for v2 - Subdomains []string `json:"events"` - Page int `json:"page"` - PageSize int `json:"pagesize"` - Total int `json:"total"` -} - -// Source is the passive scraping agent -type Source struct { - apiKeys []string - timeTaken time.Duration - errors int - results int - skipped bool -} - -// Run function returns all subdomains found with the service -func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { - results := make(chan subscraping.Result) - s.errors = 0 - s.results = 0 - - go func() { - defer func(startTime time.Time) { - s.timeTaken = time.Since(startTime) - close(results) - }(time.Now()) - - randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) - if randomApiKey == "" { - s.skipped = true - return - } - - var baseURL string - - authHeader := map[string]string{"X-Key": randomApiKey} - - if isV2(ctx, session, authHeader) { - baseURL = fmt.Sprintf(baseAPIURLFmt, v2, domain) - } else { - authHeader = map[string]string{"X-Token": randomApiKey} - v1URLWithPageSize, err := addURLParam(fmt.Sprintf(baseAPIURLFmt, v1, domain), v1PageSizeParam, strconv.Itoa(maxV1PageSize)) - if err != nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - s.errors++ - return - } - baseURL = v1URLWithPageSize.String() - } - - if baseURL == "" { - results <- subscraping.Result{ - Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("can't get API URL"), - } - s.errors++ - return - } - - s.enumerate(ctx, session, baseURL, firstPage, authHeader, results) - }() - return results -} - -func (s *Source) enumerate(ctx context.Context, session *subscraping.Session, baseURL string, page int, authHeader map[string]string, results chan subscraping.Result) { - pageURL, err := addURLParam(baseURL, pageParam, strconv.Itoa(page)) - if err != nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - s.errors++ - return - } - - resp, err := session.Get(ctx, pageURL.String(), "", authHeader) - if err != nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - s.errors++ - session.DiscardHTTPResponse(resp) - return - } - - var response subdomainsResponse - err = jsoniter.NewDecoder(resp.Body).Decode(&response) - if err != nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - s.errors++ - resp.Body.Close() - return - } - - // Check error messages - if response.Message != "" && response.Status != nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: errors.New(response.Message)} - s.errors++ - return - } - - resp.Body.Close() - - for _, subdomain := range response.Subdomains { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} - s.results++ - } - - totalPages := int(math.Ceil(float64(response.Total) / float64(response.PageSize))) - nextPage := response.Page + 1 - if nextPage <= totalPages { - s.enumerate(ctx, session, baseURL, nextPage, authHeader, results) - } -} - -// Name returns the name of the source -func (s *Source) Name() string { - return "binaryedge" -} - -func (s *Source) IsDefault() bool { - return false -} - -func (s *Source) HasRecursiveSupport() bool { - return true -} - -func (s *Source) NeedsKey() bool { - return true -} - -func (s *Source) AddApiKeys(keys []string) { - s.apiKeys = keys -} - -func (s *Source) Statistics() subscraping.Statistics { - return subscraping.Statistics{ - Errors: s.errors, - Results: s.results, - TimeTaken: s.timeTaken, - Skipped: s.skipped, - } -} - -func isV2(ctx context.Context, session *subscraping.Session, authHeader map[string]string) bool { - resp, err := session.Get(ctx, v2SubscriptionURL, "", authHeader) - if err != nil { - session.DiscardHTTPResponse(resp) - return false - } - - resp.Body.Close() - - return true -} - -func addURLParam(targetURL, name, value string) (*url.URL, error) { - u, err := url.Parse(targetURL) - if err != nil { - return u, err - } - q, _ := url.ParseQuery(u.RawQuery) - q.Add(name, value) - u.RawQuery = q.Encode() - - return u, nil -} diff --git a/v2/pkg/subscraping/sources/bufferover/bufferover.go b/v2/pkg/subscraping/sources/bufferover/bufferover.go index bbfc884a3..207cb8190 100644 --- a/v2/pkg/subscraping/sources/bufferover/bufferover.go +++ b/v2/pkg/subscraping/sources/bufferover/bufferover.go @@ -69,11 +69,11 @@ func (s *Source) getData(ctx context.Context, sourceURL string, apiKey string, s if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ - resp.Body.Close() + session.DiscardHTTPResponse(resp) return } - resp.Body.Close() + session.DiscardHTTPResponse(resp) metaErrors := bufforesponse.Meta.Errors diff --git a/v2/pkg/subscraping/sources/builtwith/builtwith.go b/v2/pkg/subscraping/sources/builtwith/builtwith.go index 24f2e60cc..4c6b516f6 100644 --- a/v2/pkg/subscraping/sources/builtwith/builtwith.go +++ b/v2/pkg/subscraping/sources/builtwith/builtwith.go @@ -68,10 +68,10 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ - resp.Body.Close() + session.DiscardHTTPResponse(resp) return } - resp.Body.Close() + session.DiscardHTTPResponse(resp) for _, result := range data.Results { for _, path := range result.Result.Paths { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: fmt.Sprintf("%s.%s", path.SubDomain, path.Domain)} diff --git a/v2/pkg/subscraping/sources/c99/c99.go b/v2/pkg/subscraping/sources/c99/c99.go index bfd7fcf55..c2e45e818 100644 --- a/v2/pkg/subscraping/sources/c99/c99.go +++ b/v2/pkg/subscraping/sources/c99/c99.go @@ -56,7 +56,12 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se return } - defer resp.Body.Close() + defer func() { + if err := resp.Body.Close(); err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + } + }() var response dnsdbLookupResponse err = jsoniter.NewDecoder(resp.Body).Decode(&response) diff --git a/v2/pkg/subscraping/sources/censys/censys.go b/v2/pkg/subscraping/sources/censys/censys.go index 04ebe48e5..4427821ae 100644 --- a/v2/pkg/subscraping/sources/censys/censys.go +++ b/v2/pkg/subscraping/sources/censys/censys.go @@ -124,11 +124,11 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ - resp.Body.Close() + session.DiscardHTTPResponse(resp) return } - resp.Body.Close() + session.DiscardHTTPResponse(resp) for _, hit := range censysResponse.Result.Hits { for _, name := range hit.Names { diff --git a/v2/pkg/subscraping/sources/certspotter/certspotter.go b/v2/pkg/subscraping/sources/certspotter/certspotter.go index cf49d1dc9..e501e1730 100644 --- a/v2/pkg/subscraping/sources/certspotter/certspotter.go +++ b/v2/pkg/subscraping/sources/certspotter/certspotter.go @@ -59,10 +59,10 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ - resp.Body.Close() + session.DiscardHTTPResponse(resp) return } - resp.Body.Close() + session.DiscardHTTPResponse(resp) for _, cert := range response { for _, subdomain := range cert.DNSNames { @@ -92,10 +92,10 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ - resp.Body.Close() + session.DiscardHTTPResponse(resp) return } - resp.Body.Close() + session.DiscardHTTPResponse(resp) if len(response) == 0 { break diff --git a/v2/pkg/subscraping/sources/chinaz/chinaz.go b/v2/pkg/subscraping/sources/chinaz/chinaz.go index d7063973f..c6e43c05d 100644 --- a/v2/pkg/subscraping/sources/chinaz/chinaz.go +++ b/v2/pkg/subscraping/sources/chinaz/chinaz.go @@ -48,7 +48,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se body, err := io.ReadAll(resp.Body) - resp.Body.Close() + session.DiscardHTTPResponse(resp) SubdomainList := jsoniter.Get(body, "Result").Get("ContributingSubdomainList") diff --git a/v2/pkg/subscraping/sources/commoncrawl/commoncrawl.go b/v2/pkg/subscraping/sources/commoncrawl/commoncrawl.go index 34e50be9e..09ad4967d 100644 --- a/v2/pkg/subscraping/sources/commoncrawl/commoncrawl.go +++ b/v2/pkg/subscraping/sources/commoncrawl/commoncrawl.go @@ -59,13 +59,13 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ - resp.Body.Close() + session.DiscardHTTPResponse(resp) return } - resp.Body.Close() + session.DiscardHTTPResponse(resp) years := make([]string, 0) - for i := 0; i < maxYearsBack; i++ { + for i := range maxYearsBack { years = append(years, strconv.Itoa(year-i)) } @@ -155,7 +155,7 @@ func (s *Source) getSubdomains(ctx context.Context, searchURL, domain string, se } } } - resp.Body.Close() + session.DiscardHTTPResponse(resp) return true } } diff --git a/v2/pkg/subscraping/sources/crtsh/crtsh.go b/v2/pkg/subscraping/sources/crtsh/crtsh.go index 4d008349f..76ac02b45 100644 --- a/v2/pkg/subscraping/sources/crtsh/crtsh.go +++ b/v2/pkg/subscraping/sources/crtsh/crtsh.go @@ -14,6 +14,7 @@ import ( // postgres driver _ "github.com/lib/pq" + "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" contextutil "github.com/projectdiscovery/utils/context" ) @@ -60,7 +61,11 @@ func (s *Source) getSubdomainsFromSQL(ctx context.Context, domain string, sessio return 0 } - defer db.Close() + defer func() { + if closeErr := db.Close(); closeErr != nil { + gologger.Warning().Msgf("Could not close database connection: %s\n", closeErr) + } + }() limitClause := "" if all, ok := ctx.Value(contextutil.ContextArg("All")).(contextutil.ContextArg); ok { @@ -119,7 +124,7 @@ func (s *Source) getSubdomainsFromSQL(ctx context.Context, domain string, sessio } count++ - for _, subdomain := range strings.Split(data, "\n") { + for subdomain := range strings.SplitSeq(data, "\n") { for _, value := range session.Extractor.Extract(subdomain) { if value != "" { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: value} @@ -145,14 +150,14 @@ func (s *Source) getSubdomainsFromHTTP(ctx context.Context, domain string, sessi if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ - resp.Body.Close() + session.DiscardHTTPResponse(resp) return false } - resp.Body.Close() + session.DiscardHTTPResponse(resp) for _, subdomain := range subdomains { - for _, sub := range strings.Split(subdomain.NameValue, "\n") { + for sub := range strings.SplitSeq(subdomain.NameValue, "\n") { for _, value := range session.Extractor.Extract(sub) { if value != "" { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: value} diff --git a/v2/pkg/subscraping/sources/digitalyama/digitalyama.go b/v2/pkg/subscraping/sources/digitalyama/digitalyama.go index eaa6f8420..7ab719576 100644 --- a/v2/pkg/subscraping/sources/digitalyama/digitalyama.go +++ b/v2/pkg/subscraping/sources/digitalyama/digitalyama.go @@ -54,7 +54,12 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se s.errors++ return } - defer resp.Body.Close() + defer func() { + if err := resp.Body.Close(); err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + } + }() if resp.StatusCode != 200 { var errResponse struct { diff --git a/v2/pkg/subscraping/sources/digitorus/digitorus.go b/v2/pkg/subscraping/sources/digitorus/digitorus.go index 6cf61dbcf..8c680a2b6 100644 --- a/v2/pkg/subscraping/sources/digitorus/digitorus.go +++ b/v2/pkg/subscraping/sources/digitorus/digitorus.go @@ -41,7 +41,12 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se return } - defer resp.Body.Close() + defer func() { + if err := resp.Body.Close(); err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + } + }() scanner := bufio.NewScanner(resp.Body) for scanner.Scan() { diff --git a/v2/pkg/subscraping/sources/dnsdb/dnsdb.go b/v2/pkg/subscraping/sources/dnsdb/dnsdb.go index 2b18b8568..e5dc0e246 100644 --- a/v2/pkg/subscraping/sources/dnsdb/dnsdb.go +++ b/v2/pkg/subscraping/sources/dnsdb/dnsdb.go @@ -104,7 +104,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se } else if err != nil { results <- subscraping.Result{Source: sourceName, Type: subscraping.Error, Error: err} s.errors++ - resp.Body.Close() + session.DiscardHTTPResponse(resp) return } @@ -113,7 +113,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se if err != nil { results <- subscraping.Result{Source: sourceName, Type: subscraping.Error, Error: err} s.errors++ - resp.Body.Close() + session.DiscardHTTPResponse(resp) return } @@ -154,7 +154,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se s.errors++ } - resp.Body.Close() + session.DiscardHTTPResponse(resp) break } }() diff --git a/v2/pkg/subscraping/sources/dnsdumpster/dnsdumpster.go b/v2/pkg/subscraping/sources/dnsdumpster/dnsdumpster.go index 2155e31cf..72be41c3b 100644 --- a/v2/pkg/subscraping/sources/dnsdumpster/dnsdumpster.go +++ b/v2/pkg/subscraping/sources/dnsdumpster/dnsdumpster.go @@ -53,14 +53,14 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se session.DiscardHTTPResponse(resp) return } - defer resp.Body.Close() + defer session.DiscardHTTPResponse(resp) var response response err = json.NewDecoder(resp.Body).Decode(&response) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ - resp.Body.Close() + session.DiscardHTTPResponse(resp) return } diff --git a/v2/pkg/subscraping/sources/dnsrepo/dnsrepo.go b/v2/pkg/subscraping/sources/dnsrepo/dnsrepo.go index 75cf859bd..e98c82399 100644 --- a/v2/pkg/subscraping/sources/dnsrepo/dnsrepo.go +++ b/v2/pkg/subscraping/sources/dnsrepo/dnsrepo.go @@ -64,7 +64,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se session.DiscardHTTPResponse(resp) return } - resp.Body.Close() + session.DiscardHTTPResponse(resp) var result DnsRepoResponse err = json.Unmarshal(responseData, &result) if err != nil { diff --git a/v2/pkg/subscraping/sources/facebook/ctlogs.go b/v2/pkg/subscraping/sources/facebook/ctlogs.go index 3f19af736..f96436008 100644 --- a/v2/pkg/subscraping/sources/facebook/ctlogs.go +++ b/v2/pkg/subscraping/sources/facebook/ctlogs.go @@ -45,7 +45,11 @@ func (k *apiKey) FetchAccessToken() { k.Error = err return } - defer resp.Body.Close() + defer func() { + if err := resp.Body.Close(); err != nil { + gologger.Error().Msgf("error closing response body: %s", err) + } + }() bin, err := io.ReadAll(resp.Body) if err != nil { k.Error = err @@ -113,7 +117,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} return } - resp.Body.Close() + session.DiscardHTTPResponse(resp) response := &response{} if err := json.Unmarshal(bin, response); err != nil { s.errors++ diff --git a/v2/pkg/subscraping/sources/facebook/ctlogs_test.go b/v2/pkg/subscraping/sources/facebook/ctlogs_test.go index 8869e7901..e9204055f 100644 --- a/v2/pkg/subscraping/sources/facebook/ctlogs_test.go +++ b/v2/pkg/subscraping/sources/facebook/ctlogs_test.go @@ -8,6 +8,7 @@ import ( "strings" "testing" + "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/retryablehttp-go" "github.com/projectdiscovery/utils/generic" ) @@ -41,7 +42,11 @@ func TestFacebookSource(t *testing.T) { if err != nil { t.Fatal(err) } - defer resp.Body.Close() + defer func() { + if err := resp.Body.Close(); err != nil { + gologger.Error().Msgf("error closing response body: %s", err) + } + }() bin, err := io.ReadAll(resp.Body) if err != nil { t.Fatal(err) diff --git a/v2/pkg/subscraping/sources/fofa/fofa.go b/v2/pkg/subscraping/sources/fofa/fofa.go index 9a5294c75..78be69d1f 100644 --- a/v2/pkg/subscraping/sources/fofa/fofa.go +++ b/v2/pkg/subscraping/sources/fofa/fofa.go @@ -54,7 +54,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se } // fofa api doc https://fofa.info/static_pages/api_help - qbase64 := base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("domain=\"%s\"", domain))) + qbase64 := base64.StdEncoding.EncodeToString(fmt.Appendf(nil, "domain=\"%s\"", domain)) resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://fofa.info/api/v1/search/all?full=true&fields=host&page=1&size=10000&email=%s&key=%s&qbase64=%s", randomApiKey.username, randomApiKey.secret, qbase64)) if err != nil && resp == nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} @@ -68,10 +68,10 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ - resp.Body.Close() + session.DiscardHTTPResponse(resp) return } - resp.Body.Close() + session.DiscardHTTPResponse(resp) if response.Error { results <- subscraping.Result{ diff --git a/v2/pkg/subscraping/sources/fullhunt/fullhunt.go b/v2/pkg/subscraping/sources/fullhunt/fullhunt.go index 10c054e89..60c64e52d 100644 --- a/v2/pkg/subscraping/sources/fullhunt/fullhunt.go +++ b/v2/pkg/subscraping/sources/fullhunt/fullhunt.go @@ -56,10 +56,10 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ - resp.Body.Close() + session.DiscardHTTPResponse(resp) return } - resp.Body.Close() + session.DiscardHTTPResponse(resp) for _, record := range response.Hosts { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: record} s.results++ diff --git a/v2/pkg/subscraping/sources/github/github.go b/v2/pkg/subscraping/sources/github/github.go index 6034b2dbc..2e8b3224b 100644 --- a/v2/pkg/subscraping/sources/github/github.go +++ b/v2/pkg/subscraping/sources/github/github.go @@ -100,7 +100,7 @@ func (s *Source) enumerate(ctx context.Context, searchURL string, domainRegexp * if isForbidden && ratelimitRemaining == 0 { retryAfterSeconds, _ := strconv.ParseInt(resp.Header.Get("Retry-After"), 10, 64) tokens.setCurrentTokenExceeded(retryAfterSeconds) - resp.Body.Close() + session.DiscardHTTPResponse(resp) s.enumerate(ctx, searchURL, domainRegexp, tokens, session, results) } @@ -112,11 +112,11 @@ func (s *Source) enumerate(ctx context.Context, searchURL string, domainRegexp * if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ - resp.Body.Close() + session.DiscardHTTPResponse(resp) return } - resp.Body.Close() + session.DiscardHTTPResponse(resp) err = s.proccesItems(ctx, data.Items, domainRegexp, s.Name(), session, results) if err != nil { @@ -173,7 +173,7 @@ func (s *Source) proccesItems(ctx context.Context, items []item, domainRegexp *r s.results++ } } - resp.Body.Close() + session.DiscardHTTPResponse(resp) } // find subdomains in text matches diff --git a/v2/pkg/subscraping/sources/gitlab/gitlab.go b/v2/pkg/subscraping/sources/gitlab/gitlab.go index 9477b0bdf..902ef4691 100644 --- a/v2/pkg/subscraping/sources/gitlab/gitlab.go +++ b/v2/pkg/subscraping/sources/gitlab/gitlab.go @@ -74,7 +74,7 @@ func (s *Source) enumerate(ctx context.Context, searchURL string, domainRegexp * return } - defer resp.Body.Close() + defer session.DiscardHTTPResponse(resp) var items []item err = jsoniter.NewDecoder(resp.Body).Decode(&items) @@ -114,7 +114,7 @@ func (s *Source) enumerate(ctx context.Context, searchURL string, domainRegexp * s.results++ } } - resp.Body.Close() + session.DiscardHTTPResponse(resp) } defer wg.Done() }(it) diff --git a/v2/pkg/subscraping/sources/hackertarget/hackertarget.go b/v2/pkg/subscraping/sources/hackertarget/hackertarget.go index de20cbbf0..c84957722 100644 --- a/v2/pkg/subscraping/sources/hackertarget/hackertarget.go +++ b/v2/pkg/subscraping/sources/hackertarget/hackertarget.go @@ -37,7 +37,12 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se return } - defer resp.Body.Close() + defer func() { + if err := resp.Body.Close(); err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + } + }() scanner := bufio.NewScanner(resp.Body) for scanner.Scan() { diff --git a/v2/pkg/subscraping/sources/hudsonrock/hudsonrock.go b/v2/pkg/subscraping/sources/hudsonrock/hudsonrock.go index b109a3192..51d6bc2fa 100644 --- a/v2/pkg/subscraping/sources/hudsonrock/hudsonrock.go +++ b/v2/pkg/subscraping/sources/hudsonrock/hudsonrock.go @@ -47,14 +47,14 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se session.DiscardHTTPResponse(resp) return } - defer resp.Body.Close() + defer session.DiscardHTTPResponse(resp) var response hudsonrockResponse err = json.NewDecoder(resp.Body).Decode(&response) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ - resp.Body.Close() + session.DiscardHTTPResponse(resp) return } diff --git a/v2/pkg/subscraping/sources/hunter/hunter.go b/v2/pkg/subscraping/sources/hunter/hunter.go index dc3b77021..960563279 100644 --- a/v2/pkg/subscraping/sources/hunter/hunter.go +++ b/v2/pkg/subscraping/sources/hunter/hunter.go @@ -59,8 +59,8 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se var pages = 1 for currentPage := 1; currentPage <= pages; currentPage++ { // hunter api doc https://hunter.qianxin.com/home/helpCenter?r=5-1-2 - qbase64 := base64.URLEncoding.EncodeToString([]byte(fmt.Sprintf("domain=\"%s\"", domain))) - resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://hunter.qianxin.com/openApi/search?api-key=%s&search=%s&page=1&page_size=100&is_web=3", randomApiKey, qbase64)) + qbase64 := base64.URLEncoding.EncodeToString(fmt.Appendf(nil, "domain=\"%s\"", domain)) + resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://hunter.qianxin.com/openApi/search?api-key=%s&search=%s&page=%d&page_size=100&is_web=3", randomApiKey, qbase64, currentPage)) if err != nil && resp == nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ @@ -73,10 +73,10 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ - resp.Body.Close() + session.DiscardHTTPResponse(resp) return } - resp.Body.Close() + session.DiscardHTTPResponse(resp) if response.Code == 401 || response.Code == 400 { results <- subscraping.Result{ diff --git a/v2/pkg/subscraping/sources/intelx/intelx.go b/v2/pkg/subscraping/sources/intelx/intelx.go index c08b3ff3c..b3a3a45a1 100644 --- a/v2/pkg/subscraping/sources/intelx/intelx.go +++ b/v2/pkg/subscraping/sources/intelx/intelx.go @@ -98,11 +98,11 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ - resp.Body.Close() + session.DiscardHTTPResponse(resp) return } - resp.Body.Close() + session.DiscardHTTPResponse(resp) resultsURL := fmt.Sprintf("https://%s/phonebook/search/result?k=%s&id=%s&limit=10000", randomApiKey.host, randomApiKey.key, response.ID) status := 0 @@ -119,7 +119,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ - resp.Body.Close() + session.DiscardHTTPResponse(resp) return } @@ -127,10 +127,10 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ - resp.Body.Close() + session.DiscardHTTPResponse(resp) return } - resp.Body.Close() + session.DiscardHTTPResponse(resp) status = response.Status for _, hostname := range response.Selectors { diff --git a/v2/pkg/subscraping/sources/leakix/leakix.go b/v2/pkg/subscraping/sources/leakix/leakix.go index 7061ccd11..4159e6aba 100644 --- a/v2/pkg/subscraping/sources/leakix/leakix.go +++ b/v2/pkg/subscraping/sources/leakix/leakix.go @@ -46,7 +46,9 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se s.errors++ return } - defer resp.Body.Close() + + defer session.DiscardHTTPResponse(resp) + if resp.StatusCode != 200 { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("request failed with status %d", resp.StatusCode)} s.errors++ diff --git a/v2/pkg/subscraping/sources/netlas/netlas.go b/v2/pkg/subscraping/sources/netlas/netlas.go index 4c11a263d..c29bc2c8b 100644 --- a/v2/pkg/subscraping/sources/netlas/netlas.go +++ b/v2/pkg/subscraping/sources/netlas/netlas.go @@ -77,7 +77,12 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se s.errors++ return } - defer resp.Body.Close() + defer func() { + if err := resp.Body.Close(); err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + } + }() body, err := io.ReadAll(resp.Body) if err != nil { @@ -99,7 +104,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se apiUrl := "https://app.netlas.io/api/domains/download/" query := fmt.Sprintf("domain:*.%s AND NOT domain:%s", domain, domain) - requestBody := map[string]interface{}{ + requestBody := map[string]any{ "q": query, "fields": []string{"*"}, "source_type": "include", @@ -124,7 +129,12 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se s.errors++ return } - defer resp.Body.Close() + defer func() { + if err := resp.Body.Close(); err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + } + }() body, err = io.ReadAll(resp.Body) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("error reading ressponse body")} diff --git a/v2/pkg/subscraping/sources/pugrecon/pugrecon.go b/v2/pkg/subscraping/sources/pugrecon/pugrecon.go new file mode 100644 index 000000000..e7f2255f8 --- /dev/null +++ b/v2/pkg/subscraping/sources/pugrecon/pugrecon.go @@ -0,0 +1,152 @@ +// Package pugrecon logic +package pugrecon + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "net/http" + "time" + + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" +) + +// pugreconResult stores a single result from the pugrecon API +type pugreconResult struct { + Name string `json:"name"` +} + +// pugreconAPIResponse stores the response from the pugrecon API +type pugreconAPIResponse struct { + Results []pugreconResult `json:"results"` + QuotaRemaining int `json:"quota_remaining"` + Limited bool `json:"limited"` + TotalResults int `json:"total_results"` + Message string `json:"message"` +} + +// Source is the passive scraping agent +type Source struct { + apiKeys []string + timeTaken time.Duration + errors int + results int + skipped bool +} + +// Run function returns all subdomains found with the service +func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { + results := make(chan subscraping.Result) + s.errors = 0 + s.results = 0 + + go func() { + defer func(startTime time.Time) { + s.timeTaken = time.Since(startTime) + close(results) + }(time.Now()) + + randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) + if randomApiKey == "" { + s.skipped = true + return + } + + // Prepare POST request data + postData := map[string]string{"domain_name": domain} + bodyBytes, err := json.Marshal(postData) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("failed to marshal request body: %w", err)} + s.errors++ + return + } + bodyReader := bytes.NewReader(bodyBytes) + + // Prepare headers + headers := map[string]string{ + "Authorization": "Bearer " + randomApiKey, + "Content-Type": "application/json", + "Accept": "application/json", + } + + apiURL := "https://pugrecon.com/api/v1/domains" + resp, err := session.HTTPRequest(ctx, http.MethodPost, apiURL, "", headers, bodyReader, subscraping.BasicAuth{}) // Use HTTPRequest for full header control + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + session.DiscardHTTPResponse(resp) + return + } + defer func() { + if err := resp.Body.Close(); err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("failed to close response body: %w", err)} + s.errors++ + } + }() + + if resp.StatusCode != http.StatusOK { + errorMsg := fmt.Sprintf("received status code %d", resp.StatusCode) + // Attempt to read error message from body if possible + var apiResp pugreconAPIResponse + if json.NewDecoder(resp.Body).Decode(&apiResp) == nil && apiResp.Message != "" { + errorMsg = fmt.Sprintf("%s: %s", errorMsg, apiResp.Message) + } + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("%s", errorMsg)} + s.errors++ + return + } + + var response pugreconAPIResponse + err = json.NewDecoder(resp.Body).Decode(&response) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + return + } + + for _, subdomain := range response.Results { + results <- subscraping.Result{ + Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain.Name, + } + s.results++ + } + }() + + return results +} + +// Name returns the name of the source +func (s *Source) Name() string { + return "pugrecon" +} + +// IsDefault returns false as this is not a default source. +func (s *Source) IsDefault() bool { + return false +} + +// HasRecursiveSupport returns false as this source does not support recursive searches. +func (s *Source) HasRecursiveSupport() bool { + return false +} + +// NeedsKey returns true as this source requires an API key. +func (s *Source) NeedsKey() bool { + return true +} + +// AddApiKeys adds the API keys for the source. +func (s *Source) AddApiKeys(keys []string) { + s.apiKeys = keys +} + +// Statistics returns the statistics for the source. +func (s *Source) Statistics() subscraping.Statistics { + return subscraping.Statistics{ + Errors: s.errors, + Results: s.results, + TimeTaken: s.timeTaken, + Skipped: s.skipped, + } +} diff --git a/v2/pkg/subscraping/sources/quake/quake.go b/v2/pkg/subscraping/sources/quake/quake.go index 075d35e13..5cac4ed2e 100644 --- a/v2/pkg/subscraping/sources/quake/quake.go +++ b/v2/pkg/subscraping/sources/quake/quake.go @@ -58,44 +58,58 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se } // quake api doc https://quake.360.cn/quake/#/help - var requestBody = []byte(fmt.Sprintf(`{"query":"domain: %s", "include":["service.http.host"], "latest": true, "start":0, "size":500}`, domain)) - resp, err := session.Post(ctx, "https://quake.360.net/api/v3/search/quake_service", "", map[string]string{ - "Content-Type": "application/json", "X-QuakeToken": randomApiKey, - }, bytes.NewReader(requestBody)) - if err != nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - s.errors++ + var pageSize = 500 + var start = 0 + var totalResults = -1 + + for { + var requestBody = fmt.Appendf(nil, `{"query":"domain: %s", "include":["service.http.host"], "latest": true, "size":%d, "start":%d}`, domain, pageSize, start) + resp, err := session.Post(ctx, "https://quake.360.net/api/v3/search/quake_service", "", map[string]string{ + "Content-Type": "application/json", "X-QuakeToken": randomApiKey, + }, bytes.NewReader(requestBody)) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + session.DiscardHTTPResponse(resp) + return + } + + var response quakeResults + err = jsoniter.NewDecoder(resp.Body).Decode(&response) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + session.DiscardHTTPResponse(resp) + return + } session.DiscardHTTPResponse(resp) - return - } - var response quakeResults - err = jsoniter.NewDecoder(resp.Body).Decode(&response) - if err != nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - s.errors++ - resp.Body.Close() - return - } - resp.Body.Close() + if response.Code != 0 { + results <- subscraping.Result{ + Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("%s", response.Message), + } + s.errors++ + return + } - if response.Code != 0 { - results <- subscraping.Result{ - Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("%s", response.Message), + if totalResults == -1 { + totalResults = response.Meta.Pagination.Total } - s.errors++ - return - } - if response.Meta.Pagination.Total > 0 { for _, quakeDomain := range response.Data { subdomain := quakeDomain.Service.HTTP.Host if strings.ContainsAny(subdomain, "暂无权限") { - subdomain = "" + continue } results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} s.results++ } + + if len(response.Data) == 0 || start+pageSize >= totalResults { + break + } + + start += pageSize } }() diff --git a/v2/pkg/subscraping/sources/rapiddns/rapiddns.go b/v2/pkg/subscraping/sources/rapiddns/rapiddns.go index 5daae6153..89ec70632 100644 --- a/v2/pkg/subscraping/sources/rapiddns/rapiddns.go +++ b/v2/pkg/subscraping/sources/rapiddns/rapiddns.go @@ -48,11 +48,11 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ - resp.Body.Close() + session.DiscardHTTPResponse(resp) return } - resp.Body.Close() + session.DiscardHTTPResponse(resp) src := string(body) for _, subdomain := range session.Extractor.Extract(src) { diff --git a/v2/pkg/subscraping/sources/reconcloud/reconcloud.go b/v2/pkg/subscraping/sources/reconcloud/reconcloud.go index 5a638b5f4..8aacc7d4e 100644 --- a/v2/pkg/subscraping/sources/reconcloud/reconcloud.go +++ b/v2/pkg/subscraping/sources/reconcloud/reconcloud.go @@ -56,10 +56,10 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ - resp.Body.Close() + session.DiscardHTTPResponse(resp) return } - resp.Body.Close() + session.DiscardHTTPResponse(resp) if len(response.CloudAssetsList) > 0 { for _, cloudAsset := range response.CloudAssetsList { diff --git a/v2/pkg/subscraping/sources/redhuntlabs/redhuntlabs.go b/v2/pkg/subscraping/sources/redhuntlabs/redhuntlabs.go index ecbdcace2..a07e9a5b4 100644 --- a/v2/pkg/subscraping/sources/redhuntlabs/redhuntlabs.go +++ b/v2/pkg/subscraping/sources/redhuntlabs/redhuntlabs.go @@ -67,12 +67,12 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se err = jsoniter.NewDecoder(resp.Body).Decode(&response) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - resp.Body.Close() + session.DiscardHTTPResponse(resp) s.errors++ return } - resp.Body.Close() + session.DiscardHTTPResponse(resp) if response.Metadata.ResultCount > pageSize { totalPages := (response.Metadata.ResultCount + pageSize - 1) / pageSize for page := 1; page <= totalPages; page++ { @@ -88,12 +88,12 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se err = jsoniter.NewDecoder(resp.Body).Decode(&response) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - resp.Body.Close() + session.DiscardHTTPResponse(resp) s.errors++ continue } - resp.Body.Close() + session.DiscardHTTPResponse(resp) for _, subdomain := range response.Subdomains { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} diff --git a/v2/pkg/subscraping/sources/riddler/riddler.go b/v2/pkg/subscraping/sources/riddler/riddler.go index 0afe725eb..9a3401c27 100644 --- a/v2/pkg/subscraping/sources/riddler/riddler.go +++ b/v2/pkg/subscraping/sources/riddler/riddler.go @@ -48,7 +48,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se s.results++ } } - resp.Body.Close() + session.DiscardHTTPResponse(resp) }() return results diff --git a/v2/pkg/subscraping/sources/robtex/robtext.go b/v2/pkg/subscraping/sources/robtex/robtext.go index 5d130a42d..9214363ed 100644 --- a/v2/pkg/subscraping/sources/robtex/robtext.go +++ b/v2/pkg/subscraping/sources/robtex/robtext.go @@ -89,6 +89,8 @@ func enumerate(ctx context.Context, session *subscraping.Session, targetURL stri return results, err } + defer session.DiscardHTTPResponse(resp) + scanner := bufio.NewScanner(resp.Body) for scanner.Scan() { line := scanner.Text() @@ -104,8 +106,6 @@ func enumerate(ctx context.Context, session *subscraping.Session, targetURL stri results = append(results, response) } - resp.Body.Close() - return results, nil } diff --git a/v2/pkg/subscraping/sources/rsecloud/rsecloud.go b/v2/pkg/subscraping/sources/rsecloud/rsecloud.go new file mode 100644 index 000000000..8601b7732 --- /dev/null +++ b/v2/pkg/subscraping/sources/rsecloud/rsecloud.go @@ -0,0 +1,117 @@ +package rsecloud + +import ( + "context" + "fmt" + "time" + + jsoniter "github.com/json-iterator/go" + + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" +) + +type response struct { + Count int `json:"count"` + Data []string `json:"data"` + Page int `json:"page"` + PageSize int `json:"pagesize"` + TotalPages int `json:"total_pages"` +} + +// Source is the passive scraping agent +type Source struct { + apiKeys []string + timeTaken time.Duration + errors int + results int + skipped bool +} + +// Run function returns all subdomains found with the service +func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { + results := make(chan subscraping.Result) + s.errors = 0 + s.results = 0 + + go func() { + defer func(startTime time.Time) { + s.timeTaken = time.Since(startTime) + close(results) + }(time.Now()) + + randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) + if randomApiKey == "" { + s.skipped = true + return + } + + headers := map[string]string{"Content-Type": "application/json", "X-API-Key": randomApiKey} + + fetchSubdomains := func(endpoint string) { + page := 1 + for { + resp, err := session.Get(ctx, fmt.Sprintf("https://api.rsecloud.com/api/v2/subdomains/%s/%s?page=%d", endpoint, domain, page), "", headers) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + session.DiscardHTTPResponse(resp) + return + } + + var rseCloudResponse response + err = jsoniter.NewDecoder(resp.Body).Decode(&rseCloudResponse) + session.DiscardHTTPResponse(resp) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + return + } + + for _, subdomain := range rseCloudResponse.Data { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} + s.results++ + } + + if page >= rseCloudResponse.TotalPages { + break + } + page++ + } + } + + fetchSubdomains("active") + fetchSubdomains("passive") + }() + + return results +} + +// Name returns the name of the source +func (s *Source) Name() string { + return "rsecloud" +} + +func (s *Source) IsDefault() bool { + return true +} + +func (s *Source) HasRecursiveSupport() bool { + return false +} + +func (s *Source) NeedsKey() bool { + return true +} + +func (s *Source) AddApiKeys(keys []string) { + s.apiKeys = keys +} + +func (s *Source) Statistics() subscraping.Statistics { + return subscraping.Statistics{ + Errors: s.errors, + Results: s.results, + TimeTaken: s.timeTaken, + Skipped: s.skipped, + } +} diff --git a/v2/pkg/subscraping/sources/securitytrails/securitytrails.go b/v2/pkg/subscraping/sources/securitytrails/securitytrails.go index e254d9c20..e6d38fe44 100644 --- a/v2/pkg/subscraping/sources/securitytrails/securitytrails.go +++ b/v2/pkg/subscraping/sources/securitytrails/securitytrails.go @@ -60,7 +60,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se var err error if scrollId == "" { - var requestBody = []byte(fmt.Sprintf(`{"query":"apex_domain='%s'"}`, domain)) + var requestBody = fmt.Appendf(nil, `{"query":"apex_domain='%s'"}`, domain) resp, err = session.Post(ctx, "https://api.securitytrails.com/v1/domains/list?include_ips=false&scroll=true", "", headers, bytes.NewReader(requestBody)) } else { @@ -83,11 +83,11 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ - resp.Body.Close() + session.DiscardHTTPResponse(resp) return } - resp.Body.Close() + session.DiscardHTTPResponse(resp) for _, record := range securityTrailsResponse.Records { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: record.Hostname} diff --git a/v2/pkg/subscraping/sources/shodan/shodan.go b/v2/pkg/subscraping/sources/shodan/shodan.go index 8d4925d79..d2224a810 100644 --- a/v2/pkg/subscraping/sources/shodan/shodan.go +++ b/v2/pkg/subscraping/sources/shodan/shodan.go @@ -56,7 +56,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se return } - defer resp.Body.Close() + defer session.DiscardHTTPResponse(resp) var response dnsdbLookupResponse err = jsoniter.NewDecoder(resp.Body).Decode(&response) diff --git a/v2/pkg/subscraping/sources/sitedossier/sitedossier.go b/v2/pkg/subscraping/sources/sitedossier/sitedossier.go index dacc3c66b..38d31dd5c 100644 --- a/v2/pkg/subscraping/sources/sitedossier/sitedossier.go +++ b/v2/pkg/subscraping/sources/sitedossier/sitedossier.go @@ -63,10 +63,10 @@ func (s *Source) enumerate(ctx context.Context, session *subscraping.Session, ba if err != nil { results <- subscraping.Result{Source: "sitedossier", Type: subscraping.Error, Error: err} s.errors++ - resp.Body.Close() + session.DiscardHTTPResponse(resp) return } - resp.Body.Close() + session.DiscardHTTPResponse(resp) src := string(body) for _, subdomain := range session.Extractor.Extract(src) { diff --git a/v2/pkg/subscraping/sources/threatbook/threatbook.go b/v2/pkg/subscraping/sources/threatbook/threatbook.go index 1c10a8d10..0033a4d32 100644 --- a/v2/pkg/subscraping/sources/threatbook/threatbook.go +++ b/v2/pkg/subscraping/sources/threatbook/threatbook.go @@ -64,10 +64,10 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ - resp.Body.Close() + session.DiscardHTTPResponse(resp) return } - resp.Body.Close() + session.DiscardHTTPResponse(resp) if response.ResponseCode != 0 { results <- subscraping.Result{ diff --git a/v2/pkg/subscraping/sources/threatcrowd/threatcrowd.go b/v2/pkg/subscraping/sources/threatcrowd/threatcrowd.go index cce9e05db..e09ab91ef 100644 --- a/v2/pkg/subscraping/sources/threatcrowd/threatcrowd.go +++ b/v2/pkg/subscraping/sources/threatcrowd/threatcrowd.go @@ -51,7 +51,12 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se s.errors++ return } - defer resp.Body.Close() + defer func() { + if err := resp.Body.Close(); err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + } + }() if resp.StatusCode != http.StatusOK { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("unexpected status code: %d", resp.StatusCode)} diff --git a/v2/pkg/subscraping/sources/threatminer/threatminer.go b/v2/pkg/subscraping/sources/threatminer/threatminer.go index adffabd89..6776f9ef9 100644 --- a/v2/pkg/subscraping/sources/threatminer/threatminer.go +++ b/v2/pkg/subscraping/sources/threatminer/threatminer.go @@ -44,7 +44,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se return } - defer resp.Body.Close() + defer session.DiscardHTTPResponse(resp) var data response err = jsoniter.NewDecoder(resp.Body).Decode(&data) diff --git a/v2/pkg/subscraping/sources/virustotal/virustotal.go b/v2/pkg/subscraping/sources/virustotal/virustotal.go index 254965008..7ac4c6c50 100644 --- a/v2/pkg/subscraping/sources/virustotal/virustotal.go +++ b/v2/pkg/subscraping/sources/virustotal/virustotal.go @@ -49,9 +49,9 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se if randomApiKey == "" { return } - var cursor string = "" + var cursor = "" for { - var url string = fmt.Sprintf("https://www.virustotal.com/api/v3/domains/%s/subdomains?limit=40", domain) + var url = fmt.Sprintf("https://www.virustotal.com/api/v3/domains/%s/subdomains?limit=40", domain) if cursor != "" { url = fmt.Sprintf("%s&cursor=%s", url, cursor) } @@ -62,7 +62,12 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se session.DiscardHTTPResponse(resp) return } - defer resp.Body.Close() + defer func() { + if err := resp.Body.Close(); err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + } + }() var data response err = jsoniter.NewDecoder(resp.Body).Decode(&data) diff --git a/v2/pkg/subscraping/sources/waybackarchive/waybackarchive.go b/v2/pkg/subscraping/sources/waybackarchive/waybackarchive.go index 5edc78382..f3cba9809 100644 --- a/v2/pkg/subscraping/sources/waybackarchive/waybackarchive.go +++ b/v2/pkg/subscraping/sources/waybackarchive/waybackarchive.go @@ -39,7 +39,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se return } - defer resp.Body.Close() + defer session.DiscardHTTPResponse(resp) scanner := bufio.NewScanner(resp.Body) for scanner.Scan() { diff --git a/v2/pkg/subscraping/sources/whoisxmlapi/whoisxmlapi.go b/v2/pkg/subscraping/sources/whoisxmlapi/whoisxmlapi.go index eafd340af..18375e3ff 100644 --- a/v2/pkg/subscraping/sources/whoisxmlapi/whoisxmlapi.go +++ b/v2/pkg/subscraping/sources/whoisxmlapi/whoisxmlapi.go @@ -67,11 +67,11 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ - resp.Body.Close() + session.DiscardHTTPResponse(resp) return } - resp.Body.Close() + session.DiscardHTTPResponse(resp) for _, record := range data.Result.Records { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: record.Domain} diff --git a/v2/pkg/testutils/integration.go b/v2/pkg/testutils/integration.go index 2358cad20..d18230e95 100644 --- a/v2/pkg/testutils/integration.go +++ b/v2/pkg/testutils/integration.go @@ -27,8 +27,8 @@ func RunSubfinderAndGetResults(debug bool, domain string, extra ...string) ([]st return nil, err } var parts []string - items := strings.Split(string(data), "\n") - for _, i := range items { + items := strings.SplitSeq(string(data), "\n") + for i := range items { if i != "" { parts = append(parts, i) }