diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index 54bac718b..e6fabe953 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -22,13 +22,12 @@ jobs: - uses: actions/checkout@v4 - uses: projectdiscovery/actions/setup/go@v1 with: - go-version-file: v2/go.mod + go-version-file: go.mod - name: Run golangci-lint uses: golangci/golangci-lint-action@v8 with: version: latest args: --timeout 5m - working-directory: v2/ build: name: Test Builds @@ -36,14 +35,13 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-latest, windows-latest, macOS-13] + os: [ubuntu-latest, windows-latest, macOS-latest] steps: - uses: actions/checkout@v4 - uses: projectdiscovery/actions/setup/go@v1 with: - go-version-file: v2/go.mod + go-version-file: go.mod - run: go build ./... - working-directory: v2/ - name: Run tests env: @@ -75,12 +73,11 @@ jobs: with: timeout_seconds: 360 max_attempts: 3 - command: cd v2; go test ./... -v ${{ github.event.inputs.short == 'true' && '-short' || '' }} + command: go test ./... -v ${{ github.event.inputs.short == 'true' && '-short' || '' }} - name: Race Condition Tests run: go build -race ./... - working-directory: v2/ - name: Run Example run: go run . - working-directory: v2/examples + working-directory: examples diff --git a/.github/workflows/compat-checks.yaml b/.github/workflows/compat-checks.yaml index 0b28347be..8efb822d2 100644 --- a/.github/workflows/compat-checks.yaml +++ b/.github/workflows/compat-checks.yaml @@ -16,5 +16,5 @@ jobs: - uses: actions/checkout@v4 - uses: projectdiscovery/actions/setup/go/compat-checks@master with: - go-version-file: 'v2/go.mod' + go-version-file: 'go.mod' diff --git a/.github/workflows/dockerhub-push.yml b/.github/workflows/dockerhub-push.yml index 8c328ce4b..4409dcc30 100644 --- a/.github/workflows/dockerhub-push.yml +++ b/.github/workflows/dockerhub-push.yml @@ -37,4 +37,4 @@ jobs: context: . platforms: linux/amd64,linux/arm64,linux/arm push: true - tags: projectdiscovery/subfinder:latest,projectdiscovery/subfinder:${{ steps.meta.outputs.TAG }} \ No newline at end of file + tags: projectdiscovery/subfinder:latest,projectdiscovery/subfinder:${{ steps.meta.outputs.TAG }} diff --git a/.github/workflows/release-binary.yml b/.github/workflows/release-binary.yml index 487b3f5d3..ea68f5d0b 100644 --- a/.github/workflows/release-binary.yml +++ b/.github/workflows/release-binary.yml @@ -25,9 +25,8 @@ jobs: with: args: "release --clean" version: latest - workdir: v2/ env: GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" SLACK_WEBHOOK: "${{ secrets.RELEASE_SLACK_WEBHOOK }}" DISCORD_WEBHOOK_ID: "${{ secrets.DISCORD_WEBHOOK_ID }}" - DISCORD_WEBHOOK_TOKEN: "${{ secrets.DISCORD_WEBHOOK_TOKEN }}" \ No newline at end of file + DISCORD_WEBHOOK_TOKEN: "${{ secrets.DISCORD_WEBHOOK_TOKEN }}" diff --git a/.github/workflows/release-test.yml b/.github/workflows/release-test.yml index d5b099447..1c8fbbf8b 100644 --- a/.github/workflows/release-test.yml +++ b/.github/workflows/release-test.yml @@ -26,4 +26,3 @@ jobs: with: args: "release --clean --snapshot" version: latest - workdir: v2 \ No newline at end of file diff --git a/v2/.goreleaser.yml b/.goreleaser.yml similarity index 100% rename from v2/.goreleaser.yml rename to .goreleaser.yml diff --git a/Dockerfile b/Dockerfile index 7d2291080..025cab042 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,7 +3,6 @@ FROM golang:1.24-alpine AS build-env RUN apk add build-base WORKDIR /app COPY . /app -WORKDIR /app/v2 RUN go mod download RUN go build ./cmd/subfinder @@ -11,6 +10,6 @@ RUN go build ./cmd/subfinder FROM alpine:latest RUN apk upgrade --no-cache \ && apk add --no-cache bind-tools ca-certificates -COPY --from=build-env /app/v2/subfinder /usr/local/bin/ +COPY --from=build-env /app/subfinder /usr/local/bin/ ENTRYPOINT ["subfinder"] diff --git a/v2/Makefile b/Makefile similarity index 100% rename from v2/Makefile rename to Makefile diff --git a/README.md b/README.md index 4a3ce144e..dad545074 100644 --- a/README.md +++ b/README.md @@ -109,6 +109,13 @@ OPTIMIZATION: -max-time int minutes to wait for enumeration results (default 10) ``` +## Environment Variables + +Subfinder supports environment variables to specify custom paths for configuration files: + +- `SUBFINDER_CONFIG` - Path to config.yaml file (overrides default `$CONFIG/subfinder/config.yaml`) +- `SUBFINDER_PROVIDER_CONFIG` - Path to provider-config.yaml file (overrides default `$CONFIG/subfinder/provider-config.yaml`) + # Installation `subfinder` requires **go1.24** to install successfully. Run the following command to install the latest version: @@ -129,7 +136,7 @@ Learn about how to run Subfinder here: https://docs.projectdiscovery.io/tools/su ## Subfinder Go library -Subfinder can also be used as library and a minimal examples of using subfinder SDK is available [here](v2/examples/main.go) +Subfinder can also be used as library and a minimal examples of using subfinder SDK is available [here](examples/main.go) diff --git a/v2/cmd/subfinder/main.go b/cmd/subfinder/main.go similarity index 100% rename from v2/cmd/subfinder/main.go rename to cmd/subfinder/main.go diff --git a/v2/examples/main.go b/examples/main.go similarity index 100% rename from v2/examples/main.go rename to examples/main.go diff --git a/v2/go.mod b/go.mod similarity index 89% rename from v2/go.mod rename to go.mod index fdb011821..e09106058 100644 --- a/v2/go.mod +++ b/go.mod @@ -14,8 +14,8 @@ require ( github.com/projectdiscovery/fdmax v0.0.4 github.com/projectdiscovery/gologger v1.1.54 github.com/projectdiscovery/ratelimit v0.0.81 - github.com/projectdiscovery/retryablehttp-go v1.0.114 - github.com/projectdiscovery/utils v0.4.20 + github.com/projectdiscovery/retryablehttp-go v1.0.115 + github.com/projectdiscovery/utils v0.4.21 github.com/rs/xid v1.5.0 github.com/stretchr/testify v1.10.0 github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 @@ -45,13 +45,12 @@ require ( github.com/cheggaaa/pb/v3 v3.1.4 // indirect github.com/cloudflare/circl v1.6.1 // indirect github.com/dimchansky/utfbom v1.1.1 // indirect - github.com/dlclark/regexp2 v1.11.4 // indirect + github.com/dlclark/regexp2 v1.11.5 // indirect github.com/docker/go-units v0.5.0 // indirect github.com/dsnet/compress v0.0.2-0.20230904184137-39efe44ab707 // indirect github.com/fatih/color v1.15.0 // indirect github.com/gaissmai/bart v0.20.4 // indirect github.com/go-ole/go-ole v1.2.6 // indirect - github.com/golang/protobuf v1.5.3 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/google/go-github/v30 v30.1.0 // indirect github.com/google/go-querystring v1.1.0 // indirect @@ -78,11 +77,11 @@ require ( github.com/pierrec/lz4/v4 v4.1.21 // indirect github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect github.com/projectdiscovery/blackrock v0.0.1 // indirect - github.com/projectdiscovery/cdncheck v1.1.23 // indirect + github.com/projectdiscovery/cdncheck v1.1.24 // indirect github.com/projectdiscovery/fastdialer v0.4.1 // indirect - github.com/projectdiscovery/hmap v0.0.89 // indirect + github.com/projectdiscovery/hmap v0.0.90 // indirect github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983 // indirect - github.com/projectdiscovery/networkpolicy v0.1.15 // indirect + github.com/projectdiscovery/networkpolicy v0.1.16 // indirect github.com/refraction-networking/utls v1.7.0 // indirect github.com/rivo/uniseg v0.4.7 // indirect github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d // indirect @@ -101,7 +100,7 @@ require ( github.com/tidwall/tinyqueue v0.1.1 // indirect github.com/tklauser/go-sysconf v0.3.12 // indirect github.com/tklauser/numcpus v0.6.1 // indirect - github.com/ulikunitz/xz v0.5.12 // indirect + github.com/ulikunitz/xz v0.5.15 // indirect github.com/weppos/publicsuffix-go v0.30.1 // indirect github.com/yuin/goldmark v1.7.4 // indirect github.com/yuin/goldmark-emoji v1.0.3 // indirect @@ -114,28 +113,26 @@ require ( go4.org v0.0.0-20230225012048-214862532bf5 // indirect golang.org/x/crypto v0.36.0 // indirect golang.org/x/mod v0.22.0 // indirect - golang.org/x/oauth2 v0.11.0 // indirect + golang.org/x/oauth2 v0.27.0 // indirect golang.org/x/sync v0.12.0 // indirect golang.org/x/term v0.30.0 // indirect golang.org/x/text v0.23.0 // indirect golang.org/x/time v0.5.0 // indirect golang.org/x/tools v0.29.0 // indirect - google.golang.org/appengine v1.6.7 // indirect - google.golang.org/protobuf v1.33.0 // indirect gopkg.in/djherbis/times.v1 v1.3.0 // indirect ) require ( github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08 // indirect - github.com/davecgh/go-spew v1.1.1 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/logrusorgru/aurora v2.0.3+incompatible // indirect github.com/miekg/dns v1.1.62 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/pkg/errors v0.9.1 // indirect - github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/projectdiscovery/goflags v0.1.74 - github.com/projectdiscovery/retryabledns v1.0.101 // indirect + github.com/projectdiscovery/retryabledns v1.0.102 // indirect golang.org/x/net v0.38.0 // indirect golang.org/x/sys v0.31.0 // indirect ) diff --git a/v2/go.sum b/go.sum similarity index 96% rename from v2/go.sum rename to go.sum index c0de7afbd..e52607b63 100644 --- a/v2/go.sum +++ b/go.sum @@ -84,12 +84,13 @@ github.com/corpix/uarand v0.2.0 h1:U98xXwud/AVuCpkpgfPF7J5TQgr7R5tqT8VZP5KWbzE= github.com/corpix/uarand v0.2.0/go.mod h1:/3Z1QIqWkDIhf6XWn/08/uMHoQ8JUoTIKc2iPchBOmM= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U= github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE= -github.com/dlclark/regexp2 v1.11.4 h1:rPYF9/LECdNymJufQKmri9gV604RvvABwgOA8un7yAo= -github.com/dlclark/regexp2 v1.11.4/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= +github.com/dlclark/regexp2 v1.11.5 h1:Q/sSnsKerHeCkc/jSTNq1oCm7KiVgUMZRDUoRu0JQZQ= +github.com/dlclark/regexp2 v1.11.5/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/dsnet/compress v0.0.2-0.20230904184137-39efe44ab707 h1:2tV76y6Q9BB+NEBasnqvs7e49aEBFI8ejC89PSnWH+4= @@ -122,8 +123,6 @@ github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5y github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= -github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= @@ -249,14 +248,15 @@ github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFu github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw= github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/projectdiscovery/blackrock v0.0.1 h1:lHQqhaaEFjgf5WkuItbpeCZv2DUIE45k0VbGJyft6LQ= github.com/projectdiscovery/blackrock v0.0.1/go.mod h1:ANUtjDfaVrqB453bzToU+YB4cUbvBRpLvEwoWIwlTss= -github.com/projectdiscovery/cdncheck v1.1.23 h1:LOd6Y7hnV6sXFBs4qGDM0N9xfheAmqLhsfH2cog+M2c= -github.com/projectdiscovery/cdncheck v1.1.23/go.mod h1:dFEGsG0qAJY0AaRr2N1BY0OtZiTxS4kYeT5+OkF8t1U= +github.com/projectdiscovery/cdncheck v1.1.24 h1:6pJ4XnovIrTWzlCJs5/QD1tv6wvK0wiICmmdY0/8WAs= +github.com/projectdiscovery/cdncheck v1.1.24/go.mod h1:dFEGsG0qAJY0AaRr2N1BY0OtZiTxS4kYeT5+OkF8t1U= github.com/projectdiscovery/chaos-client v0.5.2 h1:dN+7GXEypsJAbCD//dBcUxzAEAEH1fjc/7Rf4F/RiNU= github.com/projectdiscovery/chaos-client v0.5.2/go.mod h1:KnoJ/NJPhll42uaqlDga6oafFfNw5l2XI2ajRijtDuU= github.com/projectdiscovery/dnsx v1.2.2 h1:ZjUov0GOyrS8ERlKAAhk+AOkqzaYHBzCP0qZfO+6Ihg= @@ -269,20 +269,20 @@ github.com/projectdiscovery/goflags v0.1.74 h1:n85uTRj5qMosm0PFBfsvOL24I7TdWRcWq github.com/projectdiscovery/goflags v0.1.74/go.mod h1:UMc9/7dFz2oln+10tv6cy+7WZKTHf9UGhaNkF95emh4= github.com/projectdiscovery/gologger v1.1.54 h1:WMzvJ8j/4gGfPKpCttSTaYCVDU1MWQSJnk3wU8/U6Ws= github.com/projectdiscovery/gologger v1.1.54/go.mod h1:vza/8pe2OKOt+ujFWncngknad1XWr8EnLKlbcejOyUE= -github.com/projectdiscovery/hmap v0.0.89 h1:H+XIzk2YcE/9PpW/1N9NdQSrJWm2vthGPNIxSM+WHNU= -github.com/projectdiscovery/hmap v0.0.89/go.mod h1:N3gXFDLN6GqkYsk+2ZkReVOo32OBUV+PNiYyWhWG4ZE= +github.com/projectdiscovery/hmap v0.0.90 h1:p8HWGvPI88hgJoAb4ayR1Oo5VzqPrOCdFG7mASUhQI4= +github.com/projectdiscovery/hmap v0.0.90/go.mod h1:dcjd9P82mkBpFGEy0wBU/3qql5Bx14kmJZvVg7o7vXY= github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983 h1:ZScLodGSezQVwsQDtBSMFp72WDq0nNN+KE/5DHKY5QE= github.com/projectdiscovery/machineid v0.0.0-20240226150047-2e2c51e35983/go.mod h1:3G3BRKui7nMuDFAZKR/M2hiOLtaOmyukT20g88qRQjI= -github.com/projectdiscovery/networkpolicy v0.1.15 h1:jHHPo43s/TSiWmm6T8kJuMqTwL3ukU92iQhxq0K0jg0= -github.com/projectdiscovery/networkpolicy v0.1.15/go.mod h1:GWMDGJmgJ9qGoVTUOxbq1oLIbEx0pPsL0VKlriCkn2g= +github.com/projectdiscovery/networkpolicy v0.1.16 h1:H2VnLmMD7SvxF+rao+639nn8KX/kbPFY+mc8FxeltsI= +github.com/projectdiscovery/networkpolicy v0.1.16/go.mod h1:Vs/IRcJq4QUicjd/tl9gkhQWy7d/LssOwWbaz4buJ0U= github.com/projectdiscovery/ratelimit v0.0.81 h1:u6lW+rAhS/UO0amHTYmYLipPK8NEotA9521hdojBtgI= github.com/projectdiscovery/ratelimit v0.0.81/go.mod h1:tK04WXHuC4i6AsFkByInODSNf45gd9sfaMHzmy2bAsA= -github.com/projectdiscovery/retryabledns v1.0.101 h1:8DIVD8CL34Lc9h6KeOopPUfsPlcFxMlrnKOaI9VeOMk= -github.com/projectdiscovery/retryabledns v1.0.101/go.mod h1:fQI91PKUyTZYL2pYloyA9Bh3Bq8IgOB6X+bN+8Xm14I= -github.com/projectdiscovery/retryablehttp-go v1.0.114 h1:JFvk7RJ2AUrHV9dScHcnyaBpQRGq1d8/QfrpccCT0xc= -github.com/projectdiscovery/retryablehttp-go v1.0.114/go.mod h1:ZXHlpbSw9w3nZqe1LH0GPX2UDAmv2QpUOoafy+xydYs= -github.com/projectdiscovery/utils v0.4.20 h1:7Fmjb+4YZJSzn7bL21sjF3wAR53eSi7VdAfDkDBUUwY= -github.com/projectdiscovery/utils v0.4.20/go.mod h1:RnC23+hI8j4drZFHQpMX92hV9++9d/yBeNr1pzcbF7Y= +github.com/projectdiscovery/retryabledns v1.0.102 h1:R8PzFCVofqLX3Bn4kdjOsE9wZ83FQjXZMDNs4/bHxzI= +github.com/projectdiscovery/retryabledns v1.0.102/go.mod h1:3+GL+YuHpV0Fp6UG7MbIG8mVxXHjfPO5ioQdwlnV08E= +github.com/projectdiscovery/retryablehttp-go v1.0.115 h1:ubIaVyHNj0/qxNv4gar+8/+L3G2Fhpfk54iMDctC7+E= +github.com/projectdiscovery/retryablehttp-go v1.0.115/go.mod h1:XlxLSMBVM7fTXeLVOLjVn1FLuRgQtD49NMFs9sQygfA= +github.com/projectdiscovery/utils v0.4.21 h1:yAothTUSF6NwZ9yoC4iGe5gSBrovqKR9JwwW3msxk3Q= +github.com/projectdiscovery/utils v0.4.21/go.mod h1:HJuJFqjB6EmVaDl0ilFPKvLoMaX2GyE6Il2TqKXNs8I= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/refraction-networking/utls v1.7.0 h1:9JTnze/Md74uS3ZWiRAabityY0un69rOLXsBf8LGgTs= github.com/refraction-networking/utls v1.7.0/go.mod h1:lV0Gwc1/Fi+HYH8hOtgFRdHfKo4FKSn6+FdyOz9hRms= @@ -359,8 +359,8 @@ github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9f github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 h1:nrZ3ySNYwJbSpD6ce9duiP+QkD3JuLCcWkdaehUS/3Y= github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80/go.mod h1:iFyPdL66DjUD96XmzVL3ZntbzcflLnznH0fr99w5VqE= github.com/ulikunitz/xz v0.5.8/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= -github.com/ulikunitz/xz v0.5.12 h1:37Nm15o69RwBkXM0J6A5OlE67RZTfzUxTj8fB3dfcsc= -github.com/ulikunitz/xz v0.5.12/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= +github.com/ulikunitz/xz v0.5.15 h1:9DNdB5s+SgV3bQ2ApL10xRc35ck0DuIX/isZvIk+ubY= +github.com/ulikunitz/xz v0.5.15/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/weppos/publicsuffix-go v0.13.0/go.mod h1:z3LCPQ38eedDQSwmsSRW4Y7t2L8Ln16JPQ02lHAdn5k= github.com/weppos/publicsuffix-go v0.30.1-0.20230422193905-8fecedd899db/go.mod h1:aiQaH1XpzIfgrJq3S1iw7w+3EDbRP7mF5fmwUhWyRUs= github.com/weppos/publicsuffix-go v0.30.1 h1:8q+QwBS1MY56Zjfk/50ycu33NN8aa1iCCEQwo/71Oos= @@ -481,8 +481,8 @@ golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4Iltr golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.5.0/go.mod h1:9/XBHVqLaWO3/BRHs5jbpYCnOZVjj5V0ndyaAM7KB4I= golang.org/x/oauth2 v0.6.0/go.mod h1:ycmewcwgD4Rpr3eZJLSB4Kyyljb3qDh40vJ8STE5HKw= -golang.org/x/oauth2 v0.11.0 h1:vPL4xzxBM4niKCW6g9whtaWVXTJf1U5e4aZxxFx/gbU= -golang.org/x/oauth2 v0.11.0/go.mod h1:LdF7O/8bLR/qWK9DrpXmbHLTouvRHK0SgJl0GmDBchk= +golang.org/x/oauth2 v0.27.0 h1:da9Vo7/tDv5RH/7nZDz1eMGS/q1Vv1N/7FCrBhI9I3M= +golang.org/x/oauth2 v0.27.0/go.mod h1:onh5ek6nERTohokkhCD/y2cV4Do3fxFHFuAejCkRWT8= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -600,7 +600,6 @@ google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7 google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= @@ -626,8 +625,6 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= -google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= diff --git a/v2/pkg/passive/doc.go b/pkg/passive/doc.go similarity index 100% rename from v2/pkg/passive/doc.go rename to pkg/passive/doc.go diff --git a/v2/pkg/passive/passive.go b/pkg/passive/passive.go similarity index 100% rename from v2/pkg/passive/passive.go rename to pkg/passive/passive.go diff --git a/v2/pkg/passive/sources.go b/pkg/passive/sources.go similarity index 98% rename from v2/pkg/passive/sources.go rename to pkg/passive/sources.go index 20013042f..e74d874be 100644 --- a/v2/pkg/passive/sources.go +++ b/pkg/passive/sources.go @@ -26,6 +26,7 @@ import ( "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/dnsdb" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/dnsdumpster" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/dnsrepo" + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/driftnet" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/facebook" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/fofa" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/fullhunt" @@ -70,6 +71,7 @@ var AllSources = [...]subscraping.Source{ &dnsdb.Source{}, &dnsdumpster.Source{}, &dnsrepo.Source{}, + &driftnet.Source{}, &fofa.Source{}, &fullhunt.Source{}, &github.Source{}, diff --git a/v2/pkg/passive/sources_test.go b/pkg/passive/sources_test.go similarity index 98% rename from v2/pkg/passive/sources_test.go rename to pkg/passive/sources_test.go index a63760296..4febbf9fa 100644 --- a/v2/pkg/passive/sources_test.go +++ b/pkg/passive/sources_test.go @@ -26,6 +26,7 @@ var ( "dnsdumpster", "dnsdb", "dnsrepo", + "driftnet", "fofa", "fullhunt", "github", @@ -72,6 +73,7 @@ var ( "digitorus", "dnsdumpster", "dnsrepo", + "driftnet", "fofa", "fullhunt", "hackertarget", @@ -101,6 +103,7 @@ var ( "crtsh", "dnsdb", "digitorus", + "driftnet", "hackertarget", "securitytrails", "virustotal", diff --git a/v2/pkg/passive/sources_w_auth_test.go b/pkg/passive/sources_w_auth_test.go similarity index 100% rename from v2/pkg/passive/sources_w_auth_test.go rename to pkg/passive/sources_w_auth_test.go diff --git a/v2/pkg/passive/sources_wo_auth_test.go b/pkg/passive/sources_wo_auth_test.go similarity index 96% rename from v2/pkg/passive/sources_wo_auth_test.go rename to pkg/passive/sources_wo_auth_test.go index 036588cc3..01fc57d98 100644 --- a/v2/pkg/passive/sources_wo_auth_test.go +++ b/pkg/passive/sources_wo_auth_test.go @@ -33,6 +33,7 @@ func TestSourcesWithoutKeys(t *testing.T) { "digitorus", // failing with "Failed to retrieve certificate" "dnsdumpster", // failing with "unexpected status code 403 received" "anubis", // failing with "too many redirects" + "threatcrowd", // failing with "randomly failing with unmarshal error when hit multiple times" } domain := "hackerone.com" diff --git a/v2/pkg/resolve/client.go b/pkg/resolve/client.go similarity index 100% rename from v2/pkg/resolve/client.go rename to pkg/resolve/client.go diff --git a/v2/pkg/resolve/doc.go b/pkg/resolve/doc.go similarity index 100% rename from v2/pkg/resolve/doc.go rename to pkg/resolve/doc.go diff --git a/v2/pkg/resolve/resolve.go b/pkg/resolve/resolve.go similarity index 100% rename from v2/pkg/resolve/resolve.go rename to pkg/resolve/resolve.go diff --git a/v2/pkg/runner/banners.go b/pkg/runner/banners.go similarity index 97% rename from v2/pkg/runner/banners.go rename to pkg/runner/banners.go index a0d3a7d1c..d02334782 100644 --- a/v2/pkg/runner/banners.go +++ b/pkg/runner/banners.go @@ -17,7 +17,7 @@ const banner = ` const ToolName = `subfinder` // Version is the current version of subfinder -const version = `v2.8.0` +const version = `v2.9.0` // showBanner is used to show the banner to the user func showBanner() { diff --git a/v2/pkg/runner/config.go b/pkg/runner/config.go similarity index 100% rename from v2/pkg/runner/config.go rename to pkg/runner/config.go diff --git a/v2/pkg/runner/doc.go b/pkg/runner/doc.go similarity index 100% rename from v2/pkg/runner/doc.go rename to pkg/runner/doc.go diff --git a/v2/pkg/runner/enumerate.go b/pkg/runner/enumerate.go similarity index 100% rename from v2/pkg/runner/enumerate.go rename to pkg/runner/enumerate.go diff --git a/v2/pkg/runner/enumerate_test.go b/pkg/runner/enumerate_test.go similarity index 100% rename from v2/pkg/runner/enumerate_test.go rename to pkg/runner/enumerate_test.go diff --git a/v2/pkg/runner/initialize.go b/pkg/runner/initialize.go similarity index 100% rename from v2/pkg/runner/initialize.go rename to pkg/runner/initialize.go diff --git a/v2/pkg/runner/options.go b/pkg/runner/options.go similarity index 97% rename from v2/pkg/runner/options.go rename to pkg/runner/options.go index 7f39e0315..a2fd4d411 100644 --- a/v2/pkg/runner/options.go +++ b/pkg/runner/options.go @@ -15,6 +15,7 @@ import ( "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/subfinder/v2/pkg/passive" "github.com/projectdiscovery/subfinder/v2/pkg/resolve" + envutil "github.com/projectdiscovery/utils/env" fileutil "github.com/projectdiscovery/utils/file" folderutil "github.com/projectdiscovery/utils/folder" logutil "github.com/projectdiscovery/utils/log" @@ -23,8 +24,8 @@ import ( var ( configDir = folderutil.AppConfigDirOrDefault(".", "subfinder") - defaultConfigLocation = filepath.Join(configDir, "config.yaml") - defaultProviderConfigLocation = filepath.Join(configDir, "provider-config.yaml") + defaultConfigLocation = envutil.GetEnvOrDefault("SUBFINDER_CONFIG", filepath.Join(configDir, "config.yaml")) + defaultProviderConfigLocation = envutil.GetEnvOrDefault("SUBFINDER_PROVIDER_CONFIG", filepath.Join(configDir, "provider-config.yaml")) ) // Options contains the configuration options for tuning diff --git a/v2/pkg/runner/outputter.go b/pkg/runner/outputter.go similarity index 100% rename from v2/pkg/runner/outputter.go rename to pkg/runner/outputter.go diff --git a/v2/pkg/runner/runner.go b/pkg/runner/runner.go similarity index 100% rename from v2/pkg/runner/runner.go rename to pkg/runner/runner.go diff --git a/v2/pkg/runner/stats.go b/pkg/runner/stats.go similarity index 100% rename from v2/pkg/runner/stats.go rename to pkg/runner/stats.go diff --git a/v2/pkg/runner/util.go b/pkg/runner/util.go similarity index 100% rename from v2/pkg/runner/util.go rename to pkg/runner/util.go diff --git a/v2/pkg/runner/validate.go b/pkg/runner/validate.go similarity index 100% rename from v2/pkg/runner/validate.go rename to pkg/runner/validate.go diff --git a/v2/pkg/subscraping/agent.go b/pkg/subscraping/agent.go similarity index 100% rename from v2/pkg/subscraping/agent.go rename to pkg/subscraping/agent.go diff --git a/v2/pkg/subscraping/doc.go b/pkg/subscraping/doc.go similarity index 100% rename from v2/pkg/subscraping/doc.go rename to pkg/subscraping/doc.go diff --git a/v2/pkg/subscraping/extractor.go b/pkg/subscraping/extractor.go similarity index 100% rename from v2/pkg/subscraping/extractor.go rename to pkg/subscraping/extractor.go diff --git a/v2/pkg/subscraping/sources/alienvault/alienvault.go b/pkg/subscraping/sources/alienvault/alienvault.go similarity index 100% rename from v2/pkg/subscraping/sources/alienvault/alienvault.go rename to pkg/subscraping/sources/alienvault/alienvault.go diff --git a/v2/pkg/subscraping/sources/anubis/anubis.go b/pkg/subscraping/sources/anubis/anubis.go similarity index 100% rename from v2/pkg/subscraping/sources/anubis/anubis.go rename to pkg/subscraping/sources/anubis/anubis.go diff --git a/v2/pkg/subscraping/sources/bevigil/bevigil.go b/pkg/subscraping/sources/bevigil/bevigil.go similarity index 100% rename from v2/pkg/subscraping/sources/bevigil/bevigil.go rename to pkg/subscraping/sources/bevigil/bevigil.go diff --git a/v2/pkg/subscraping/sources/bufferover/bufferover.go b/pkg/subscraping/sources/bufferover/bufferover.go similarity index 100% rename from v2/pkg/subscraping/sources/bufferover/bufferover.go rename to pkg/subscraping/sources/bufferover/bufferover.go diff --git a/v2/pkg/subscraping/sources/builtwith/builtwith.go b/pkg/subscraping/sources/builtwith/builtwith.go similarity index 100% rename from v2/pkg/subscraping/sources/builtwith/builtwith.go rename to pkg/subscraping/sources/builtwith/builtwith.go diff --git a/v2/pkg/subscraping/sources/c99/c99.go b/pkg/subscraping/sources/c99/c99.go similarity index 100% rename from v2/pkg/subscraping/sources/c99/c99.go rename to pkg/subscraping/sources/c99/c99.go diff --git a/v2/pkg/subscraping/sources/censys/censys.go b/pkg/subscraping/sources/censys/censys.go similarity index 100% rename from v2/pkg/subscraping/sources/censys/censys.go rename to pkg/subscraping/sources/censys/censys.go diff --git a/v2/pkg/subscraping/sources/certspotter/certspotter.go b/pkg/subscraping/sources/certspotter/certspotter.go similarity index 100% rename from v2/pkg/subscraping/sources/certspotter/certspotter.go rename to pkg/subscraping/sources/certspotter/certspotter.go diff --git a/v2/pkg/subscraping/sources/chaos/chaos.go b/pkg/subscraping/sources/chaos/chaos.go similarity index 100% rename from v2/pkg/subscraping/sources/chaos/chaos.go rename to pkg/subscraping/sources/chaos/chaos.go diff --git a/v2/pkg/subscraping/sources/chinaz/chinaz.go b/pkg/subscraping/sources/chinaz/chinaz.go similarity index 100% rename from v2/pkg/subscraping/sources/chinaz/chinaz.go rename to pkg/subscraping/sources/chinaz/chinaz.go diff --git a/v2/pkg/subscraping/sources/commoncrawl/commoncrawl.go b/pkg/subscraping/sources/commoncrawl/commoncrawl.go similarity index 100% rename from v2/pkg/subscraping/sources/commoncrawl/commoncrawl.go rename to pkg/subscraping/sources/commoncrawl/commoncrawl.go diff --git a/v2/pkg/subscraping/sources/crtsh/crtsh.go b/pkg/subscraping/sources/crtsh/crtsh.go similarity index 100% rename from v2/pkg/subscraping/sources/crtsh/crtsh.go rename to pkg/subscraping/sources/crtsh/crtsh.go diff --git a/v2/pkg/subscraping/sources/digitalyama/digitalyama.go b/pkg/subscraping/sources/digitalyama/digitalyama.go similarity index 100% rename from v2/pkg/subscraping/sources/digitalyama/digitalyama.go rename to pkg/subscraping/sources/digitalyama/digitalyama.go diff --git a/v2/pkg/subscraping/sources/digitorus/digitorus.go b/pkg/subscraping/sources/digitorus/digitorus.go similarity index 100% rename from v2/pkg/subscraping/sources/digitorus/digitorus.go rename to pkg/subscraping/sources/digitorus/digitorus.go diff --git a/v2/pkg/subscraping/sources/dnsdb/dnsdb.go b/pkg/subscraping/sources/dnsdb/dnsdb.go similarity index 100% rename from v2/pkg/subscraping/sources/dnsdb/dnsdb.go rename to pkg/subscraping/sources/dnsdb/dnsdb.go diff --git a/v2/pkg/subscraping/sources/dnsdumpster/dnsdumpster.go b/pkg/subscraping/sources/dnsdumpster/dnsdumpster.go similarity index 100% rename from v2/pkg/subscraping/sources/dnsdumpster/dnsdumpster.go rename to pkg/subscraping/sources/dnsdumpster/dnsdumpster.go diff --git a/v2/pkg/subscraping/sources/dnsrepo/dnsrepo.go b/pkg/subscraping/sources/dnsrepo/dnsrepo.go similarity index 100% rename from v2/pkg/subscraping/sources/dnsrepo/dnsrepo.go rename to pkg/subscraping/sources/dnsrepo/dnsrepo.go diff --git a/pkg/subscraping/sources/driftnet/driftnet.go b/pkg/subscraping/sources/driftnet/driftnet.go new file mode 100644 index 000000000..758bbfa44 --- /dev/null +++ b/pkg/subscraping/sources/driftnet/driftnet.go @@ -0,0 +1,195 @@ +package driftnet + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/url" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" +) + +const ( + // baseURL is the base URL for the driftnet API + baseURL = "https://api.driftnet.io/v1/" + + // summaryLimit is the size of the summary limit that we send to the API + summaryLimit = 10000 +) + +// Source is the passive scraping agent +type Source struct { + apiKeys []string + timeTaken time.Duration + errors atomic.Int32 + results atomic.Int32 + skipped bool +} + +// endpointConfig describes a driftnet endpoint that can used +type endpointConfig struct { + // The API endpoint to be touched + endpoint string + + // The API parameter used for query + param string + + // The context that we should restrict to in results from this endpoint + context string +} + +// endpoints is a set of endpoint configs +var endpoints = []endpointConfig{ + {"ct/log", "field=host:", "cert-dns-name"}, + {"scan/protocols", "field=host:", "cert-dns-name"}, + {"scan/domains", "field=host:", "cert-dns-name"}, + {"domain/rdns", "host=", "dns-ptr"}, +} + +// summaryResponse is an API response +type summaryResponse struct { + Summary struct { + Other int `json:"other"` + Values map[string]int `json:"values"` + } `json:"summary"` +} + +// Run function returns all subdomains found with the service +func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { + // Final results channel + results := make(chan subscraping.Result) + s.errors.Store(0) + s.results.Store(0) + + // Waitgroup for subsources + var wg sync.WaitGroup + wg.Add(len(endpoints)) + + // Map for dedupe between subsources + dedupe := sync.Map{} + + // Close down results when all subsources finished + go func(startTime time.Time) { + wg.Wait() + s.timeTaken = time.Since(startTime) + close(results) + }(time.Now()) + + // Start up requests for all subsources + for i := range endpoints { + go s.runSubsource(ctx, domain, session, results, &wg, &dedupe, endpoints[i]) + } + + // Return the results channel + return results +} + +// Name returns the name of the source +func (s *Source) Name() string { + return "driftnet" +} + +// IsDefault indicates that this source should used as part of the default execution. +func (s *Source) IsDefault() bool { + return true +} + +// HasRecursiveSupport indicates that we accept subdomains in addition to apex domains +func (s *Source) HasRecursiveSupport() bool { + return true +} + +// NeedsKey indicates that we need an API key +func (s *Source) NeedsKey() bool { + return true +} + +// AddApiKeys provides us with the API key(s) +func (s *Source) AddApiKeys(keys []string) { + s.apiKeys = keys +} + +// Statistics returns statistics about the scraping process +func (s *Source) Statistics() subscraping.Statistics { + return subscraping.Statistics{ + Errors: int(s.errors.Load()), + Results: int(s.results.Load()), + TimeTaken: s.timeTaken, + Skipped: s.skipped, + } +} + +// runSubsource queries a specific driftnet endpoint for subdomains and sends results to the channel +func (s *Source) runSubsource(ctx context.Context, domain string, session *subscraping.Session, results chan subscraping.Result, wg *sync.WaitGroup, dedupe *sync.Map, epConfig endpointConfig) { + // Default headers + headers := map[string]string{ + "accept": "application/json", + } + + // Pick an API key + randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) + if randomApiKey != "" { + headers["authorization"] = "Bearer " + randomApiKey + } + + // Request + requestURL := fmt.Sprintf("%s%s?%s%s&summarize=host&summary_context=%s&summary_limit=%d", baseURL, epConfig.endpoint, epConfig.param, url.QueryEscape(domain), epConfig.context, summaryLimit) + resp, err := session.Get(ctx, requestURL, "", headers) + if err != nil { + // HTTP 204 is not an error from the Driftnet API + if resp == nil || resp.StatusCode != http.StatusNoContent { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors.Add(1) + } + + wg.Done() + return + } + + defer session.DiscardHTTPResponse(resp) + + // 204 means no results, any other response code is an error + if resp.StatusCode != 200 { + if resp.StatusCode != 204 { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("request failed with status %d", resp.StatusCode)} + s.errors.Add(1) + } + + wg.Done() + return + } + + // Parse and return results + var summary summaryResponse + decoder := json.NewDecoder(resp.Body) + err = decoder.Decode(&summary) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors.Add(1) + wg.Done() + return + } + + for subdomain := range summary.Summary.Values { + // We can get certificate results which aren't actually subdomains of the target domain. Skip them. + if !strings.HasSuffix(subdomain, "."+domain) { + continue + } + + // Avoid returning the same result more than once from the same source (can happen as we are using multiple endpoints) + if _, present := dedupe.LoadOrStore(strings.ToLower(subdomain), true); !present { + results <- subscraping.Result{ + Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain, + } + s.results.Add(1) + } + } + + // Complete! + wg.Done() +} diff --git a/v2/pkg/subscraping/sources/facebook/ctlogs.go b/pkg/subscraping/sources/facebook/ctlogs.go similarity index 100% rename from v2/pkg/subscraping/sources/facebook/ctlogs.go rename to pkg/subscraping/sources/facebook/ctlogs.go diff --git a/v2/pkg/subscraping/sources/facebook/ctlogs_test.go b/pkg/subscraping/sources/facebook/ctlogs_test.go similarity index 100% rename from v2/pkg/subscraping/sources/facebook/ctlogs_test.go rename to pkg/subscraping/sources/facebook/ctlogs_test.go diff --git a/v2/pkg/subscraping/sources/facebook/types.go b/pkg/subscraping/sources/facebook/types.go similarity index 100% rename from v2/pkg/subscraping/sources/facebook/types.go rename to pkg/subscraping/sources/facebook/types.go diff --git a/v2/pkg/subscraping/sources/fofa/fofa.go b/pkg/subscraping/sources/fofa/fofa.go similarity index 100% rename from v2/pkg/subscraping/sources/fofa/fofa.go rename to pkg/subscraping/sources/fofa/fofa.go diff --git a/v2/pkg/subscraping/sources/fullhunt/fullhunt.go b/pkg/subscraping/sources/fullhunt/fullhunt.go similarity index 100% rename from v2/pkg/subscraping/sources/fullhunt/fullhunt.go rename to pkg/subscraping/sources/fullhunt/fullhunt.go diff --git a/v2/pkg/subscraping/sources/github/github.go b/pkg/subscraping/sources/github/github.go similarity index 100% rename from v2/pkg/subscraping/sources/github/github.go rename to pkg/subscraping/sources/github/github.go diff --git a/v2/pkg/subscraping/sources/github/tokenmanager.go b/pkg/subscraping/sources/github/tokenmanager.go similarity index 100% rename from v2/pkg/subscraping/sources/github/tokenmanager.go rename to pkg/subscraping/sources/github/tokenmanager.go diff --git a/v2/pkg/subscraping/sources/gitlab/gitlab.go b/pkg/subscraping/sources/gitlab/gitlab.go similarity index 100% rename from v2/pkg/subscraping/sources/gitlab/gitlab.go rename to pkg/subscraping/sources/gitlab/gitlab.go diff --git a/v2/pkg/subscraping/sources/hackertarget/hackertarget.go b/pkg/subscraping/sources/hackertarget/hackertarget.go similarity index 100% rename from v2/pkg/subscraping/sources/hackertarget/hackertarget.go rename to pkg/subscraping/sources/hackertarget/hackertarget.go diff --git a/v2/pkg/subscraping/sources/hudsonrock/hudsonrock.go b/pkg/subscraping/sources/hudsonrock/hudsonrock.go similarity index 100% rename from v2/pkg/subscraping/sources/hudsonrock/hudsonrock.go rename to pkg/subscraping/sources/hudsonrock/hudsonrock.go diff --git a/v2/pkg/subscraping/sources/hunter/hunter.go b/pkg/subscraping/sources/hunter/hunter.go similarity index 100% rename from v2/pkg/subscraping/sources/hunter/hunter.go rename to pkg/subscraping/sources/hunter/hunter.go diff --git a/v2/pkg/subscraping/sources/intelx/intelx.go b/pkg/subscraping/sources/intelx/intelx.go similarity index 100% rename from v2/pkg/subscraping/sources/intelx/intelx.go rename to pkg/subscraping/sources/intelx/intelx.go diff --git a/v2/pkg/subscraping/sources/leakix/leakix.go b/pkg/subscraping/sources/leakix/leakix.go similarity index 100% rename from v2/pkg/subscraping/sources/leakix/leakix.go rename to pkg/subscraping/sources/leakix/leakix.go diff --git a/v2/pkg/subscraping/sources/netlas/netlas.go b/pkg/subscraping/sources/netlas/netlas.go similarity index 91% rename from v2/pkg/subscraping/sources/netlas/netlas.go rename to pkg/subscraping/sources/netlas/netlas.go index c29bc2c8b..cd9f612e6 100644 --- a/v2/pkg/subscraping/sources/netlas/netlas.go +++ b/pkg/subscraping/sources/netlas/netlas.go @@ -63,7 +63,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se // Pick an API key randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) - resp, err := session.HTTPRequest(ctx, http.MethodGet, countUrl, "", map[string]string{ + resp1, err := session.HTTPRequest(ctx, http.MethodGet, countUrl, "", map[string]string{ "accept": "application/json", "X-API-Key": randomApiKey, }, nil, subscraping.BasicAuth{}) @@ -72,19 +72,19 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ return - } else if resp.StatusCode != 200 { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("request rate limited with status code %d", resp.StatusCode)} + } else if resp1.StatusCode != 200 { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("request rate limited with status code %d", resp1.StatusCode)} s.errors++ return } defer func() { - if err := resp.Body.Close(); err != nil { + if err := resp1.Body.Close(); err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ } }() - body, err := io.ReadAll(resp.Body) + body, err := io.ReadAll(resp1.Body) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("error reading ressponse body")} s.errors++ @@ -120,7 +120,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se // Pick an API key randomApiKey = subscraping.PickRandom(s.apiKeys, s.Name()) - resp, err = session.HTTPRequest(ctx, http.MethodPost, apiUrl, "", map[string]string{ + resp2, err := session.HTTPRequest(ctx, http.MethodPost, apiUrl, "", map[string]string{ "accept": "application/json", "X-API-Key": randomApiKey, "Content-Type": "application/json"}, strings.NewReader(string(jsonRequestBody)), subscraping.BasicAuth{}) @@ -130,20 +130,20 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se return } defer func() { - if err := resp.Body.Close(); err != nil { + if err := resp2.Body.Close(); err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ } }() - body, err = io.ReadAll(resp.Body) + body, err = io.ReadAll(resp2.Body) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("error reading ressponse body")} s.errors++ return } - if resp.StatusCode == 429 { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("request rate limited with status code %d", resp.StatusCode)} + if resp2.StatusCode == 429 { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("request rate limited with status code %d", resp2.StatusCode)} s.errors++ return } diff --git a/v2/pkg/subscraping/sources/pugrecon/pugrecon.go b/pkg/subscraping/sources/pugrecon/pugrecon.go similarity index 100% rename from v2/pkg/subscraping/sources/pugrecon/pugrecon.go rename to pkg/subscraping/sources/pugrecon/pugrecon.go diff --git a/v2/pkg/subscraping/sources/quake/quake.go b/pkg/subscraping/sources/quake/quake.go similarity index 100% rename from v2/pkg/subscraping/sources/quake/quake.go rename to pkg/subscraping/sources/quake/quake.go diff --git a/v2/pkg/subscraping/sources/rapiddns/rapiddns.go b/pkg/subscraping/sources/rapiddns/rapiddns.go similarity index 100% rename from v2/pkg/subscraping/sources/rapiddns/rapiddns.go rename to pkg/subscraping/sources/rapiddns/rapiddns.go diff --git a/v2/pkg/subscraping/sources/reconcloud/reconcloud.go b/pkg/subscraping/sources/reconcloud/reconcloud.go similarity index 100% rename from v2/pkg/subscraping/sources/reconcloud/reconcloud.go rename to pkg/subscraping/sources/reconcloud/reconcloud.go diff --git a/v2/pkg/subscraping/sources/redhuntlabs/redhuntlabs.go b/pkg/subscraping/sources/redhuntlabs/redhuntlabs.go similarity index 100% rename from v2/pkg/subscraping/sources/redhuntlabs/redhuntlabs.go rename to pkg/subscraping/sources/redhuntlabs/redhuntlabs.go diff --git a/v2/pkg/subscraping/sources/riddler/riddler.go b/pkg/subscraping/sources/riddler/riddler.go similarity index 100% rename from v2/pkg/subscraping/sources/riddler/riddler.go rename to pkg/subscraping/sources/riddler/riddler.go diff --git a/v2/pkg/subscraping/sources/robtex/robtext.go b/pkg/subscraping/sources/robtex/robtext.go similarity index 100% rename from v2/pkg/subscraping/sources/robtex/robtext.go rename to pkg/subscraping/sources/robtex/robtext.go diff --git a/v2/pkg/subscraping/sources/rsecloud/rsecloud.go b/pkg/subscraping/sources/rsecloud/rsecloud.go similarity index 100% rename from v2/pkg/subscraping/sources/rsecloud/rsecloud.go rename to pkg/subscraping/sources/rsecloud/rsecloud.go diff --git a/v2/pkg/subscraping/sources/securitytrails/securitytrails.go b/pkg/subscraping/sources/securitytrails/securitytrails.go similarity index 100% rename from v2/pkg/subscraping/sources/securitytrails/securitytrails.go rename to pkg/subscraping/sources/securitytrails/securitytrails.go diff --git a/v2/pkg/subscraping/sources/shodan/shodan.go b/pkg/subscraping/sources/shodan/shodan.go similarity index 100% rename from v2/pkg/subscraping/sources/shodan/shodan.go rename to pkg/subscraping/sources/shodan/shodan.go diff --git a/v2/pkg/subscraping/sources/sitedossier/sitedossier.go b/pkg/subscraping/sources/sitedossier/sitedossier.go similarity index 100% rename from v2/pkg/subscraping/sources/sitedossier/sitedossier.go rename to pkg/subscraping/sources/sitedossier/sitedossier.go diff --git a/v2/pkg/subscraping/sources/threatbook/threatbook.go b/pkg/subscraping/sources/threatbook/threatbook.go similarity index 100% rename from v2/pkg/subscraping/sources/threatbook/threatbook.go rename to pkg/subscraping/sources/threatbook/threatbook.go diff --git a/v2/pkg/subscraping/sources/threatcrowd/threatcrowd.go b/pkg/subscraping/sources/threatcrowd/threatcrowd.go similarity index 100% rename from v2/pkg/subscraping/sources/threatcrowd/threatcrowd.go rename to pkg/subscraping/sources/threatcrowd/threatcrowd.go diff --git a/v2/pkg/subscraping/sources/threatminer/threatminer.go b/pkg/subscraping/sources/threatminer/threatminer.go similarity index 100% rename from v2/pkg/subscraping/sources/threatminer/threatminer.go rename to pkg/subscraping/sources/threatminer/threatminer.go diff --git a/v2/pkg/subscraping/sources/virustotal/virustotal.go b/pkg/subscraping/sources/virustotal/virustotal.go similarity index 100% rename from v2/pkg/subscraping/sources/virustotal/virustotal.go rename to pkg/subscraping/sources/virustotal/virustotal.go diff --git a/v2/pkg/subscraping/sources/waybackarchive/waybackarchive.go b/pkg/subscraping/sources/waybackarchive/waybackarchive.go similarity index 100% rename from v2/pkg/subscraping/sources/waybackarchive/waybackarchive.go rename to pkg/subscraping/sources/waybackarchive/waybackarchive.go diff --git a/v2/pkg/subscraping/sources/whoisxmlapi/whoisxmlapi.go b/pkg/subscraping/sources/whoisxmlapi/whoisxmlapi.go similarity index 100% rename from v2/pkg/subscraping/sources/whoisxmlapi/whoisxmlapi.go rename to pkg/subscraping/sources/whoisxmlapi/whoisxmlapi.go diff --git a/v2/pkg/subscraping/sources/zoomeyeapi/zoomeyeapi.go b/pkg/subscraping/sources/zoomeyeapi/zoomeyeapi.go similarity index 100% rename from v2/pkg/subscraping/sources/zoomeyeapi/zoomeyeapi.go rename to pkg/subscraping/sources/zoomeyeapi/zoomeyeapi.go diff --git a/v2/pkg/subscraping/types.go b/pkg/subscraping/types.go similarity index 100% rename from v2/pkg/subscraping/types.go rename to pkg/subscraping/types.go diff --git a/v2/pkg/subscraping/utils.go b/pkg/subscraping/utils.go similarity index 100% rename from v2/pkg/subscraping/utils.go rename to pkg/subscraping/utils.go diff --git a/v2/pkg/testutils/integration.go b/pkg/testutils/integration.go similarity index 100% rename from v2/pkg/testutils/integration.go rename to pkg/testutils/integration.go