From ab44b6f87f6dde017c529e71fe246cc647204ded Mon Sep 17 00:00:00 2001 From: zeroyu Date: Thu, 27 May 2021 16:44:25 +0800 Subject: [PATCH 01/11] [+] add quake api for subfinder --- v2/pkg/passive/sources.go | 4 ++ v2/pkg/runner/config.go | 4 ++ v2/pkg/subscraping/sources/quake/quake.go | 87 +++++++++++++++++++++++ v2/pkg/subscraping/types.go | 1 + 4 files changed, 96 insertions(+) create mode 100644 v2/pkg/subscraping/sources/quake/quake.go diff --git a/v2/pkg/passive/sources.go b/v2/pkg/passive/sources.go index 3ac0bec1b..671a109fb 100644 --- a/v2/pkg/passive/sources.go +++ b/v2/pkg/passive/sources.go @@ -18,6 +18,7 @@ import ( "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/hackertarget" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/intelx" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/passivetotal" + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/quake" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/rapiddns" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/recon" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/riddler" @@ -58,6 +59,7 @@ var DefaultSources = []string{ "threatcrowd", "threatminer", "virustotal", + "quake", } // DefaultRecursiveSources contains list of default recursive sources @@ -198,6 +200,8 @@ func (a *Agent) addSources(sources []string) { a.sources[source] = &waybackarchive.Source{} case "zoomeye": a.sources[source] = &zoomeye.Source{} + case "quake": + a.sources[source] = &quake.Source{} } } } diff --git a/v2/pkg/runner/config.go b/v2/pkg/runner/config.go index 81ebe40e7..879d35ae8 100644 --- a/v2/pkg/runner/config.go +++ b/v2/pkg/runner/config.go @@ -46,6 +46,7 @@ type ConfigFile struct { URLScan []string `yaml:"urlscan"` Virustotal []string `yaml:"virustotal"` ZoomEye []string `yaml:"zoomeye"` + Quake []string `yaml:"quake"` // Version indicates the version of subfinder installed. Version string `yaml:"subfinder-version"` } @@ -194,6 +195,9 @@ func (c *ConfigFile) GetKeys() subscraping.Keys { keys.ZoomEyePassword = parts[1] } } + if len(c.Quake) > 0 { + keys.Quake = c.Quake[rand.Intn(len(c.Quake))] + } return keys } diff --git a/v2/pkg/subscraping/sources/quake/quake.go b/v2/pkg/subscraping/sources/quake/quake.go new file mode 100644 index 000000000..d6b172e88 --- /dev/null +++ b/v2/pkg/subscraping/sources/quake/quake.go @@ -0,0 +1,87 @@ +// Package quake logic +package quake + +import ( + "bytes" + "context" + "fmt" + jsoniter "github.com/json-iterator/go" + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" + "strings" +) + +type quakeResults struct { + Code int `json:"code"` + Message string `json:"message"` + Data []struct { + Service struct { + HTTP struct { + //Title string `json:"title"` + Host string `json:"host"` + StatusCode int `json:"status_code"` + //ResponseHeaders string `json:"response_headers"` + } `json:"http"` + } + } + Meta struct { + Pagination struct{ + Total int `json:"total"` + } `json:"pagination"` + } `json:"meta"` +} + +// Source is the passive scraping agent +type Source struct{} + +// Run function returns all subdomains found with the service +func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { + results := make(chan subscraping.Result) + go func() { + defer close(results) + + if session.Keys.Quake == "" { + return + } + + // quake api doc https://quake.360.cn/quake/#/help + var requestBody = []byte(`{"query":"domain: *.` + domain + `", "start":0, "size":1000}`) + resp, err := session.Post(ctx, "https://quake.360.cn/api/v3/search/quake_service", "", map[string]string{"Content-Type": "application/json", "X-QuakeToken": session.Keys.Quake},bytes.NewReader(requestBody)) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + session.DiscardHTTPResponse(resp) + return + } + + var response quakeResults + err = jsoniter.NewDecoder(resp.Body).Decode(&response) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + resp.Body.Close() + return + } + resp.Body.Close() + + if response.Code != 0 { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("%s", response.Message)} + return + } + + if response.Meta.Pagination.Total > 0 { + for _, quakeDomain := range response.Data { + subdomain := quakeDomain.Service.HTTP.Host + if strings.ContainsAny(subdomain, "暂无权限") { + subdomain = "" + } + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} + } + } + }() + + return results +} + +// Name returns the name of the source +func (s *Source) Name() string { + return "quake" +} + diff --git a/v2/pkg/subscraping/types.go b/v2/pkg/subscraping/types.go index 8a0c445f4..d5f7431d2 100755 --- a/v2/pkg/subscraping/types.go +++ b/v2/pkg/subscraping/types.go @@ -56,6 +56,7 @@ type Keys struct { Virustotal string `json:"virustotal"` ZoomEyeUsername string `json:"zoomeye_username"` ZoomEyePassword string `json:"zoomeye_password"` + Quake string `json:"quake"` } // Result is a result structure returned by a source From ab5374dac1072704bc52e9bd5829a65ed55a210a Mon Sep 17 00:00:00 2001 From: sandeep <8293321+ehsandeep@users.noreply.github.com> Date: Fri, 28 May 2021 16:58:35 +0530 Subject: [PATCH 02/11] syntax improvements --- v2/pkg/subscraping/sources/quake/quake.go | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/v2/pkg/subscraping/sources/quake/quake.go b/v2/pkg/subscraping/sources/quake/quake.go index d6b172e88..02ff187bc 100644 --- a/v2/pkg/subscraping/sources/quake/quake.go +++ b/v2/pkg/subscraping/sources/quake/quake.go @@ -17,14 +17,14 @@ type quakeResults struct { Service struct { HTTP struct { //Title string `json:"title"` - Host string `json:"host"` - StatusCode int `json:"status_code"` + Host string `json:"host"` + StatusCode int `json:"status_code"` //ResponseHeaders string `json:"response_headers"` } `json:"http"` } } Meta struct { - Pagination struct{ + Pagination struct { Total int `json:"total"` } `json:"pagination"` } `json:"meta"` @@ -39,13 +39,13 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se go func() { defer close(results) - if session.Keys.Quake == "" { + if session.Keys.Quake == "" { return } // quake api doc https://quake.360.cn/quake/#/help var requestBody = []byte(`{"query":"domain: *.` + domain + `", "start":0, "size":1000}`) - resp, err := session.Post(ctx, "https://quake.360.cn/api/v3/search/quake_service", "", map[string]string{"Content-Type": "application/json", "X-QuakeToken": session.Keys.Quake},bytes.NewReader(requestBody)) + resp, err := session.Post(ctx, "https://quake.360.cn/api/v3/search/quake_service", "", map[string]string{"Content-Type": "application/json", "X-QuakeToken": session.Keys.Quake}, bytes.NewReader(requestBody)) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} session.DiscardHTTPResponse(resp) @@ -69,7 +69,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se if response.Meta.Pagination.Total > 0 { for _, quakeDomain := range response.Data { subdomain := quakeDomain.Service.HTTP.Host - if strings.ContainsAny(subdomain, "暂无权限") { + if strings.ContainsAny(subdomain, "暂无权限") { subdomain = "" } results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} @@ -84,4 +84,3 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se func (s *Source) Name() string { return "quake" } - From dd9278a8a7f82eb5dab7dc6aad6c72d6a83607fb Mon Sep 17 00:00:00 2001 From: sandeep <8293321+ehsandeep@users.noreply.github.com> Date: Fri, 28 May 2021 17:05:32 +0530 Subject: [PATCH 03/11] Update quake.go --- v2/pkg/subscraping/sources/quake/quake.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/v2/pkg/subscraping/sources/quake/quake.go b/v2/pkg/subscraping/sources/quake/quake.go index 02ff187bc..0503c3463 100644 --- a/v2/pkg/subscraping/sources/quake/quake.go +++ b/v2/pkg/subscraping/sources/quake/quake.go @@ -5,9 +5,10 @@ import ( "bytes" "context" "fmt" + "strings" + jsoniter "github.com/json-iterator/go" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" - "strings" ) type quakeResults struct { @@ -16,10 +17,10 @@ type quakeResults struct { Data []struct { Service struct { HTTP struct { - //Title string `json:"title"` + // Title string `json:"title"` Host string `json:"host"` StatusCode int `json:"status_code"` - //ResponseHeaders string `json:"response_headers"` + // ResponseHeaders string `json:"response_headers"` } `json:"http"` } } From 67ab6b5122f489874d27feb0722b9f6152f66d2b Mon Sep 17 00:00:00 2001 From: zeroyu Date: Mon, 28 Jun 2021 16:46:31 +0800 Subject: [PATCH 04/11] [+] fix status_code field bugs on different operating systems --- v2/pkg/subscraping/sources/quake/quake.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/v2/pkg/subscraping/sources/quake/quake.go b/v2/pkg/subscraping/sources/quake/quake.go index 0503c3463..a871f44bc 100644 --- a/v2/pkg/subscraping/sources/quake/quake.go +++ b/v2/pkg/subscraping/sources/quake/quake.go @@ -18,8 +18,8 @@ type quakeResults struct { Service struct { HTTP struct { // Title string `json:"title"` - Host string `json:"host"` - StatusCode int `json:"status_code"` + Host string `json:"host"` + //StatusCode int `json:"status_code"` // ResponseHeaders string `json:"response_headers"` } `json:"http"` } From c95de0c630cd7fce162b67f34687fe8958a98a8a Mon Sep 17 00:00:00 2001 From: Ice3man Date: Mon, 28 Jun 2021 15:35:00 +0530 Subject: [PATCH 05/11] Update quake.go --- v2/pkg/subscraping/sources/quake/quake.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/v2/pkg/subscraping/sources/quake/quake.go b/v2/pkg/subscraping/sources/quake/quake.go index a871f44bc..55028a7b7 100644 --- a/v2/pkg/subscraping/sources/quake/quake.go +++ b/v2/pkg/subscraping/sources/quake/quake.go @@ -19,7 +19,7 @@ type quakeResults struct { HTTP struct { // Title string `json:"title"` Host string `json:"host"` - //StatusCode int `json:"status_code"` + // StatusCode int `json:"status_code"` // ResponseHeaders string `json:"response_headers"` } `json:"http"` } From f64e87f3fd183060f5787ca36d5063b9df76a331 Mon Sep 17 00:00:00 2001 From: sandeep Date: Fri, 20 Aug 2021 04:46:12 +0530 Subject: [PATCH 06/11] Update config.go --- v2/pkg/runner/config.go | 1 + 1 file changed, 1 insertion(+) diff --git a/v2/pkg/runner/config.go b/v2/pkg/runner/config.go index d426cc6bd..fb0552c9a 100644 --- a/v2/pkg/runner/config.go +++ b/v2/pkg/runner/config.go @@ -202,6 +202,7 @@ func (c *ConfigFile) GetKeys() subscraping.Keys { } if len(c.Quake) > 0 { keys.Quake = c.Quake[rand.Intn(len(c.Quake))] + } if len(c.Fofa) > 0 { fofaKeys := c.Fofa[rand.Intn(len(c.Fofa))] parts := strings.Split(fofaKeys, ":") From 666cebf17f9bc6798bef72ac1f083fca0510ec34 Mon Sep 17 00:00:00 2001 From: forgedhallpass <13679401+forgedhallpass@users.noreply.github.com> Date: Wed, 31 Aug 2022 18:13:02 +0300 Subject: [PATCH 07/11] quake: reducing the pagination size to the maximum allowed of 500 --- v2/pkg/subscraping/sources/quake/quake.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/v2/pkg/subscraping/sources/quake/quake.go b/v2/pkg/subscraping/sources/quake/quake.go index 766c62dce..805856879 100644 --- a/v2/pkg/subscraping/sources/quake/quake.go +++ b/v2/pkg/subscraping/sources/quake/quake.go @@ -21,7 +21,7 @@ type quakeResults struct { Host string `json:"host"` } `json:"http"` } - } + } `json:"data"` Meta struct { Pagination struct { Total int `json:"total"` @@ -46,7 +46,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se } // quake api doc https://quake.360.cn/quake/#/help - var requestBody = []byte(fmt.Sprintf(`{"query":"domain: *.%s", "start":0, "size":1000}`, domain)) + var requestBody = []byte(fmt.Sprintf(`{"query":"domain: *.%s", "start":0, "size":500}`, domain)) resp, err := session.Post(ctx, "https://quake.360.cn/api/v3/search/quake_service", "", map[string]string{"Content-Type": "application/json", "X-QuakeToken": randomApiKey}, bytes.NewReader(requestBody)) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} From a69762e3891a87d8a53de5d117cadb0edd032368 Mon Sep 17 00:00:00 2001 From: z3r0yu Date: Thu, 20 Oct 2022 18:45:56 +0800 Subject: [PATCH 08/11] add source recon.cloud --- README.md | 2 +- v2/pkg/passive/sources.go | 3 + v2/pkg/passive/sources_test.go | 2 + .../sources/reconcloud/reconcloud.go | 80 +++++++++++++++++++ 4 files changed, 86 insertions(+), 1 deletion(-) create mode 100644 v2/pkg/subscraping/sources/reconcloud/reconcloud.go diff --git a/README.md b/README.md index 04263bda5..224934f82 100644 --- a/README.md +++ b/README.md @@ -114,7 +114,7 @@ go install -v github.com/projectdiscovery/subfinder/v2/cmd/subfinder@latest Subfinder will work after using the installation instructions however to configure Subfinder to work with certain services, you will need to have setup API keys. The following services do not work without an API key: -[BeVigil](https://bevigil.com/osint-api), [Binaryedge](https://binaryedge.io), [C99](https://api.c99.nl/), [Certspotter](https://sslmate.com/certspotter/api/), [Chinaz](http://my.chinaz.com/ChinazAPI/DataCenter/MyDataApi), [Censys](https://censys.io), [Chaos](https://chaos.projectdiscovery.io), [DnsDB](https://api.dnsdb.info), [Fofa](https://fofa.info/static_pages/api_help), [Github](https://github.com), [Intelx](https://intelx.io), [Passivetotal](http://passivetotal.org), [Robtex](https://www.robtex.com/api/), [SecurityTrails](http://securitytrails.com), [Shodan](https://shodan.io), [Threatbook](https://x.threatbook.cn/en), [Virustotal](https://www.virustotal.com), [WhoisXML API](https://whoisxmlapi.com/), [Zoomeye](https://www.zoomeye.org) +[BeVigil](https://bevigil.com/osint-api), [Binaryedge](https://binaryedge.io), [C99](https://api.c99.nl/), [Certspotter](https://sslmate.com/certspotter/api/), [Chinaz](http://my.chinaz.com/ChinazAPI/DataCenter/MyDataApi), [Censys](https://censys.io), [Chaos](https://chaos.projectdiscovery.io), [DnsDB](https://api.dnsdb.info), [Fofa](https://fofa.info/static_pages/api_help), [Github](https://github.com), [Intelx](https://intelx.io), [Passivetotal](http://passivetotal.org), [Robtex](https://www.robtex.com/api/), [SecurityTrails](http://securitytrails.com), [Shodan](https://shodan.io), [Threatbook](https://x.threatbook.cn/en), [Virustotal](https://www.virustotal.com), [WhoisXML API](https://whoisxmlapi.com/), [Zoomeye](https://www.zoomeye.org), [Recon.Cloud](https://recon.cloud/) These values are stored in the `$HOME/.config/subfinder/provider-config.yaml` file which will be created when you run the tool for the first time. The configuration file uses the YAML format. Multiple API keys can be specified for each of these services from which one of them will be used for enumeration. diff --git a/v2/pkg/passive/sources.go b/v2/pkg/passive/sources.go index 2544742f0..7070f3ab2 100644 --- a/v2/pkg/passive/sources.go +++ b/v2/pkg/passive/sources.go @@ -2,6 +2,7 @@ package passive import ( "fmt" + "strings" "golang.org/x/exp/maps" @@ -31,6 +32,7 @@ import ( "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/passivetotal" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/quake" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/rapiddns" + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/reconcloud" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/riddler" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/robtex" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/securitytrails" @@ -71,6 +73,7 @@ var AllSources = [...]subscraping.Source{ &rapiddns.Source{}, &riddler.Source{}, &robtex.Source{}, + &reconcloud.Source{}, &securitytrails.Source{}, &shodan.Source{}, &sitedossier.Source{}, diff --git a/v2/pkg/passive/sources_test.go b/v2/pkg/passive/sources_test.go index 4ff093fc6..2f02c9386 100644 --- a/v2/pkg/passive/sources_test.go +++ b/v2/pkg/passive/sources_test.go @@ -36,6 +36,7 @@ var ( "rapiddns", "riddler", "robtex", + "reconcloud", "securitytrails", "shodan", "sitedossier", @@ -70,6 +71,7 @@ var ( "quake", "robtex", "riddler", + "reconcloud", "securitytrails", "shodan", "threatminer", diff --git a/v2/pkg/subscraping/sources/reconcloud/reconcloud.go b/v2/pkg/subscraping/sources/reconcloud/reconcloud.go new file mode 100644 index 000000000..07f35d8cd --- /dev/null +++ b/v2/pkg/subscraping/sources/reconcloud/reconcloud.go @@ -0,0 +1,80 @@ +// Package reconcloud logic +package reconcloud + +import ( + "context" + "fmt" + jsoniter "github.com/json-iterator/go" + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" +) + +type reconCloudResponse struct { + MsgType string `json:"msg_type"` + RequestID string `json:"request_id"` + OnCache bool `json:"on_cache"` + Step string `json:"step"` + CloudAssetsList []cloudAssetsList `json:"cloud_assets_list"` +} + +type cloudAssetsList struct { + Key string `json:"key"` + Domain string `json:"domain"` + CloudProvider string `json:"cloud_provider"` +} + +// Source is the passive scraping agent +type Source struct{} + +// Run function returns all subdomains found with the service +func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { + results := make(chan subscraping.Result) + + go func() { + defer close(results) + + resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://recon.cloud/api/search?domain=%s", domain)) + if err != nil && resp == nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + session.DiscardHTTPResponse(resp) + return + } + + var response reconCloudResponse + err = jsoniter.NewDecoder(resp.Body).Decode(&response) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + resp.Body.Close() + return + } + resp.Body.Close() + + if len(response.CloudAssetsList) > 0 { + for _, cloudAsset := range response.CloudAssetsList { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: cloudAsset.Domain} + } + } + }() + + return results +} + +// Name returns the name of the source +func (s *Source) Name() string { + return "reconcloud" +} + +func (s *Source) IsDefault() bool { + return true +} + +func (s *Source) HasRecursiveSupport() bool { + return true +} + +func (s *Source) NeedsKey() bool { + return false +} + +func (s *Source) AddApiKeys(_ []string) { + // no key needed +} From 41251b2360a23a6e7cc730a08962cd2ee354dac4 Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Mon, 24 Oct 2022 00:24:02 +0200 Subject: [PATCH 09/11] fixing recursive total --- v2/pkg/passive/sources_test.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/v2/pkg/passive/sources_test.go b/v2/pkg/passive/sources_test.go index 2f02c9386..b2432bf05 100644 --- a/v2/pkg/passive/sources_test.go +++ b/v2/pkg/passive/sources_test.go @@ -88,6 +88,7 @@ var ( "dnsdumpster", "hackertarget", "passivetotal", + "reconcloud", "securitytrails", "sonarsearch", "virustotal", @@ -136,13 +137,13 @@ func TestSourceFiltering(t *testing.T) { {someSources, someExclusions, false, false, len(someSources) - len(someExclusions)}, {someSources, someExclusions, false, true, 1}, {someSources, someExclusions, true, false, len(AllSources) - len(someExclusions)}, - {someSources, someExclusions, true, true, 9}, + {someSources, someExclusions, true, true, 10}, {someSources, []string{}, false, false, len(someSources)}, {someSources, []string{}, true, false, len(AllSources)}, {[]string{}, []string{}, false, false, len(expectedDefaultSources)}, - {[]string{}, []string{}, false, true, 9}, + {[]string{}, []string{}, false, true, 10}, {[]string{}, []string{}, true, false, len(AllSources)}, {[]string{}, []string{}, true, true, len(expectedDefaultRecursiveSources)}, } From bebe76e67dec4f0b29507db0794e690999aefe31 Mon Sep 17 00:00:00 2001 From: z3r0yu Date: Mon, 24 Oct 2022 13:07:31 +0800 Subject: [PATCH 10/11] update a source from hunter --- README.md | 2 +- v2/pkg/passive/sources.go | 2 + v2/pkg/passive/sources_test.go | 6 +- v2/pkg/subscraping/sources/hunter/hunter.go | 104 ++++++++++++++++++++ 4 files changed, 111 insertions(+), 3 deletions(-) create mode 100644 v2/pkg/subscraping/sources/hunter/hunter.go diff --git a/README.md b/README.md index 224934f82..cf81d318d 100644 --- a/README.md +++ b/README.md @@ -114,7 +114,7 @@ go install -v github.com/projectdiscovery/subfinder/v2/cmd/subfinder@latest Subfinder will work after using the installation instructions however to configure Subfinder to work with certain services, you will need to have setup API keys. The following services do not work without an API key: -[BeVigil](https://bevigil.com/osint-api), [Binaryedge](https://binaryedge.io), [C99](https://api.c99.nl/), [Certspotter](https://sslmate.com/certspotter/api/), [Chinaz](http://my.chinaz.com/ChinazAPI/DataCenter/MyDataApi), [Censys](https://censys.io), [Chaos](https://chaos.projectdiscovery.io), [DnsDB](https://api.dnsdb.info), [Fofa](https://fofa.info/static_pages/api_help), [Github](https://github.com), [Intelx](https://intelx.io), [Passivetotal](http://passivetotal.org), [Robtex](https://www.robtex.com/api/), [SecurityTrails](http://securitytrails.com), [Shodan](https://shodan.io), [Threatbook](https://x.threatbook.cn/en), [Virustotal](https://www.virustotal.com), [WhoisXML API](https://whoisxmlapi.com/), [Zoomeye](https://www.zoomeye.org), [Recon.Cloud](https://recon.cloud/) +[BeVigil](https://bevigil.com/osint-api), [Binaryedge](https://binaryedge.io), [C99](https://api.c99.nl/), [Certspotter](https://sslmate.com/certspotter/api/), [Chinaz](http://my.chinaz.com/ChinazAPI/DataCenter/MyDataApi), [Censys](https://censys.io), [Chaos](https://chaos.projectdiscovery.io), [DnsDB](https://api.dnsdb.info), [Fofa](https://fofa.info/static_pages/api_help), [Github](https://github.com), [Intelx](https://intelx.io), [Passivetotal](http://passivetotal.org), [Robtex](https://www.robtex.com/api/), [SecurityTrails](http://securitytrails.com), [Shodan](https://shodan.io), [Threatbook](https://x.threatbook.cn/en), [Virustotal](https://www.virustotal.com), [WhoisXML API](https://whoisxmlapi.com/), [Zoomeye](https://www.zoomeye.org), [Recon.Cloud](https://recon.cloud/), [Hunter](https://hunter.qianxin.com/) These values are stored in the `$HOME/.config/subfinder/provider-config.yaml` file which will be created when you run the tool for the first time. The configuration file uses the YAML format. Multiple API keys can be specified for each of these services from which one of them will be used for enumeration. diff --git a/v2/pkg/passive/sources.go b/v2/pkg/passive/sources.go index 7070f3ab2..3d6cf6090 100644 --- a/v2/pkg/passive/sources.go +++ b/v2/pkg/passive/sources.go @@ -28,6 +28,7 @@ import ( "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/fullhunt" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/github" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/hackertarget" + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/hunter" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/intelx" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/passivetotal" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/quake" @@ -67,6 +68,7 @@ var AllSources = [...]subscraping.Source{ &fullhunt.Source{}, &github.Source{}, &hackertarget.Source{}, + &hunter.Source{}, &intelx.Source{}, &passivetotal.Source{}, &quake.Source{}, diff --git a/v2/pkg/passive/sources_test.go b/v2/pkg/passive/sources_test.go index b2432bf05..4680d8cd2 100644 --- a/v2/pkg/passive/sources_test.go +++ b/v2/pkg/passive/sources_test.go @@ -30,6 +30,7 @@ var ( "fullhunt", "github", "hackertarget", + "hunter", "intelx", "passivetotal", "quake", @@ -66,6 +67,7 @@ var ( "fofa", "fullhunt", "hackertarget", + "hunter", "intelx", "passivetotal", "quake", @@ -137,13 +139,13 @@ func TestSourceFiltering(t *testing.T) { {someSources, someExclusions, false, false, len(someSources) - len(someExclusions)}, {someSources, someExclusions, false, true, 1}, {someSources, someExclusions, true, false, len(AllSources) - len(someExclusions)}, - {someSources, someExclusions, true, true, 10}, + {someSources, someExclusions, true, true, 11}, {someSources, []string{}, false, false, len(someSources)}, {someSources, []string{}, true, false, len(AllSources)}, {[]string{}, []string{}, false, false, len(expectedDefaultSources)}, - {[]string{}, []string{}, false, true, 10}, + {[]string{}, []string{}, false, true, 11}, {[]string{}, []string{}, true, false, len(AllSources)}, {[]string{}, []string{}, true, true, len(expectedDefaultRecursiveSources)}, } diff --git a/v2/pkg/subscraping/sources/hunter/hunter.go b/v2/pkg/subscraping/sources/hunter/hunter.go new file mode 100644 index 000000000..69214c5f1 --- /dev/null +++ b/v2/pkg/subscraping/sources/hunter/hunter.go @@ -0,0 +1,104 @@ +package hunter + +import ( + "context" + "encoding/base64" + "fmt" + jsoniter "github.com/json-iterator/go" + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" +) + +type hunterResp struct { + Code int `json:"code"` + Data hunterData `json:"data"` + Message string `json:"message"` +} + +type infoArr struct { + URL string `json:"url"` + IP string `json:"ip"` + Port int `json:"port"` + Domain string `json:"domain"` + Protocol string `json:"protocol"` +} + +type hunterData struct { + InfoArr []infoArr `json:"arr"` + Total int `json:"total"` +} + +// Source is the passive scraping agent +type Source struct { + apiKeys []string +} + +// Run function returns all subdomains found with the service +func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { + results := make(chan subscraping.Result) + + go func() { + defer close(results) + + randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) + if randomApiKey == "" { + return + } + + var pages = 1 + for currentPage := 1; currentPage <= pages; currentPage++ { + // hunter api doc https://hunter.qianxin.com/home/helpCenter?r=5-1-2 + qbase64 := base64.URLEncoding.EncodeToString([]byte(fmt.Sprintf("domain=\"%s\"", domain))) + resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://hunter.qianxin.com/openApi/search?api-key=%s&search=%s&page=1&page_size=100&is_web=3", randomApiKey, qbase64)) + if err != nil && resp == nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + session.DiscardHTTPResponse(resp) + return + } + + var response hunterResp + err = jsoniter.NewDecoder(resp.Body).Decode(&response) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + resp.Body.Close() + return + } + resp.Body.Close() + + if response.Code == 401 || response.Code == 400 { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("%s", response.Message)} + return + } + + if response.Data.Total > 0 { + for _, hunterInfo := range response.Data.InfoArr { + subdomain := hunterInfo.Domain + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} + } + } + pages = int(response.Data.Total/1000) + 1 + } + }() + + return results +} + +// Name returns the name of the source +func (s *Source) Name() string { + return "hunter" +} + +func (s *Source) IsDefault() bool { + return true +} + +func (s *Source) HasRecursiveSupport() bool { + return false +} + +func (s *Source) NeedsKey() bool { + return true +} + +func (s *Source) AddApiKeys(keys []string) { + s.apiKeys = keys +} From 326414f59aa2ba809d890b51a4690b4bd4cf4a7b Mon Sep 17 00:00:00 2001 From: Mzack9999 Date: Mon, 24 Oct 2022 08:33:25 +0200 Subject: [PATCH 11/11] keeping only hunter + fixing tests --- README.md | 2 +- v2/pkg/passive/sources.go | 2 - v2/pkg/passive/sources_test.go | 7 +- .../sources/reconcloud/reconcloud.go | 80 ------------------- 4 files changed, 3 insertions(+), 88 deletions(-) delete mode 100644 v2/pkg/subscraping/sources/reconcloud/reconcloud.go diff --git a/README.md b/README.md index cf81d318d..bd2122de7 100644 --- a/README.md +++ b/README.md @@ -114,7 +114,7 @@ go install -v github.com/projectdiscovery/subfinder/v2/cmd/subfinder@latest Subfinder will work after using the installation instructions however to configure Subfinder to work with certain services, you will need to have setup API keys. The following services do not work without an API key: -[BeVigil](https://bevigil.com/osint-api), [Binaryedge](https://binaryedge.io), [C99](https://api.c99.nl/), [Certspotter](https://sslmate.com/certspotter/api/), [Chinaz](http://my.chinaz.com/ChinazAPI/DataCenter/MyDataApi), [Censys](https://censys.io), [Chaos](https://chaos.projectdiscovery.io), [DnsDB](https://api.dnsdb.info), [Fofa](https://fofa.info/static_pages/api_help), [Github](https://github.com), [Intelx](https://intelx.io), [Passivetotal](http://passivetotal.org), [Robtex](https://www.robtex.com/api/), [SecurityTrails](http://securitytrails.com), [Shodan](https://shodan.io), [Threatbook](https://x.threatbook.cn/en), [Virustotal](https://www.virustotal.com), [WhoisXML API](https://whoisxmlapi.com/), [Zoomeye](https://www.zoomeye.org), [Recon.Cloud](https://recon.cloud/), [Hunter](https://hunter.qianxin.com/) +[BeVigil](https://bevigil.com/osint-api), [Binaryedge](https://binaryedge.io), [C99](https://api.c99.nl/), [Certspotter](https://sslmate.com/certspotter/api/), [Chinaz](http://my.chinaz.com/ChinazAPI/DataCenter/MyDataApi), [Censys](https://censys.io), [Chaos](https://chaos.projectdiscovery.io), [DnsDB](https://api.dnsdb.info), [Fofa](https://fofa.info/static_pages/api_help), [Github](https://github.com), [Intelx](https://intelx.io), [Passivetotal](http://passivetotal.org), [Robtex](https://www.robtex.com/api/), [SecurityTrails](http://securitytrails.com), [Shodan](https://shodan.io), [Threatbook](https://x.threatbook.cn/en), [Virustotal](https://www.virustotal.com), [WhoisXML API](https://whoisxmlapi.com/), [Zoomeye](https://www.zoomeye.org), [Hunter](https://hunter.qianxin.com/) These values are stored in the `$HOME/.config/subfinder/provider-config.yaml` file which will be created when you run the tool for the first time. The configuration file uses the YAML format. Multiple API keys can be specified for each of these services from which one of them will be used for enumeration. diff --git a/v2/pkg/passive/sources.go b/v2/pkg/passive/sources.go index 3d6cf6090..2820b17e6 100644 --- a/v2/pkg/passive/sources.go +++ b/v2/pkg/passive/sources.go @@ -33,7 +33,6 @@ import ( "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/passivetotal" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/quake" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/rapiddns" - "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/reconcloud" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/riddler" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/robtex" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/securitytrails" @@ -75,7 +74,6 @@ var AllSources = [...]subscraping.Source{ &rapiddns.Source{}, &riddler.Source{}, &robtex.Source{}, - &reconcloud.Source{}, &securitytrails.Source{}, &shodan.Source{}, &sitedossier.Source{}, diff --git a/v2/pkg/passive/sources_test.go b/v2/pkg/passive/sources_test.go index 4680d8cd2..9b7195ec1 100644 --- a/v2/pkg/passive/sources_test.go +++ b/v2/pkg/passive/sources_test.go @@ -37,7 +37,6 @@ var ( "rapiddns", "riddler", "robtex", - "reconcloud", "securitytrails", "shodan", "sitedossier", @@ -73,7 +72,6 @@ var ( "quake", "robtex", "riddler", - "reconcloud", "securitytrails", "shodan", "threatminer", @@ -90,7 +88,6 @@ var ( "dnsdumpster", "hackertarget", "passivetotal", - "reconcloud", "securitytrails", "sonarsearch", "virustotal", @@ -139,13 +136,13 @@ func TestSourceFiltering(t *testing.T) { {someSources, someExclusions, false, false, len(someSources) - len(someExclusions)}, {someSources, someExclusions, false, true, 1}, {someSources, someExclusions, true, false, len(AllSources) - len(someExclusions)}, - {someSources, someExclusions, true, true, 11}, + {someSources, someExclusions, true, true, 9}, {someSources, []string{}, false, false, len(someSources)}, {someSources, []string{}, true, false, len(AllSources)}, {[]string{}, []string{}, false, false, len(expectedDefaultSources)}, - {[]string{}, []string{}, false, true, 11}, + {[]string{}, []string{}, false, true, 9}, {[]string{}, []string{}, true, false, len(AllSources)}, {[]string{}, []string{}, true, true, len(expectedDefaultRecursiveSources)}, } diff --git a/v2/pkg/subscraping/sources/reconcloud/reconcloud.go b/v2/pkg/subscraping/sources/reconcloud/reconcloud.go deleted file mode 100644 index 07f35d8cd..000000000 --- a/v2/pkg/subscraping/sources/reconcloud/reconcloud.go +++ /dev/null @@ -1,80 +0,0 @@ -// Package reconcloud logic -package reconcloud - -import ( - "context" - "fmt" - jsoniter "github.com/json-iterator/go" - "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" -) - -type reconCloudResponse struct { - MsgType string `json:"msg_type"` - RequestID string `json:"request_id"` - OnCache bool `json:"on_cache"` - Step string `json:"step"` - CloudAssetsList []cloudAssetsList `json:"cloud_assets_list"` -} - -type cloudAssetsList struct { - Key string `json:"key"` - Domain string `json:"domain"` - CloudProvider string `json:"cloud_provider"` -} - -// Source is the passive scraping agent -type Source struct{} - -// Run function returns all subdomains found with the service -func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { - results := make(chan subscraping.Result) - - go func() { - defer close(results) - - resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://recon.cloud/api/search?domain=%s", domain)) - if err != nil && resp == nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - session.DiscardHTTPResponse(resp) - return - } - - var response reconCloudResponse - err = jsoniter.NewDecoder(resp.Body).Decode(&response) - if err != nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - resp.Body.Close() - return - } - resp.Body.Close() - - if len(response.CloudAssetsList) > 0 { - for _, cloudAsset := range response.CloudAssetsList { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: cloudAsset.Domain} - } - } - }() - - return results -} - -// Name returns the name of the source -func (s *Source) Name() string { - return "reconcloud" -} - -func (s *Source) IsDefault() bool { - return true -} - -func (s *Source) HasRecursiveSupport() bool { - return true -} - -func (s *Source) NeedsKey() bool { - return false -} - -func (s *Source) AddApiKeys(_ []string) { - // no key needed -}