From 032c8ec9205e9a782d4de20a491129fd28dd0201 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 19 Jul 2021 14:03:50 +0000 Subject: [PATCH 1/3] chore(deps): bump golang from 1.16-alpine to 1.16.6-alpine Bumps golang from 1.16-alpine to 1.16.6-alpine. --- updated-dependencies: - dependency-name: golang dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index f02f1044e..ac84a2fb9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # Build -FROM golang:1.16-alpine AS build-env +FROM golang:1.16.6-alpine AS build-env RUN GO111MODULE=on go get -v github.com/projectdiscovery/subfinder/v2/cmd/subfinder # Release From 3533d5ba3fc8ebba37fc0f14c54a47f540b80802 Mon Sep 17 00:00:00 2001 From: yhy Date: Fri, 30 Jul 2021 20:38:25 +0800 Subject: [PATCH 2/3] Add Chinaz for subfinder --- v2/pkg/passive/sources.go | 4 ++ v2/pkg/runner/config.go | 4 ++ v2/pkg/subscraping/sources/chinaz/chinaz.go | 56 +++++++++++++++++++++ v2/pkg/subscraping/types.go | 1 + 4 files changed, 65 insertions(+) create mode 100644 v2/pkg/subscraping/sources/chinaz/chinaz.go diff --git a/v2/pkg/passive/sources.go b/v2/pkg/passive/sources.go index 3ac0bec1b..5f5c3413f 100644 --- a/v2/pkg/passive/sources.go +++ b/v2/pkg/passive/sources.go @@ -10,6 +10,7 @@ import ( "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/censys" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/certspotter" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/chaos" + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/chinaz" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/commoncrawl" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/crtsh" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/dnsdb" @@ -44,6 +45,7 @@ var DefaultSources = []string{ "certspotter", "censys", "chaos", + "chinaz", "crtsh", "dnsdumpster", "hackertarget", @@ -150,6 +152,8 @@ func (a *Agent) addSources(sources []string) { a.sources[source] = &certspotter.Source{} case "chaos": a.sources[source] = &chaos.Source{} + case "chinaz": + a.sources[source] = &chinaz.Source{} case "commoncrawl": a.sources[source] = &commoncrawl.Source{} case "crtsh": diff --git a/v2/pkg/runner/config.go b/v2/pkg/runner/config.go index 81ebe40e7..dbd0cb4c1 100644 --- a/v2/pkg/runner/config.go +++ b/v2/pkg/runner/config.go @@ -33,6 +33,7 @@ type ConfigFile struct { Censys []string `yaml:"censys"` Certspotter []string `yaml:"certspotter"` Chaos []string `yaml:"chaos"` + Chinaz []string `yaml:"chinaz"` DNSDB []string `yaml:"dnsdb"` GitHub []string `yaml:"github"` IntelX []string `yaml:"intelx"` @@ -135,6 +136,9 @@ func (c *ConfigFile) GetKeys() subscraping.Keys { if len(c.Chaos) > 0 { keys.Chaos = c.Chaos[rand.Intn(len(c.Chaos))] } + if len(c.Chinaz) > 0 { + keys.Chinaz = c.Chinaz[rand.Intn(len(c.Chinaz))] + } if (len(c.DNSDB)) > 0 { keys.DNSDB = c.DNSDB[rand.Intn(len(c.DNSDB))] } diff --git a/v2/pkg/subscraping/sources/chinaz/chinaz.go b/v2/pkg/subscraping/sources/chinaz/chinaz.go new file mode 100644 index 000000000..78e3cc44f --- /dev/null +++ b/v2/pkg/subscraping/sources/chinaz/chinaz.go @@ -0,0 +1,56 @@ +package chinaz +// chinaz http://my.chinaz.com/ChinazAPI/DataCenter/MyDataApi +import ( + "context" + "fmt" + jsoniter "github.com/json-iterator/go" + "github.com/yhy0/subfinder/v2/pkg/subscraping" + "io/ioutil" +) + +// Source is the passive scraping agent +type Source struct{} + +// Run function returns all subdomains found with the service +func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { + results := make(chan subscraping.Result) + + go func() { + defer close(results) + + if session.Keys.Chinaz == "" { + return + } + + resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://apidatav2.chinaz.com/single/alexa?key=%s&domain=%s", session.Keys.Chinaz, domain)) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + session.DiscardHTTPResponse(resp) + return + } + + body, err := ioutil.ReadAll(resp.Body) + + resp.Body.Close() + + SubdomainList :=jsoniter.Get(body, "Result").Get("ContributingSubdomainList") + + if SubdomainList.ToBool() { + _data := []byte(SubdomainList.ToString()) + for i := 0 ; i< SubdomainList.Size() ; i++{ + subdomain := jsoniter.Get(_data,i,"DataUrl").ToString() + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} + } + } else { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + return + } + }() + + return results +} + +// Name returns the name of the source +func (s *Source) Name() string { + return "chinaz" +} diff --git a/v2/pkg/subscraping/types.go b/v2/pkg/subscraping/types.go index 8a0c445f4..ac6aca272 100755 --- a/v2/pkg/subscraping/types.go +++ b/v2/pkg/subscraping/types.go @@ -40,6 +40,7 @@ type Keys struct { CensysSecret string `json:"censysPassword"` Certspotter string `json:"certspotter"` Chaos string `json:"chaos"` + Chinaz string `json:"chinaz"` DNSDB string `json:"dnsdb"` GitHub []string `json:"github"` IntelXHost string `json:"intelXHost"` From c9f0c06aa5f1847301e5b4b68f142162998ea452 Mon Sep 17 00:00:00 2001 From: yhy <31311038+yhy0@users.noreply.github.com> Date: Fri, 30 Jul 2021 21:41:10 +0800 Subject: [PATCH 3/3] Update chinaz.go --- v2/pkg/subscraping/sources/chinaz/chinaz.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/v2/pkg/subscraping/sources/chinaz/chinaz.go b/v2/pkg/subscraping/sources/chinaz/chinaz.go index 78e3cc44f..b30e93e96 100644 --- a/v2/pkg/subscraping/sources/chinaz/chinaz.go +++ b/v2/pkg/subscraping/sources/chinaz/chinaz.go @@ -4,7 +4,7 @@ import ( "context" "fmt" jsoniter "github.com/json-iterator/go" - "github.com/yhy0/subfinder/v2/pkg/subscraping" + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" "io/ioutil" )