From 5cf75b79fb3a7ee95c3ee0356800cad8593f8381 Mon Sep 17 00:00:00 2001 From: Tay Date: Mon, 1 Nov 2021 16:30:13 +0900 Subject: [PATCH 1/2] Add C99 source --- v2/pkg/passive/sources.go | 5 ++ v2/pkg/subscraping/sources/c99/c99.go | 71 +++++++++++++++++++++++++++ v2/pkg/subscraping/types.go | 1 + 3 files changed, 77 insertions(+) create mode 100644 v2/pkg/subscraping/sources/c99/c99.go diff --git a/v2/pkg/passive/sources.go b/v2/pkg/passive/sources.go index 72b9a3e78..75f6a3838 100644 --- a/v2/pkg/passive/sources.go +++ b/v2/pkg/passive/sources.go @@ -7,6 +7,7 @@ import ( "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/archiveis" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/binaryedge" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/bufferover" + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/c99" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/censys" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/certspotter" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/chaos" @@ -43,6 +44,7 @@ var DefaultSources = []string{ "alienvault", "anubis", "bufferover", + "c99", "certspotter", "censys", "chaos", @@ -87,6 +89,7 @@ var DefaultAllSources = []string{ "archiveis", "binaryedge", "bufferover", + "c99", "censys", "certspotter", "chaos", @@ -149,6 +152,8 @@ func (a *Agent) addSources(sources []string) { a.sources[source] = &binaryedge.Source{} case "bufferover": a.sources[source] = &bufferover.Source{} + case "c99": + a.sources[source] = &c99.Source{} case "censys": a.sources[source] = &censys.Source{} case "certspotter": diff --git a/v2/pkg/subscraping/sources/c99/c99.go b/v2/pkg/subscraping/sources/c99/c99.go new file mode 100644 index 000000000..211669f5a --- /dev/null +++ b/v2/pkg/subscraping/sources/c99/c99.go @@ -0,0 +1,71 @@ +// Package c99 logic +package c99 + +import ( + "context" + "fmt" + "strings" + + jsoniter "github.com/json-iterator/go" + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" +) + +// Source is the passive scraping agent +type Source struct{} + +type dnsdbLookupResponse struct { + Success bool `json:"success"` + Subdomains []struct { + Subdomain string `json:"subdomain"` + IP string `json:"ip"` + Cloudflare bool `json:"cloudflare"` + } `json:"subdomains"` + Error string `json:"error"` +} + +// Run function returns all subdomains found with the service +func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { + results := make(chan subscraping.Result) + + go func() { + defer close(results) + + if session.Keys.C99 == "" { + return + } + + searchURL := fmt.Sprintf("https://api.c99.nl/subdomainfinder?key=%s&domain=%s&json", session.Keys.C99, domain) + resp, err := session.SimpleGet(ctx, searchURL) + if err != nil { + session.DiscardHTTPResponse(resp) + return + } + + defer resp.Body.Close() + + var response dnsdbLookupResponse + err = jsoniter.NewDecoder(resp.Body).Decode(&response) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + return + } + + if response.Error != "" { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("%v", response.Error)} + return + } + + for _, data := range response.Subdomains { + if !strings.HasPrefix(data.Subdomain, ".") { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: data.Subdomain} + } + } + }() + + return results +} + +// Name returns the name of the source +func (s *Source) Name() string { + return "c99" +} diff --git a/v2/pkg/subscraping/types.go b/v2/pkg/subscraping/types.go index 5d9525214..7e835c5e3 100644 --- a/v2/pkg/subscraping/types.go +++ b/v2/pkg/subscraping/types.go @@ -40,6 +40,7 @@ type Session struct { // Keys contains the current API Keys we have in store type Keys struct { Binaryedge string `json:"binaryedge"` + C99 string `json:"c99"` CensysToken string `json:"censysUsername"` CensysSecret string `json:"censysPassword"` Certspotter string `json:"certspotter"` From 4cd60ec09a141b1131741f7ad50ee593c2b3feb5 Mon Sep 17 00:00:00 2001 From: Tay Date: Tue, 21 Dec 2021 19:36:45 +0900 Subject: [PATCH 2/2] Add c99 to config --- v2/pkg/runner/config.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/v2/pkg/runner/config.go b/v2/pkg/runner/config.go index 0d6d2f9a2..36e2743ba 100644 --- a/v2/pkg/runner/config.go +++ b/v2/pkg/runner/config.go @@ -30,6 +30,7 @@ type ConfigFile struct { ExcludeSources []string `yaml:"exclude-sources,omitempty"` // API keys for different sources Binaryedge []string `yaml:"binaryedge"` + C99 []string `yaml:"c99"` Censys []string `yaml:"censys"` Certspotter []string `yaml:"certspotter"` Chaos []string `yaml:"chaos"` @@ -121,6 +122,9 @@ func (c *ConfigFile) GetKeys() subscraping.Keys { if len(c.Binaryedge) > 0 { keys.Binaryedge = c.Binaryedge[rand.Intn(len(c.Binaryedge))] } + if len(c.C99) > 0 { + keys.C99 = c.C99[rand.Intn(len(c.C99))] + } if len(c.Censys) > 0 { censysKeys := c.Censys[rand.Intn(len(c.Censys))]