From 24c39d0778079dc7aa103b1fabcc50a31164c793 Mon Sep 17 00:00:00 2001 From: mzack Date: Thu, 7 Jun 2018 00:16:49 +0200 Subject: [PATCH 1/2] shodan implementation POC code #61 Todo: Test --- libsubfinder/engines/passive/passive.go | 11 +++++ libsubfinder/helper/state.go | 8 ++- libsubfinder/sources/shodan/shodan.go | 66 +++++++++++++++++++++++++ 3 files changed, 83 insertions(+), 2 deletions(-) create mode 100644 libsubfinder/sources/shodan/shodan.go diff --git a/libsubfinder/engines/passive/passive.go b/libsubfinder/engines/passive/passive.go index 153e1cbcc..9beca9dec 100644 --- a/libsubfinder/engines/passive/passive.go +++ b/libsubfinder/engines/passive/passive.go @@ -42,6 +42,7 @@ import ( "github.com/Ice3man543/subfinder/libsubfinder/sources/ptrarchive" "github.com/Ice3man543/subfinder/libsubfinder/sources/riddler" "github.com/Ice3man543/subfinder/libsubfinder/sources/securitytrails" + "github.com/Ice3man543/subfinder/libsubfinder/sources/shodan" "github.com/Ice3man543/subfinder/libsubfinder/sources/threatcrowd" "github.com/Ice3man543/subfinder/libsubfinder/sources/threatminer" "github.com/Ice3man543/subfinder/libsubfinder/sources/virustotal" @@ -75,6 +76,7 @@ type Source struct { Virustotal bool Waybackarchive bool CertificateTransparency bool + Shodan bool } func (s *Source) enableAll() { @@ -99,6 +101,7 @@ func (s *Source) enableAll() { s.Virustotal = true s.Waybackarchive = true s.CertificateTransparency = true + s.Shodan = true } func (s *Source) enable(dataSources []string) { @@ -146,6 +149,8 @@ func (s *Source) enable(dataSources []string) { s.Waybackarchive = true case "certificatetransparency": s.CertificateTransparency = true + case "shodan": + s.Shodan = true } } } @@ -214,6 +219,9 @@ func (s *Source) printSummary() { if s.CertificateTransparency { fmt.Printf("\nRunning Source: %sCertificateTransparency%s\n", helper.Info, helper.Reset) } + if s.Shodan { + fmt.Printf("\nRunning Source: %sShodan%s\n", helper.Info, helper.Reset) + } } //nbrActive ses reflection to get automatic active amount of searches @@ -324,6 +332,9 @@ func discover(state *helper.State, domain string, sourceConfig *Source) (subdoma if sourceConfig.CertificateTransparency == true { go certificatetransparency.Query(domain, state, ch) } + if sourceConfig.Shodan == true { + go shodan.Query(domain, state, ch) + } // Recieve data from all goroutines running for i := 0; i < sourceConfig.nbrActive(); i++ { diff --git a/libsubfinder/helper/state.go b/libsubfinder/helper/state.go index c324256e9..8c9e53ffa 100644 --- a/libsubfinder/helper/state.go +++ b/libsubfinder/helper/state.go @@ -58,13 +58,16 @@ type Config struct { CensysUsername string `json:"censysUsername"` // Censys Username CensysSecret string `json:"censysSecret"` // Censys API Key + + ShodanAPIKey string `json:"shodanApiKey"` // Shodan API Key } type Setting struct { CensysPages string // Censys pages to check. For All, use "all" AskPages string // Ask search pages to check - BaiduPages string // Ask search pages to check - BingPages string // Ask search pages to check + BaiduPages string // Baidu search pages to check + BingPages string // Bing search pages to check + ShodanPages string // Shodan search pages to check } func InitializeSettings() (setting *Setting) { @@ -75,6 +78,7 @@ func InitializeSettings() (setting *Setting) { settings.AskPages = "15" settings.BaiduPages = "5" settings.BingPages = "50" + settings.ShodanPages = "10" return &settings } diff --git a/libsubfinder/sources/shodan/shodan.go b/libsubfinder/sources/shodan/shodan.go new file mode 100644 index 000000000..75bea1f06 --- /dev/null +++ b/libsubfinder/sources/shodan/shodan.go @@ -0,0 +1,66 @@ +// +// Written By : @Mzack9999 (Marco Rivoli) +// +// Distributed Under MIT License +// Copyrights (C) 2018 Ice3man +// + +// A golang client for Shodan.io +package shodan + +import ( + "fmt" + "io/ioutil" + "regexp" + "strconv" + + "github.com/Ice3man543/subfinder/libsubfinder/helper" +) + +// all subdomains found +var subdomains []string + +// Query function returns all subdomains found using the service. +func Query(domain string, state *helper.State, ch chan helper.Result) { + + var result helper.Result + result.Subdomains = subdomains + shodanAPIKey := state.ConfigState.ShodanAPIKey + maxPages, _ := strconv.Atoi(state.CurrentSettings.ShodanPages) + for currentPage := 0; currentPage <= maxPages; currentPage++ { + resp, err := helper.GetHTTPResponse("https://api.shodan.io/shodan/host/search?query=hostname:"+domain+"&page="+strconv.Itoa(currentPage)+"&key="+shodanAPIKey, state.Timeout) + if err != nil { + result.Error = err + ch <- result + return + } + + // Get the response body + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + result.Error = err + ch <- result + return + } + src := string(body) + + re := regexp.MustCompile(`([a-z0-9]+\.)+` + domain) + match := re.FindAllString(src, -1) + + for _, subdomain := range match { + if state.Verbose == true { + if state.Color == true { + fmt.Printf("\n[%sShodan%s] %s", helper.Red, helper.Reset, subdomain) + } else { + fmt.Printf("\n[Shodan] %s", subdomain) + } + } + + subdomains = append(subdomains, subdomain) + } + } + + result.Subdomains = subdomains + result.Error = nil + ch <- result +} From 70214ff2111b2a481bc66ce73beb007de9b3fee9 Mon Sep 17 00:00:00 2001 From: mzack Date: Wed, 20 Jun 2018 21:30:54 +0200 Subject: [PATCH 2/2] implements #61 Freely inspired by Aquatone --- libsubfinder/engines/passive/passive.go | 10 ++++++---- libsubfinder/sources/shodan/shodan.go | 7 ++++--- main.go | 4 ++-- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/libsubfinder/engines/passive/passive.go b/libsubfinder/engines/passive/passive.go index 747996f67..6c600a84d 100644 --- a/libsubfinder/engines/passive/passive.go +++ b/libsubfinder/engines/passive/passive.go @@ -84,11 +84,11 @@ type Source struct { Virustotal bool Waybackarchive bool CertificateTransparency bool - Shodan bool Ipv4Info bool Yahoo bool Dogpile bool Exalead bool + Shodan bool } func (s *Source) enableAll() { @@ -115,11 +115,11 @@ func (s *Source) enableAll() { s.Virustotal = true s.Waybackarchive = true s.CertificateTransparency = true - s.Shodan = true s.Ipv4Info = true s.Yahoo = true s.Dogpile = true s.Exalead = true + s.Shodan = true } func (s *Source) enable(dataSources []string) { @@ -171,8 +171,6 @@ func (s *Source) enable(dataSources []string) { s.Waybackarchive = true case "certificatetransparency": s.CertificateTransparency = true - case "shodan": - s.Shodan = true case "ipv4info": s.Ipv4Info = true case "yahoo": @@ -181,6 +179,8 @@ func (s *Source) enable(dataSources []string) { s.Dogpile = true case "exalead": s.Exalead = true + case "shodan": + s.Shodan = true } } } @@ -242,6 +242,8 @@ func (s *Source) disable(dataSources []string) { s.Dogpile = false case "exalead": s.Dogpile = false + case "shodan": + s.Shodan = false } } } diff --git a/libsubfinder/sources/shodan/shodan.go b/libsubfinder/sources/shodan/shodan.go index e82469c10..6e44b020e 100644 --- a/libsubfinder/sources/shodan/shodan.go +++ b/libsubfinder/sources/shodan/shodan.go @@ -41,10 +41,11 @@ func Query(args ...interface{}) interface{} { fmt.Printf("\nerror: %v\n", err) return subdomains } - src := string(body) - re := regexp.MustCompile(`([a-z0-9]+\.)+` + domain) - match := re.FindAllString(src, -1) + reSub := regexp.MustCompile(`"`) + src := reSub.ReplaceAllLiteralString(string(body), " ") + + match := helper.ExtractSubdomains(src, domain) for _, subdomain := range match { if state.Verbose == true { diff --git a/main.go b/main.go index c700046a1..423aa6469 100644 --- a/main.go +++ b/main.go @@ -106,6 +106,8 @@ func main() { reflect.ValueOf(&state.ConfigState).Elem().FieldByName("CensysUsername").SetString(object[1]) } else if strings.EqualFold(object[0], "censyssecret") == true { reflect.ValueOf(&state.ConfigState).Elem().FieldByName("CensysSecret").SetString(object[1]) + } else if strings.EqualFold(object[0], "shodankey") == true { + reflect.ValueOf(&state.ConfigState).Elem().FieldByName("ShodanAPIKey").SetString(object[1]) } configJson, _ := json.MarshalIndent(state.ConfigState, "", " ") @@ -117,8 +119,6 @@ func main() { fmt.Printf("Successfully configured %s%s%s=>%s\n", helper.Info, object[0], helper.Reset, object[1]) } - - os.Exit(0) } if state.SetSetting != "none" {