From e5bcc681d15d5088f7ba0bd59eabff12d05c2f05 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fan=20Can=20Bak=C4=B1r?= Date: Mon, 22 Sep 2025 16:00:01 +0300 Subject: [PATCH 1/4] add onhype --- pkg/passive/sources.go | 2 + pkg/passive/sources_test.go | 2 + pkg/subscraping/sources/onhype/onhype.go | 129 +++++++++++++++++++++++ 3 files changed, 133 insertions(+) create mode 100644 pkg/subscraping/sources/onhype/onhype.go diff --git a/pkg/passive/sources.go b/pkg/passive/sources.go index e74d874be..f91fdf860 100644 --- a/pkg/passive/sources.go +++ b/pkg/passive/sources.go @@ -37,6 +37,7 @@ import ( "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/intelx" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/leakix" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/netlas" + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/onhype" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/pugrecon" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/quake" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/rapiddns" @@ -79,6 +80,7 @@ var AllSources = [...]subscraping.Source{ &hunter.Source{}, &intelx.Source{}, &netlas.Source{}, + &onhype.Source{}, &leakix.Source{}, &quake.Source{}, &pugrecon.Source{}, diff --git a/pkg/passive/sources_test.go b/pkg/passive/sources_test.go index 4febbf9fa..53f8a6b92 100644 --- a/pkg/passive/sources_test.go +++ b/pkg/passive/sources_test.go @@ -33,6 +33,7 @@ var ( "hackertarget", "intelx", "netlas", + "onhype", "quake", "pugrecon", "rapiddns", @@ -78,6 +79,7 @@ var ( "fullhunt", "hackertarget", "intelx", + "onhype", "quake", "redhuntlabs", "robtex", diff --git a/pkg/subscraping/sources/onhype/onhype.go b/pkg/subscraping/sources/onhype/onhype.go new file mode 100644 index 000000000..916ed51b5 --- /dev/null +++ b/pkg/subscraping/sources/onhype/onhype.go @@ -0,0 +1,129 @@ +// Package onhype logic +package onhype + +import ( + "context" + "fmt" + "net/http" + "net/url" + "time" + + jsoniter "github.com/json-iterator/go" + + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" +) + +type OnypheResponse struct { + Error int `json:"error"` + Results []Result `json:"results"` + Page int `json:"page"` + PageSize int `json:"page_size"` + Total int `json:"total"` +} + +type Result struct { + Hostname string `json:"hostname"` +} + +// Source is the passive scraping agent +type Source struct { + apiKeys []string + timeTaken time.Duration + errors int + results int + skipped bool +} + +// Run function returns all subdomains found with the service +func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { + results := make(chan subscraping.Result) + s.errors = 0 + s.results = 0 + + go func() { + defer func(startTime time.Time) { + s.timeTaken = time.Since(startTime) + close(results) + }(time.Now()) + + randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name()) + if randomApiKey == "" { + s.skipped = true + return + } + + headers := map[string]string{"Content-Type": "application/json", "Authorization": "bearer " + randomApiKey} + + page := 1 + for { + var resp *http.Response + var err error + + urlWithQuery := fmt.Sprintf("https://www.onyphe.io/api/v2/search/?q=%s&page=%d&size=10", + url.QueryEscape("category:resolver domain:"+domain), page) + resp, err = session.Get(ctx, urlWithQuery, "", headers) + + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + session.DiscardHTTPResponse(resp) + return + } + + var respOnyphe OnypheResponse + err = jsoniter.NewDecoder(resp.Body).Decode(&respOnyphe) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + session.DiscardHTTPResponse(resp) + return + } + + session.DiscardHTTPResponse(resp) + + for _, record := range respOnyphe.Results { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: record.Hostname} + s.results++ + } + + if len(respOnyphe.Results) == 0 || (respOnyphe.Page)*respOnyphe.PageSize >= respOnyphe.Total { + break + } + + page++ + + } + }() + + return results +} + +// Name returns the name of the source +func (s *Source) Name() string { + return "onhype" +} + +func (s *Source) IsDefault() bool { + return true +} + +func (s *Source) HasRecursiveSupport() bool { + return false +} + +func (s *Source) NeedsKey() bool { + return true +} + +func (s *Source) AddApiKeys(keys []string) { + s.apiKeys = keys +} + +func (s *Source) Statistics() subscraping.Statistics { + return subscraping.Statistics{ + Errors: s.errors, + Results: s.results, + TimeTaken: s.timeTaken, + Skipped: s.skipped, + } +} From 648f1e131125db26632d0a08d84dbd29a11e6957 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fan=20Can=20Bak=C4=B1r?= Date: Thu, 2 Oct 2025 13:27:22 +0300 Subject: [PATCH 2/4] fix pagination --- pkg/subscraping/sources/onhype/onhype.go | 191 +++++++++++++++++++++-- 1 file changed, 181 insertions(+), 10 deletions(-) diff --git a/pkg/subscraping/sources/onhype/onhype.go b/pkg/subscraping/sources/onhype/onhype.go index 916ed51b5..6eb33d68a 100644 --- a/pkg/subscraping/sources/onhype/onhype.go +++ b/pkg/subscraping/sources/onhype/onhype.go @@ -3,13 +3,13 @@ package onhype import ( "context" + "encoding/json" "fmt" "net/http" "net/url" + "strconv" "time" - jsoniter "github.com/json-iterator/go" - "github.com/projectdiscovery/subfinder/v2/pkg/subscraping" ) @@ -19,13 +19,18 @@ type OnypheResponse struct { Page int `json:"page"` PageSize int `json:"page_size"` Total int `json:"total"` + MaxPage int `json:"max_page"` } type Result struct { - Hostname string `json:"hostname"` + Subdomains []string `json:"subdomains"` + Hostname string `json:"hostname"` + Forward string `json:"forward"` + Reverse string `json:"reverse"` + Host string `json:"host"` + Domain string `json:"domain"` } -// Source is the passive scraping agent type Source struct { apiKeys []string timeTaken time.Duration @@ -55,12 +60,14 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se headers := map[string]string{"Content-Type": "application/json", "Authorization": "bearer " + randomApiKey} page := 1 + pageSize := 1000 + for { var resp *http.Response var err error - urlWithQuery := fmt.Sprintf("https://www.onyphe.io/api/v2/search/?q=%s&page=%d&size=10", - url.QueryEscape("category:resolver domain:"+domain), page) + urlWithQuery := fmt.Sprintf("https://www.onyphe.io/api/v2/search/?q=%s&page=%d&size=%d", + url.QueryEscape("category:resolver domain:"+domain), page, pageSize) resp, err = session.Get(ctx, urlWithQuery, "", headers) if err != nil { @@ -71,7 +78,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se } var respOnyphe OnypheResponse - err = jsoniter.NewDecoder(resp.Body).Decode(&respOnyphe) + err = json.NewDecoder(resp.Body).Decode(&respOnyphe) if err != nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ @@ -82,11 +89,30 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se session.DiscardHTTPResponse(resp) for _, record := range respOnyphe.Results { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: record.Hostname} - s.results++ + for _, subdomain := range record.Subdomains { + if subdomain != "" { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} + s.results++ + } + } + + if record.Hostname != "" { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: record.Hostname} + s.results++ + } + + if record.Forward != "" { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: record.Forward} + s.results++ + } + + if record.Reverse != "" { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: record.Reverse} + s.results++ + } } - if len(respOnyphe.Results) == 0 || (respOnyphe.Page)*respOnyphe.PageSize >= respOnyphe.Total { + if len(respOnyphe.Results) == 0 || page >= respOnyphe.MaxPage { break } @@ -127,3 +153,148 @@ func (s *Source) Statistics() subscraping.Statistics { Skipped: s.skipped, } } + +type OnypheResponseRaw struct { + Error int `json:"error"` + Results []Result `json:"results"` + Page json.RawMessage `json:"page"` + PageSize json.RawMessage `json:"page_size"` + Total json.RawMessage `json:"total"` + MaxPage json.RawMessage `json:"max_page"` +} + +func (o *OnypheResponse) UnmarshalJSON(data []byte) error { + var raw OnypheResponseRaw + if err := json.Unmarshal(data, &raw); err != nil { + return err + } + + o.Error = raw.Error + o.Results = raw.Results + + if pageStr := string(raw.Page); pageStr != "" { + if page, err := strconv.Atoi(pageStr); err == nil { + o.Page = page + } else { + var pageStrQuoted string + if err := json.Unmarshal(raw.Page, &pageStrQuoted); err == nil { + if page, err := strconv.Atoi(pageStrQuoted); err == nil { + o.Page = page + } + } + } + } + + if pageSizeStr := string(raw.PageSize); pageSizeStr != "" { + if pageSize, err := strconv.Atoi(pageSizeStr); err == nil { + o.PageSize = pageSize + } else { + var pageSizeStrQuoted string + if err := json.Unmarshal(raw.PageSize, &pageSizeStrQuoted); err == nil { + if pageSize, err := strconv.Atoi(pageSizeStrQuoted); err == nil { + o.PageSize = pageSize + } + } + } + } + + if totalStr := string(raw.Total); totalStr != "" { + if total, err := strconv.Atoi(totalStr); err == nil { + o.Total = total + } else { + var totalStrQuoted string + if err := json.Unmarshal(raw.Total, &totalStrQuoted); err == nil { + if total, err := strconv.Atoi(totalStrQuoted); err == nil { + o.Total = total + } + } + } + } + + if maxPageStr := string(raw.MaxPage); maxPageStr != "" { + if maxPage, err := strconv.Atoi(maxPageStr); err == nil { + o.MaxPage = maxPage + } else { + var maxPageStrQuoted string + if err := json.Unmarshal(raw.MaxPage, &maxPageStrQuoted); err == nil { + if maxPage, err := strconv.Atoi(maxPageStrQuoted); err == nil { + o.MaxPage = maxPage + } + } + } + } + + return nil +} + +type ResultRaw struct { + Subdomains json.RawMessage `json:"subdomains"` + Hostname json.RawMessage `json:"hostname"` + Forward json.RawMessage `json:"forward"` + Reverse json.RawMessage `json:"reverse"` + Host json.RawMessage `json:"host"` + Domain json.RawMessage `json:"domain"` +} + +func (r *Result) UnmarshalJSON(data []byte) error { + var raw ResultRaw + if err := json.Unmarshal(data, &raw); err != nil { + return err + } + + var subdomains []string + if err := json.Unmarshal(raw.Subdomains, &subdomains); err == nil { + r.Subdomains = subdomains + } else { + var subdomainStr string + if err := json.Unmarshal(raw.Subdomains, &subdomainStr); err == nil { + r.Subdomains = []string{subdomainStr} + } + } + + if len(raw.Hostname) > 0 { + var hostnameStr string + if err := json.Unmarshal(raw.Hostname, &hostnameStr); err == nil { + r.Hostname = hostnameStr + } else { + var hostnameArr []string + if err := json.Unmarshal(raw.Hostname, &hostnameArr); err == nil && len(hostnameArr) > 0 { + r.Hostname = hostnameArr[0] + } + } + } + + if len(raw.Forward) > 0 { + json.Unmarshal(raw.Forward, &r.Forward) + } + + if len(raw.Reverse) > 0 { + json.Unmarshal(raw.Reverse, &r.Reverse) + } + + if len(raw.Host) > 0 { + var hostStr string + if err := json.Unmarshal(raw.Host, &hostStr); err == nil { + r.Host = hostStr + } else { + var hostArr []string + if err := json.Unmarshal(raw.Host, &hostArr); err == nil && len(hostArr) > 0 { + r.Host = hostArr[0] + } + } + } + + if len(raw.Domain) > 0 { + var domainStr string + if err := json.Unmarshal(raw.Domain, &domainStr); err == nil { + r.Domain = domainStr + } else { + var domainArr []string + if err := json.Unmarshal(raw.Domain, &domainArr); err == nil && len(domainArr) > 0 { + r.Domain = domainArr[0] + } + } + } + + return nil +} From d236bba3dd2151cb57ebc36f64a0e9665dff1da1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fan=20Can=20Bak=C4=B1r?= Date: Thu, 2 Oct 2025 13:32:23 +0300 Subject: [PATCH 3/4] fix lint err --- pkg/subscraping/sources/onhype/onhype.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/subscraping/sources/onhype/onhype.go b/pkg/subscraping/sources/onhype/onhype.go index 6eb33d68a..6a571081a 100644 --- a/pkg/subscraping/sources/onhype/onhype.go +++ b/pkg/subscraping/sources/onhype/onhype.go @@ -265,11 +265,11 @@ func (r *Result) UnmarshalJSON(data []byte) error { } if len(raw.Forward) > 0 { - json.Unmarshal(raw.Forward, &r.Forward) + _ = json.Unmarshal(raw.Forward, &r.Forward) } if len(raw.Reverse) > 0 { - json.Unmarshal(raw.Reverse, &r.Reverse) + _ = json.Unmarshal(raw.Reverse, &r.Reverse) } if len(raw.Host) > 0 { From 7823d41dd12ed53bf1fee4b0a41af3a25b360e61 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Do=C4=9Fan=20Can=20Bak=C4=B1r?= Date: Thu, 2 Oct 2025 13:35:06 +0300 Subject: [PATCH 4/4] fix typo --- pkg/passive/sources.go | 4 ++-- pkg/passive/sources_test.go | 4 ++-- .../sources/{onhype/onhype.go => onyphe/onyphe.go} | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) rename pkg/subscraping/sources/{onhype/onhype.go => onyphe/onyphe.go} (99%) diff --git a/pkg/passive/sources.go b/pkg/passive/sources.go index f91fdf860..a2065b415 100644 --- a/pkg/passive/sources.go +++ b/pkg/passive/sources.go @@ -37,7 +37,7 @@ import ( "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/intelx" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/leakix" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/netlas" - "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/onhype" + "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/onyphe" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/pugrecon" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/quake" "github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/rapiddns" @@ -80,7 +80,7 @@ var AllSources = [...]subscraping.Source{ &hunter.Source{}, &intelx.Source{}, &netlas.Source{}, - &onhype.Source{}, + &onyphe.Source{}, &leakix.Source{}, &quake.Source{}, &pugrecon.Source{}, diff --git a/pkg/passive/sources_test.go b/pkg/passive/sources_test.go index 53f8a6b92..540990ecd 100644 --- a/pkg/passive/sources_test.go +++ b/pkg/passive/sources_test.go @@ -33,7 +33,7 @@ var ( "hackertarget", "intelx", "netlas", - "onhype", + "onyphe", "quake", "pugrecon", "rapiddns", @@ -79,7 +79,7 @@ var ( "fullhunt", "hackertarget", "intelx", - "onhype", + "onyphe", "quake", "redhuntlabs", "robtex", diff --git a/pkg/subscraping/sources/onhype/onhype.go b/pkg/subscraping/sources/onyphe/onyphe.go similarity index 99% rename from pkg/subscraping/sources/onhype/onhype.go rename to pkg/subscraping/sources/onyphe/onyphe.go index 6a571081a..6ea1dcbb1 100644 --- a/pkg/subscraping/sources/onhype/onhype.go +++ b/pkg/subscraping/sources/onyphe/onyphe.go @@ -1,5 +1,5 @@ -// Package onhype logic -package onhype +// Package onyphe logic +package onyphe import ( "context" @@ -126,7 +126,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se // Name returns the name of the source func (s *Source) Name() string { - return "onhype" + return "onyphe" } func (s *Source) IsDefault() bool {