diff --git a/v2/pkg/subscraping/sources/hunter/hunter.go b/v2/pkg/subscraping/sources/hunter/hunter.go index fc2e88a11..960563279 100644 --- a/v2/pkg/subscraping/sources/hunter/hunter.go +++ b/v2/pkg/subscraping/sources/hunter/hunter.go @@ -60,7 +60,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se for currentPage := 1; currentPage <= pages; currentPage++ { // hunter api doc https://hunter.qianxin.com/home/helpCenter?r=5-1-2 qbase64 := base64.URLEncoding.EncodeToString(fmt.Appendf(nil, "domain=\"%s\"", domain)) - resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://hunter.qianxin.com/openApi/search?api-key=%s&search=%s&page=1&page_size=100&is_web=3", randomApiKey, qbase64)) + resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://hunter.qianxin.com/openApi/search?api-key=%s&search=%s&page=%d&page_size=100&is_web=3", randomApiKey, qbase64, currentPage)) if err != nil && resp == nil { results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} s.errors++ diff --git a/v2/pkg/subscraping/sources/quake/quake.go b/v2/pkg/subscraping/sources/quake/quake.go index b5f6864e1..5cac4ed2e 100644 --- a/v2/pkg/subscraping/sources/quake/quake.go +++ b/v2/pkg/subscraping/sources/quake/quake.go @@ -58,44 +58,58 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se } // quake api doc https://quake.360.cn/quake/#/help - var requestBody = fmt.Appendf(nil, `{"query":"domain: %s", "include":["service.http.host"], "latest": true, "start":0, "size":500}`, domain) - resp, err := session.Post(ctx, "https://quake.360.net/api/v3/search/quake_service", "", map[string]string{ - "Content-Type": "application/json", "X-QuakeToken": randomApiKey, - }, bytes.NewReader(requestBody)) - if err != nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - s.errors++ - session.DiscardHTTPResponse(resp) - return - } + var pageSize = 500 + var start = 0 + var totalResults = -1 + + for { + var requestBody = fmt.Appendf(nil, `{"query":"domain: %s", "include":["service.http.host"], "latest": true, "size":%d, "start":%d}`, domain, pageSize, start) + resp, err := session.Post(ctx, "https://quake.360.net/api/v3/search/quake_service", "", map[string]string{ + "Content-Type": "application/json", "X-QuakeToken": randomApiKey, + }, bytes.NewReader(requestBody)) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + session.DiscardHTTPResponse(resp) + return + } - var response quakeResults - err = jsoniter.NewDecoder(resp.Body).Decode(&response) - if err != nil { - results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} - s.errors++ + var response quakeResults + err = jsoniter.NewDecoder(resp.Body).Decode(&response) + if err != nil { + results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} + s.errors++ + session.DiscardHTTPResponse(resp) + return + } session.DiscardHTTPResponse(resp) - return - } - session.DiscardHTTPResponse(resp) - if response.Code != 0 { - results <- subscraping.Result{ - Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("%s", response.Message), + if response.Code != 0 { + results <- subscraping.Result{ + Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("%s", response.Message), + } + s.errors++ + return + } + + if totalResults == -1 { + totalResults = response.Meta.Pagination.Total } - s.errors++ - return - } - if response.Meta.Pagination.Total > 0 { for _, quakeDomain := range response.Data { subdomain := quakeDomain.Service.HTTP.Host if strings.ContainsAny(subdomain, "暂无权限") { - subdomain = "" + continue } results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} s.results++ } + + if len(response.Data) == 0 || start+pageSize >= totalResults { + break + } + + start += pageSize } }() diff --git a/v2/pkg/subscraping/sources/rsecloud/rsecloud.go b/v2/pkg/subscraping/sources/rsecloud/rsecloud.go index c06361d67..8601b7732 100644 --- a/v2/pkg/subscraping/sources/rsecloud/rsecloud.go +++ b/v2/pkg/subscraping/sources/rsecloud/rsecloud.go @@ -96,7 +96,7 @@ func (s *Source) IsDefault() bool { } func (s *Source) HasRecursiveSupport() bool { - return true + return false } func (s *Source) NeedsKey() bool {