这是indexloc提供的服务，不要输入任何密码
Skip to content

Fix paginations #1603

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 6 commits into from
Jun 20, 2025
Merged

Fix paginations #1603

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
4a76a79
fix quake pagination
dogancanbakir Jun 17, 2025
d0a7433
update page param
dogancanbakir Jun 17, 2025
313cca6
skip empty
dogancanbakir Jun 17, 2025
74983c6
Merge branch 'dev' into fix_paginations
dogancanbakir Jun 20, 2025
1b943c5
fix test
dogancanbakir Jun 20, 2025
ce6ae5f
minor
dogancanbakir Jun 20, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion v2/pkg/subscraping/sources/hunter/hunter.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,7 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
for currentPage := 1; currentPage <= pages; currentPage++ {
// hunter api doc https://hunter.qianxin.com/home/helpCenter?r=5-1-2
qbase64 := base64.URLEncoding.EncodeToString(fmt.Appendf(nil, "domain=\"%s\"", domain))
resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://hunter.qianxin.com/openApi/search?api-key=%s&search=%s&page=1&page_size=100&is_web=3", randomApiKey, qbase64))
resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://hunter.qianxin.com/openApi/search?api-key=%s&search=%s&page=%d&page_size=100&is_web=3", randomApiKey, qbase64, currentPage))
if err != nil && resp == nil {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
s.errors++
Expand Down
66 changes: 40 additions & 26 deletions v2/pkg/subscraping/sources/quake/quake.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,44 +58,58 @@ func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Se
}

// quake api doc https://quake.360.cn/quake/#/help
var requestBody = fmt.Appendf(nil, `{"query":"domain: %s", "include":["service.http.host"], "latest": true, "start":0, "size":500}`, domain)
resp, err := session.Post(ctx, "https://quake.360.net/api/v3/search/quake_service", "", map[string]string{
"Content-Type": "application/json", "X-QuakeToken": randomApiKey,
}, bytes.NewReader(requestBody))
if err != nil {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
s.errors++
session.DiscardHTTPResponse(resp)
return
}
var pageSize = 500
var start = 0
var totalResults = -1

for {
var requestBody = fmt.Appendf(nil, `{"query":"domain: %s", "include":["service.http.host"], "latest": true, "size":%d, "start":%d}`, domain, pageSize, start)
resp, err := session.Post(ctx, "https://quake.360.net/api/v3/search/quake_service", "", map[string]string{
"Content-Type": "application/json", "X-QuakeToken": randomApiKey,
}, bytes.NewReader(requestBody))
if err != nil {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
s.errors++
session.DiscardHTTPResponse(resp)
return
}

var response quakeResults
err = jsoniter.NewDecoder(resp.Body).Decode(&response)
if err != nil {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
s.errors++
var response quakeResults
err = jsoniter.NewDecoder(resp.Body).Decode(&response)
if err != nil {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
s.errors++
session.DiscardHTTPResponse(resp)
return
}
session.DiscardHTTPResponse(resp)
return
}
session.DiscardHTTPResponse(resp)

if response.Code != 0 {
results <- subscraping.Result{
Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("%s", response.Message),
if response.Code != 0 {
results <- subscraping.Result{
Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("%s", response.Message),
}
s.errors++
return
}

if totalResults == -1 {
totalResults = response.Meta.Pagination.Total
}
s.errors++
return
}

if response.Meta.Pagination.Total > 0 {
for _, quakeDomain := range response.Data {
subdomain := quakeDomain.Service.HTTP.Host
if strings.ContainsAny(subdomain, "暂无权限") {
subdomain = ""
continue
}
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain}
s.results++
}

if len(response.Data) == 0 || start+pageSize >= totalResults {
break
}

start += pageSize
}
}()

Expand Down
2 changes: 1 addition & 1 deletion v2/pkg/subscraping/sources/rsecloud/rsecloud.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,7 @@ func (s *Source) IsDefault() bool {
}

func (s *Source) HasRecursiveSupport() bool {
return true
return false
}

func (s *Source) NeedsKey() bool {
Expand Down
Loading