这是indexloc提供的服务，不要输入任何密码
Skip to content

fix github perf issue #1421

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Nov 27, 2024
Merged

fix github perf issue #1421

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
5ac4567
fix github perf issue
dogancanbakir Oct 26, 2024
e82675f
Merge branch 'dev' into fix_github_perf_issue
ehsandeep Nov 4, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion v2/pkg/runner/enumerate.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ func (r *Runner) EnumerateSingleDomainWithCtx(ctx context.Context, domain string
for result := range passiveResults {
switch result.Type {
case subscraping.Error:
gologger.Warning().Msgf("Could not run source %s: %s\n", result.Source, result.Error)
gologger.Warning().Msgf("Encountered an error with source %s: %s\n", result.Source, result.Error)
case subscraping.Subdomain:
// Validate the subdomain found and remove wildcards from
if !strings.HasSuffix(result.Value, "."+domain) {
Expand Down
67 changes: 43 additions & 24 deletions v2/pkg/subscraping/sources/github/github.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import (
"regexp"
"strconv"
"strings"
"sync"
"time"

jsoniter "github.com/json-iterator/go"
Expand Down Expand Up @@ -142,40 +143,58 @@ func (s *Source) enumerate(ctx context.Context, searchURL string, domainRegexp *

// proccesItems process github response items
func (s *Source) proccesItems(ctx context.Context, items []item, domainRegexp *regexp.Regexp, name string, session *subscraping.Session, results chan subscraping.Result) error {
for _, item := range items {
// find subdomains in code
resp, err := session.SimpleGet(ctx, rawURL(item.HTMLURL))
if err != nil {
if resp != nil && resp.StatusCode != http.StatusNotFound {
session.DiscardHTTPResponse(resp)
var wg sync.WaitGroup
errChan := make(chan error, len(items))

for _, responseItem := range items {
wg.Add(1)
go func(responseItem item) {
defer wg.Done()

// find subdomains in code
resp, err := session.SimpleGet(ctx, rawURL(responseItem.HTMLURL))
if err != nil {
if resp != nil && resp.StatusCode != http.StatusNotFound {
session.DiscardHTTPResponse(resp)
}
errChan <- err
return
}
return err
}

if resp.StatusCode == http.StatusOK {
scanner := bufio.NewScanner(resp.Body)
for scanner.Scan() {
line := scanner.Text()
if line == "" {
continue
if resp.StatusCode == http.StatusOK {
scanner := bufio.NewScanner(resp.Body)
for scanner.Scan() {
line := scanner.Text()
if line == "" {
continue
}
for _, subdomain := range domainRegexp.FindAllString(normalizeContent(line), -1) {
results <- subscraping.Result{Source: name, Type: subscraping.Subdomain, Value: subdomain}
s.results++
}
}
for _, subdomain := range domainRegexp.FindAllString(normalizeContent(line), -1) {
resp.Body.Close()
}

// find subdomains in text matches
for _, textMatch := range responseItem.TextMatches {
for _, subdomain := range domainRegexp.FindAllString(normalizeContent(textMatch.Fragment), -1) {
results <- subscraping.Result{Source: name, Type: subscraping.Subdomain, Value: subdomain}
s.results++

}
}
resp.Body.Close()
}
}(responseItem)
}

// find subdomains in text matches
for _, textMatch := range item.TextMatches {
for _, subdomain := range domainRegexp.FindAllString(normalizeContent(textMatch.Fragment), -1) {
results <- subscraping.Result{Source: name, Type: subscraping.Subdomain, Value: subdomain}
s.results++
}
wg.Wait()
close(errChan)

for err := range errChan {
if err != nil {
return err
}
}

return nil
}

Expand Down
Loading