这是indexloc提供的服务，不要输入任何密码
Skip to content

Fixed crt.sh SQL query #700

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Nov 21, 2022
Merged

Fixed crt.sh SQL query #700

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
1a03959
Reading this thread https://groups.google.com/g/crtsh/c/sUmV0mBz8bQ/m…
Nov 9, 2022
a45c460
Merge branch 'dev' into fix-crtsh-sql
vzamanillo Nov 9, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 31 additions & 6 deletions v2/pkg/subscraping/sources/crtsh/crtsh.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,11 +49,33 @@ func (s *Source) getSubdomainsFromSQL(domain string, session *subscraping.Sessio

defer db.Close()

pattern := "%." + domain
query := `SELECT DISTINCT ci.NAME_VALUE as domain FROM certificate_identity ci
WHERE reverse(lower(ci.NAME_VALUE)) LIKE reverse(lower($1))
ORDER BY ci.NAME_VALUE`
rows, err := db.Query(query, pattern)
query := `WITH ci AS (
SELECT min(sub.CERTIFICATE_ID) ID,
min(sub.ISSUER_CA_ID) ISSUER_CA_ID,
array_agg(DISTINCT sub.NAME_VALUE) NAME_VALUES,
x509_commonName(sub.CERTIFICATE) COMMON_NAME,
x509_notBefore(sub.CERTIFICATE) NOT_BEFORE,
x509_notAfter(sub.CERTIFICATE) NOT_AFTER,
encode(x509_serialNumber(sub.CERTIFICATE), 'hex') SERIAL_NUMBER
FROM (SELECT *
FROM certificate_and_identities cai
WHERE plainto_tsquery('certwatch', $1) @@ identities(cai.CERTIFICATE)
AND cai.NAME_VALUE ILIKE ('%' || $1 || '%')
LIMIT 10000
) sub
GROUP BY sub.CERTIFICATE
)
SELECT array_to_string(ci.NAME_VALUES, chr(10)) NAME_VALUE
FROM ci
LEFT JOIN LATERAL (
SELECT min(ctle.ENTRY_TIMESTAMP) ENTRY_TIMESTAMP
FROM ct_log_entry ctle
WHERE ctle.CERTIFICATE_ID = ci.ID
) le ON TRUE,
ca
WHERE ci.ISSUER_CA_ID = ca.ID
ORDER BY le.ENTRY_TIMESTAMP DESC NULLS LAST;`
rows, err := db.Query(query, domain)
if err != nil {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
return 0
Expand All @@ -72,8 +94,11 @@ func (s *Source) getSubdomainsFromSQL(domain string, session *subscraping.Sessio
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
return count
}

count++
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: session.Extractor.FindString(data)}
for _, subdomain := range strings.Split(data, "\n") {
results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: session.Extractor.FindString(subdomain)}
}
}
return count
}
Expand Down