projectdiscovery · ehsandeep · Sep 12, 2023 · Aug 23, 2023 · Aug 23, 2023 · Aug 24, 2023
diff --git a/README.md b/README.md
@@ -121,7 +121,7 @@ go install -v github.com/projectdiscovery/subfinder/v2/cmd/subfinder@latest
 
 `subfinder` can be used right after the installation, however the following services require configuring API keys to work:
 
-[BeVigil](https://bevigil.com/osint-api), [BinaryEdge](https://binaryedge.io), [BufferOver](https://tls.bufferover.run), [C99](https://api.c99.nl/), [Censys](https://censys.io), [CertSpotter](https://sslmate.com/certspotter/api/), [Chaos](https://chaos.projectdiscovery.io), [Chinaz](http://my.chinaz.com/ChinazAPI/DataCenter/MyDataApi), [DnsDB](https://api.dnsdb.info), [Fofa](https://fofa.info/static_pages/api_help), [FullHunt](https://fullhunt.io), [GitHub](https://github.com), [Intelx](https://intelx.io), [PassiveTotal](http://passivetotal.org), [quake](https://quake.360.cn), [Robtex](https://www.robtex.com/api/), [RedHunt Labs](https://devportal.redhuntlabs.com), [SecurityTrails](http://securitytrails.com), [Shodan](https://shodan.io), [ThreatBook](https://x.threatbook.cn/en), [VirusTotal](https://www.virustotal.com), [WhoisXML API](https://whoisxmlapi.com/), [ZoomEye API](https://api.zoomeye.org), [dnsrepo](https://dnsrepo.noc.org), [Hunter](https://hunter.qianxin.com/), [Facebook](https://developers.facebook.com)
+[BeVigil](https://bevigil.com/osint-api), [BinaryEdge](https://binaryedge.io), [BufferOver](https://tls.bufferover.run), [C99](https://api.c99.nl/), [Censys](https://censys.io), [CertSpotter](https://sslmate.com/certspotter/api/), [Chaos](https://chaos.projectdiscovery.io), [Chinaz](http://my.chinaz.com/ChinazAPI/DataCenter/MyDataApi), [DnsDB](https://api.dnsdb.info), [Fofa](https://fofa.info/static_pages/api_help), [FullHunt](https://fullhunt.io), [GitHub](https://github.com), [Intelx](https://intelx.io), [PassiveTotal](http://passivetotal.org), [quake](https://quake.360.cn), [Robtex](https://www.robtex.com/api/), [SecurityTrails](http://securitytrails.com), [Shodan](https://shodan.io), [ThreatBook](https://x.threatbook.cn/en), [VirusTotal](https://www.virustotal.com), [WhoisXML API](https://whoisxmlapi.com/), [ZoomEye](https://www.zoomeye.org), [ZoomEye API](https://api.zoomeye.org), [dnsrepo](https://dnsrepo.noc.org), [Hunter](https://hunter.qianxin.com/), [Facebook](https://developers.facebook.com), [BuiltWith](https://api.builtwith.com/domain-api)
 
 You can also use the `subfinder -ls` command to display all the available sources.
 

diff --git a/v2/pkg/passive/sources.go b/v2/pkg/passive/sources.go
@@ -13,6 +13,7 @@ import (
 	"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/bevigil"
 	"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/binaryedge"
 	"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/bufferover"
+	"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/builtwith"
 	"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/c99"
 	"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/censys"
 	"github.com/projectdiscovery/subfinder/v2/pkg/subscraping/sources/certspotter"
@@ -92,6 +93,7 @@ var AllSources = [...]subscraping.Source{
 	&facebook.Source{},
 	// &threatminer.Source{}, // failing  api
 	// &reconcloud.Source{}, // failing due to cloudflare bot protection
+	&builtwith.Source{},
 }
 
 var sourceWarnings = mapsutil.NewSyncLockMap[string, string](

diff --git a/v2/pkg/passive/sources_test.go b/v2/pkg/passive/sources_test.go
@@ -52,6 +52,7 @@ var (
 		"facebook",
 		// "threatminer",
 		// "reconcloud",
+		"builtwith",
 	}
 
 	expectedDefaultSources = []string{
@@ -86,6 +87,7 @@ var (
 		"facebook",
 		// "threatminer",
 		// "reconcloud",
+		"builtwith",
 	}
 
 	expectedDefaultRecursiveSources = []string{

diff --git a/v2/pkg/passive/sources_wo_auth_test.go b/v2/pkg/passive/sources_wo_auth_test.go
@@ -24,10 +24,12 @@ func TestSourcesWithoutKeys(t *testing.T) {
 	}
 
 	ignoredSources := []string{
-		"commoncrawl",  // commoncrawl is under resourced and will likely time-out so step over it for this test https://groups.google.com/u/2/g/common-crawl/c/3QmQjFA_3y4/m/vTbhGqIBBQAJ
-		"riddler",      // Fails with 403: There might be too much traffic or a configuration error
-		"crtsh",        // Fails in GH Action (possibly IP-based ban) causing a timeout.
-		"hackertarget", // Fails in GH Action (possibly IP-based ban) but works locally
+		"commoncrawl",    // commoncrawl is under resourced and will likely time-out so step over it for this test https://groups.google.com/u/2/g/common-crawl/c/3QmQjFA_3y4/m/vTbhGqIBBQAJ
+		"riddler",        // Fails with 403: There might be too much traffic or a configuration error
+		"crtsh",          // Fails in GH Action (possibly IP-based ban) causing a timeout.
+		"hackertarget",   // Fails in GH Action (possibly IP-based ban) but works locally
+		"waybackarchive", // Fails randomly
+		"alienvault",     // 503 Service Temporarily Unavailable
 	}
 
 	domain := "hackerone.com"

diff --git a/v2/pkg/subscraping/sources/builtwith/builtwith.go b/v2/pkg/subscraping/sources/builtwith/builtwith.go
@@ -0,0 +1,114 @@
+// Package builtwith logic
+package builtwith
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	jsoniter "github.com/json-iterator/go"
+
+	"github.com/projectdiscovery/subfinder/v2/pkg/subscraping"
+)
+
+type response struct {
+	Results []resultItem `json:"Results"`
+}
+
+type resultItem struct {
+	Result result `json:"Result"`
+}
+
+type result struct {
+	Paths []path `json:"Paths"`
+}
+
+type path struct {
+	Domain    string `json:"Domain"`
+	Url       string `json:"Url"`
+	SubDomain string `json:"SubDomain"`
+}
+
+// Source is the passive scraping agent
+type Source struct {
+	apiKeys   []string
+	timeTaken time.Duration
+	errors    int
+	results   int
+	skipped   bool
+}
+
+// Run function returns all subdomains found with the service
+func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result {
+	results := make(chan subscraping.Result)
+	s.errors = 0
+	s.results = 0
+
+	go func() {
+		defer func(startTime time.Time) {
+			s.timeTaken = time.Since(startTime)
+			close(results)
+		}(time.Now())
+
+		randomApiKey := subscraping.PickRandom(s.apiKeys, s.Name())
+		if randomApiKey == "" {
+			return
+		}
+
+		resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://api.builtwith.com/v21/api.json?KEY=%s&HIDETEXT=yes&HIDEDL=yes&NOLIVE=yes&NOMETA=yes&NOPII=yes&NOATTR=yes&LOOKUP=%s", randomApiKey, domain))
+		if err != nil {
+			results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
+			s.errors++
+			session.DiscardHTTPResponse(resp)
+			return
+		}
+
+		var data response
+		err = jsoniter.NewDecoder(resp.Body).Decode(&data)
+		if err != nil {
+			results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err}
+			s.errors++
+			resp.Body.Close()
+			return
+		}
+		resp.Body.Close()
+		for _, result := range data.Results {
+			for _, path := range result.Result.Paths {
+				results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: fmt.Sprintf("%s.%s", path.SubDomain, path.Domain)}
+				s.results++
+			}
+		}
+	}()
+
+	return results
+}
+
+// Name returns the name of the source
+func (s *Source) Name() string {
+	return "builtwith"
+}
+
+func (s *Source) IsDefault() bool {
+	return true
+}
+
+func (s *Source) HasRecursiveSupport() bool {
+	return false
+}
+
+func (s *Source) NeedsKey() bool {
+	return true
+}
+
+func (s *Source) AddApiKeys(keys []string) {
+	s.apiKeys = keys
+}
+
+func (s *Source) Statistics() subscraping.Statistics {
+	return subscraping.Statistics{
+		Errors:    s.errors,
+		Results:   s.results,
+		TimeTaken: s.timeTaken,
+		Skipped:   s.skipped,
+	}
+}