scraper

type CollyScraper struct {
	Collector             *colly.Collector
	Transport             *http.Transport
	Response              *http.Response
	TimeoutSeconds        int
	LoadingTimeoutSeconds int
	UserAgent             string
	// contains filtered or unexported fields
}

type GoWapTransport struct {
	*http.Transport
	// contains filtered or unexported fields
}

type RodScraper struct {
	Browser               *rod.Browser
	Page                  *rod.Page
	TimeoutSeconds        int
	LoadingTimeoutSeconds int
	UserAgent             string
	// contains filtered or unexported fields
}

type ScrapedData struct {
	URLs       ScrapedURL
	HTML       string
	Headers    map[string][]string
	Scripts    []string
	Cookies    map[string]string
	Meta       map[string][]string
	DNS        map[string][]string
	CertIssuer []string
}

type ScrapedURL struct {
	URL    string `json:"url,omitempty"`
	Status int    `json:"status,omitempty"`
}

type Scraper interface {
	Init(url string) error
	CanRenderPage() bool
	Scrape(paramURL string) (*ScrapedData, error)
	EvalJS(jsProp string) (*string, error)
	SetDepth(depth int)
}