From e1b170d793cf8f5a2ec7f012e3f2171feaa8cad6 Mon Sep 17 00:00:00 2001 From: Edward Marshall Date: Tue, 25 Apr 2017 18:57:11 +0100 Subject: [PATCH 01/13] V2 Rewrite, initial commit --- Dockerfile | 26 +++- METRICS.md | 38 +++++ README.md | 39 ++--- config/config.go | 80 ++++++++++ exporter/collect.go | 36 +++++ exporter/gather.go | 146 ++++++++++++++++++ exporter/metrics.go | 74 +++++++++ exporter/structs.go | 33 ++++ main.go | 53 +++++++ .../github_exporter.py | 2 +- 10 files changed, 494 insertions(+), 33 deletions(-) create mode 100644 METRICS.md create mode 100644 config/config.go create mode 100644 exporter/collect.go create mode 100644 exporter/gather.go create mode 100644 exporter/metrics.go create mode 100644 exporter/structs.go create mode 100644 main.go rename github_exporter.py => python/github_exporter.py (99%) diff --git a/Dockerfile b/Dockerfile index c1db91d0..bba1f22b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,10 +1,24 @@ -FROM python:3.5-alpine +FROM golang:1.8.0-alpine +LABEL maintainer "Infinity Works" -RUN pip install prometheus_client requests +EXPOSE 9173 -ENV BIND_PORT 9171 +ENV GOPATH=/go + LISTEN_PORT=9173 -ADD . /usr/src/app -WORKDIR /usr/src/app +RUN addgroup exporter \ + && adduser -S -G exporter exporter \ + && apk --update add ca-certificates \ + && apk --update add --virtual build-deps git -CMD ["python", "github_exporter.py"] +COPY ./ /go/src/github.com/infinityworksltd/github-exporter + +WORKDIR /go/src/github.com/infinityworksltd/github-exporter + +RUN go get \ + && go test ./... \ + && go build -o /bin/main + +USER exporter + +CMD [ "/bin/main" ] \ No newline at end of file diff --git a/METRICS.md b/METRICS.md new file mode 100644 index 00000000..3fc4a650 --- /dev/null +++ b/METRICS.md @@ -0,0 +1,38 @@ +# Metrics + +Below are an example of the metrics as exposed by this exporter. + +``` +# HELP github_repo_open_issues open_issues +# TYPE github_repo_open_issues gauge +github_repo_open_issues{repo="docker-hub-exporter",user="infinityworksltd"} 1.0 +github_repo_open_issues{repo="prometheus-rancher-exporter",user="infinityworksltd"} 2.0 +# HELP github_repo_watchers watchers +# TYPE github_repo_watchers gauge +github_repo_watchers{repo="docker-hub-exporter",user="infinityworksltd"} 1.0 +github_repo_watchers{repo="prometheus-rancher-exporter",user="infinityworksltd"} 6.0 +# HELP github_repo_stars stars +# TYPE github_repo_stars gauge +github_repo_stars{repo="docker-hub-exporter",user="infinityworksltd"} 1.0 +github_repo_stars{repo="prometheus-rancher-exporter",user="infinityworksltd"} 6.0 +# HELP github_repo_forks forks +# TYPE github_repo_forks gauge +github_repo_forks{repo="docker-hub-exporter",user="infinityworksltd"} 0.0 +github_repo_forks{repo="prometheus-rancher-exporter",user="infinityworksltd"} 9.0 +# HELP github_repo_size_kb Size in KB for given repository +# TYPE github_repo_size_kb gauge +github_repo_size_kb{repo="docker-hub-exporter",user="infinityworksltd"} 44 +github_repo_size_kb{repo="prometheus-rancher-exporter",user="infinityworksltd"} 7242 +# HELP github_rate_limit Number of API queries allowed in a 60 minute window +# TYPE github_rate_limit gauge +github_rate_limit 60 +# HELP github_rate_remaining Number of API queries remaining in the current window +# TYPE github_rate_remaining gauge +github_rate_remaining 38 +# HELP github_rate_reset The time at which the current rate limit window resets in UTC epoch seconds +# TYPE github_rate_reset gauge +github_rate_reset 1.493139756e+09 +# HELP github_size_kb Size in KB for given repository +# TYPE github_size_kb gauge +github_size_kb{repo="CRUST",user="infinityworksltd"} 44 +``` \ No newline at end of file diff --git a/README.md b/README.md index 1a5eb2e3..f631848b 100644 --- a/README.md +++ b/README.md @@ -5,11 +5,20 @@ Exposes basic metrics for your repositories from the GitHub API, to a Prometheus ## Configuration This exporter is setup to take input from environment variables: -* `BIND_PORT` The port you wish to run the container on, defaults to `9171` + +### Required +* `ORGS` If supplied, the exporter will enumerate all repositories for that organization. Expected in the format "org1, org2". Can be ommited if using `REPOS` and not required +* `REPOS` If supplied, The images you wish to monitor, expected in the format "user/repo1, user/repo2". Can be across different Github users/orgs. Can be ommited if using `ORGS` and not required + + +### Optional * `GITHUB_TOKEN` If supplied, enables the user to supply a github authentication token that allows the API to be queried more often. Optional, but recommended. * `GITHUB_TOKEN_FILE` If supplied _instead of_ `GITHUB_TOKEN`, enables the user to supply a path to a file containing a github authentication token that allows the API to be queried more often. Optional, but recommended. -* `ORGS` If supplied, the exporter will enumerate all repositories for that organization. -* `REPOS` If supplied, The images you wish to monitor, expected in the format "user/repo1, user/repo2". Can be across different Github users/orgs. +* `API_URL` Github API URL, shouldn't need to change this. Defaults to `https://api.github.com` +* `LISTEN_PORT` The port you wish to run the container on, the Dockerfile defaults this to `9171` +* `METRICS_PATH` the metrics URL path you wish to use, defaults to `/metrics` +* `LOG_LEVEL` The level of logging the exporter will run with, defaults to `debug` + ## Install and deploy @@ -38,29 +47,7 @@ github-exporter: ## Metrics Metrics will be made available on port 9171 by default - -``` -# HELP github_repo_open_issues open_issues -# TYPE github_repo_open_issues gauge -github_repo_open_issues{repo="docker-hub-exporter",user="infinityworksltd"} 1.0 -github_repo_open_issues{repo="prometheus-rancher-exporter",user="infinityworksltd"} 2.0 -# HELP github_repo_watchers watchers -# TYPE github_repo_watchers gauge -github_repo_watchers{repo="docker-hub-exporter",user="infinityworksltd"} 1.0 -github_repo_watchers{repo="prometheus-rancher-exporter",user="infinityworksltd"} 6.0 -# HELP github_repo_stars stars -# TYPE github_repo_stars gauge -github_repo_stars{repo="docker-hub-exporter",user="infinityworksltd"} 1.0 -github_repo_stars{repo="prometheus-rancher-exporter",user="infinityworksltd"} 6.0 -# HELP github_repo_forks forks -# TYPE github_repo_forks gauge -github_repo_forks{repo="docker-hub-exporter",user="infinityworksltd"} 0.0 -github_repo_forks{repo="prometheus-rancher-exporter",user="infinityworksltd"} 9.0 -# HELP github_repo_has_issues has_issues -# TYPE github_repo_has_issues gauge -github_repo_has_issues{repo="docker-hub-exporter",user="infinityworksltd"} 1.0 -github_repo_has_issues{repo="prometheus-rancher-exporter",user="infinityworksltd"} 1.0 -``` +An example of these metrics can be found in the `METRICS.md` markdown file in the root of this repository ## Metadata [![](https://images.microbadger.com/badges/image/infinityworks/github-exporter.svg)](http://microbadger.com/images/infinityworks/github-exporter "Get your own image badge on microbadger.com") [![](https://images.microbadger.com/badges/version/infinityworks/github-exporter.svg)](http://microbadger.com/images/infinityworks/github-exporter "Get your own version badge on microbadger.com") diff --git a/config/config.go b/config/config.go new file mode 100644 index 00000000..d950d43a --- /dev/null +++ b/config/config.go @@ -0,0 +1,80 @@ +package config + +import ( + "fmt" + "strings" + + log "github.com/Sirupsen/logrus" + + "os" + + cfg "github.com/infinityworksltd/go-common/config" +) + +// Config struct holds all of the confgiguration +type Config struct { + *cfg.BaseConfig + APIURL string + Repositories string + Organisations string + APIToken string + APITokenFile string + TargetURLs []string +} + +// Init populates the Config struct based on environmental runtime configuration +func Init() Config { + + ac := cfg.Init() + + scraped, err := getScrapeURLs() + + if err != nil { + log.Errorf("Error initialising Configuration, Error: %v", err) + } + + appConfig := Config{ + &ac, + cfg.GetEnv("API_URL", "https://api.github.com"), + os.Getenv("REPOS"), + os.Getenv("ORGS"), + os.Getenv("GITHUB_TOKEN"), + os.Getenv("GITHUB_TOKEN_FILE"), + scraped, + } + + return appConfig +} + +// Init populates the Config struct based on environmental runtime configuration +func getScrapeURLs() ([]string, error) { + + urls := []string{} + apiURL := cfg.GetEnv("API_URL", "https://api.github.com") + repos := os.Getenv("REPOS") + orgs := os.Getenv("ORGS") + opts := "?&per_page=100" + + // User input validation, check that either repositories or organisations have been passed in + if len(repos) == 0 && len(orgs) == 0 { + return urls, fmt.Errorf("No organisations or repositories specified") + } + + if repos != "" { + rs := strings.Split(repos, ", ") + for _, x := range rs { + y := fmt.Sprintf("%s/repos/%s%s", apiURL, x, opts) + urls = append(urls, y) + } + } + + if orgs != "" { + o := strings.Split(orgs, ", ") + for _, x := range o { + y := fmt.Sprintf("%s/orgs/%s/repos%s", apiURL, x, opts) + urls = append(urls, y) + } + } + + return urls, nil +} diff --git a/exporter/collect.go b/exporter/collect.go new file mode 100644 index 00000000..bf2161b5 --- /dev/null +++ b/exporter/collect.go @@ -0,0 +1,36 @@ +package exporter + +import ( + log "github.com/Sirupsen/logrus" + "github.com/prometheus/client_golang/prometheus" +) + +// Describe describes all the metrics ever exported by the Exporter +func (e *Exporter) Describe(ch chan<- *prometheus.Desc) { + + for _, met := range e.APIMetrics { + ch <- met + } + +} + +// Collect function, called on by Prometheus Client library +func (e *Exporter) Collect(ch chan<- prometheus.Metric) { + + var data, rates, err = e.gatherData(ch) + + if err != nil { + log.Errorf("Error gathering Data from remote API: %v", err) + return + } + + err = e.processMetrics(data, rates, ch) + + if err != nil { + log.Error("Error Processing Metrics", err) + return + } + + log.Info("All Metrics successfully collected") + +} diff --git a/exporter/gather.go b/exporter/gather.go new file mode 100644 index 00000000..153d4b6f --- /dev/null +++ b/exporter/gather.go @@ -0,0 +1,146 @@ +package exporter + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "net/http" + "strconv" + + log "github.com/Sirupsen/logrus" + + "github.com/prometheus/client_golang/prometheus" +) + +// gatherData - Collects the data from thw API, invokes functions to transform that data into metrics +func (e *Exporter) gatherData(ch chan<- prometheus.Metric) ([]*APIResponse, *RateLimits, error) { + + apid := []*APIResponse{} + + for _, u := range e.TargetURLs { + + // Create new data slice from Struct + var d = new(APIResponse) + + resp, err := e.getHttpResponse(u) + + if err != nil { + log.Errorf("Error requesting http data from API for repository: %s. Got Error: %s", u, err) + return nil, nil, err + } + + e.toJSON(resp, &d) + + if err != nil { + log.Errorf("Error gathering JSON data for repository: %s. Got Error: %s", u, err) + return nil, nil, err + } + + apid = append(apid, d) + + log.Infof("API data fetched for repository: ", u) + } + + rates, err := e.getRates(e.APIURL) + + if err != nil { + return apid, rates, err + } + + return apid, rates, err +} + +// getJSON return json from server, return the formatted JSON +func (e *Exporter) toJSON(resp *http.Response, target interface{}) { + + //respFormatted := json.NewDecoder(resp.Body).Decode(target) + json.NewDecoder(resp.Body).Decode(target) + + // Close the response body, the underlying Transport should then close the connection. + resp.Body.Close() + + return +} + +func (e *Exporter) getAuth() (string, error) { + + if e.APIToken != "" { + return e.APIToken, nil + } else if e.APITokenFile != "" { + b, err := ioutil.ReadFile(e.APITokenFile) + if err != nil { + return "", err + } + return string(b), err + + } + + return "", nil +} + +func (e *Exporter) getRates(baseURL string) (*RateLimits, error) { + + rateEndPoint := ("/rate_limit") + url := fmt.Sprintf("%s%s", baseURL, rateEndPoint) + + resp, err := e.getHttpResponse(url) + + if err != nil { + log.Errorf("Error requesting http data from API for repository: %s. Got Error: %s", url, err) + return &RateLimits{}, err + } + + limit, err := strconv.ParseFloat(resp.Header.Get("X-RateLimit-Limit"), 64) + + if err != nil { + return &RateLimits{}, err + } + + rem, err := strconv.ParseFloat(resp.Header.Get("X-RateLimit-Remaining"), 64) + + if err != nil { + return &RateLimits{}, err + } + + reset, err := strconv.ParseFloat(resp.Header.Get("X-RateLimit-Reset"), 64) + + if err != nil { + return &RateLimits{}, err + } + + return &RateLimits{ + Limit: limit, + Remaining: rem, + Reset: reset, + }, err + +} + +func (e *Exporter) getHttpResponse(url string) (*http.Response, error) { + + client := &http.Client{} + + req, err := http.NewRequest("GET", url, nil) + + if err != nil { + return nil, err + } + + a, err := e.getAuth() + + if err != nil { + return nil, err + } + + if a != "" { + req.Header.Add("Authorization", "token "+a) + } + + resp, err := client.Do(req) + + if err != nil { + return resp, err + } + + return resp, nil +} diff --git a/exporter/metrics.go b/exporter/metrics.go new file mode 100644 index 00000000..04ae53a3 --- /dev/null +++ b/exporter/metrics.go @@ -0,0 +1,74 @@ +package exporter + +import "github.com/prometheus/client_golang/prometheus" + +// AddMetrics - Add's all of the metrics to a map, returns the map. +func AddMetrics() map[string]*prometheus.Desc { + + APIMetrics := make(map[string]*prometheus.Desc) + + APIMetrics["Stars"] = prometheus.NewDesc( + prometheus.BuildFQName("github", "repo", "stars"), + "Total number of Stars for given repository", + []string{"repo", "user"}, nil, + ) + APIMetrics["OpenIssues"] = prometheus.NewDesc( + prometheus.BuildFQName("github", "repo", "open_issues"), + "Total number of open issues for given repository", + []string{"repo", "user"}, nil, + ) + APIMetrics["Watchers"] = prometheus.NewDesc( + prometheus.BuildFQName("github", "repo", "watchers"), + "Total number of watchers/subscribers for given repository", + []string{"repo", "user"}, nil, + ) + APIMetrics["Forks"] = prometheus.NewDesc( + prometheus.BuildFQName("github", "repo", "forks"), + "Total number of forks for given repository", + []string{"repo", "user"}, nil, + ) + APIMetrics["Size"] = prometheus.NewDesc( + prometheus.BuildFQName("github", "repo", "size_kb"), + "Size in KB for given repository", + []string{"repo", "user"}, nil, + ) + APIMetrics["Limit"] = prometheus.NewDesc( + prometheus.BuildFQName("github", "rate", "limit"), + "Number of API queries allowed in a 60 minute window", + []string{}, nil, + ) + APIMetrics["Remaining"] = prometheus.NewDesc( + prometheus.BuildFQName("github", "rate", "remaining"), + "Number of API queries remaining in the current window", + []string{}, nil, + ) + APIMetrics["Reset"] = prometheus.NewDesc( + prometheus.BuildFQName("github", "rate", "reset"), + "The time at which the current rate limit window resets in UTC epoch seconds", + []string{}, nil, + ) + + return APIMetrics +} + +// processMetrics - Collects the data from the API, returns data object +func (e *Exporter) processMetrics(data []*APIResponse, rates *RateLimits, ch chan<- prometheus.Metric) error { + + // APIMetrics - range through the data object + for _, y := range data { + for _, x := range *y { + ch <- prometheus.MustNewConstMetric(e.APIMetrics["Stars"], prometheus.GaugeValue, x.Stars, x.Name, x.Owner.Login) + ch <- prometheus.MustNewConstMetric(e.APIMetrics["Forks"], prometheus.GaugeValue, x.Forks, x.Name, x.Owner.Login) + ch <- prometheus.MustNewConstMetric(e.APIMetrics["OpenIssues"], prometheus.GaugeValue, x.OpenIssues, x.Name, x.Owner.Login) + ch <- prometheus.MustNewConstMetric(e.APIMetrics["Watchers"], prometheus.GaugeValue, x.Watchers, x.Name, x.Owner.Login) + ch <- prometheus.MustNewConstMetric(e.APIMetrics["Size"], prometheus.GaugeValue, x.Size, x.Name, x.Owner.Login) + } + } + + // Rate limit stats + ch <- prometheus.MustNewConstMetric(e.APIMetrics["Limit"], prometheus.GaugeValue, rates.Limit) + ch <- prometheus.MustNewConstMetric(e.APIMetrics["Remaining"], prometheus.GaugeValue, rates.Remaining) + ch <- prometheus.MustNewConstMetric(e.APIMetrics["Reset"], prometheus.GaugeValue, rates.Reset) + + return nil +} diff --git a/exporter/structs.go b/exporter/structs.go new file mode 100644 index 00000000..e580600a --- /dev/null +++ b/exporter/structs.go @@ -0,0 +1,33 @@ +package exporter + +import ( + "github.com/infinityworksltd/github-exporter/config" + "github.com/prometheus/client_golang/prometheus" +) + +// Exporter is used to store Metrics data and embeds the config struct. +// This is done so that the relevant functions have easy access to the +// user defined runtime configuration when the Collect method is called. +type Exporter struct { + APIMetrics map[string]*prometheus.Desc + config.Config +} + +// APIResponse is used to store data from all the relevant endpoints in the API +type APIResponse []struct { + Name string `json:"name"` + Owner struct { + Login string `json:"login"` + } `json:"owner"` + Forks float64 `json:"forks"` + Stars float64 `json:"stargazers_count"` + OpenIssues float64 `json:"open_issues"` + Watchers float64 `json:"subscribers_count"` + Size float64 `json:"size"` +} + +type RateLimits struct { + Limit float64 + Remaining float64 + Reset float64 +} diff --git a/main.go b/main.go new file mode 100644 index 00000000..66b0ccf5 --- /dev/null +++ b/main.go @@ -0,0 +1,53 @@ +package main + +import ( + "net/http" + + "github.com/Sirupsen/logrus" + "github.com/fatih/structs" + conf "github.com/infinityworksltd/github-exporter/config" + "github.com/infinityworksltd/github-exporter/exporter" + "github.com/infinityworksltd/go-common/logger" + "github.com/prometheus/client_golang/prometheus" +) + +var ( + log *logrus.Logger + applicationCfg conf.Config + mets map[string]*prometheus.Desc +) + +func init() { + applicationCfg = conf.Init() + mets = exporter.AddMetrics() + log = logger.Start(&applicationCfg) +} + +func main() { + + log.WithFields(structs.Map(applicationCfg)).Info("Starting Exporter") + + exporter := exporter.Exporter{ + APIMetrics: mets, + Config: applicationCfg, + } + + // Register Metrics from each of the endpoints + // This invokes the Collect method through the prometheus client libraries. + prometheus.MustRegister(&exporter) + + // Setup HTTP handler + http.Handle(applicationCfg.MetricsPath(), prometheus.Handler()) + http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte(` + Github Exporter + +

GitHub Prometheus Metrics Exporter

+

For more information, visit GitHub

+

Metrics

+ + + `)) + }) + log.Fatal(http.ListenAndServe(":"+applicationCfg.ListenPort(), nil)) +} diff --git a/github_exporter.py b/python/github_exporter.py similarity index 99% rename from github_exporter.py rename to python/github_exporter.py index d620812d..c3d6f317 100644 --- a/github_exporter.py +++ b/python/github_exporter.py @@ -75,7 +75,7 @@ def _get_json(self, url: str): gh_token = self._get_github_token() if gh_token: payload = {"access_token": gh_token} - r = requests.get(url,params=payload) + r = requests. (url,params=payload) else: r = requests.get(url) result = json.loads(r.content.decode('UTF-8')) From a33660bedaea56452c5d7ed0e7fa739b66e97a54 Mon Sep 17 00:00:00 2001 From: Edward Marshall Date: Tue, 25 Apr 2017 19:25:17 +0100 Subject: [PATCH 02/13] Added private repo label dimension --- exporter/gather.go | 10 +++++----- exporter/metrics.go | 21 +++++++++++---------- exporter/structs.go | 1 + 3 files changed, 17 insertions(+), 15 deletions(-) diff --git a/exporter/gather.go b/exporter/gather.go index 153d4b6f..dbcd96df 100644 --- a/exporter/gather.go +++ b/exporter/gather.go @@ -38,7 +38,10 @@ func (e *Exporter) gatherData(ch chan<- prometheus.Metric) ([]*APIResponse, *Rat apid = append(apid, d) - log.Infof("API data fetched for repository: ", u) + // Close the response body, the underlying Transport should then close the connection. + resp.Body.Close() + + log.Infof("API data fetched for repository: %s", u) } rates, err := e.getRates(e.APIURL) @@ -50,15 +53,12 @@ func (e *Exporter) gatherData(ch chan<- prometheus.Metric) ([]*APIResponse, *Rat return apid, rates, err } -// getJSON return json from server, return the formatted JSON +// toJSON decodes the response to formatted JSON func (e *Exporter) toJSON(resp *http.Response, target interface{}) { //respFormatted := json.NewDecoder(resp.Body).Decode(target) json.NewDecoder(resp.Body).Decode(target) - // Close the response body, the underlying Transport should then close the connection. - resp.Body.Close() - return } diff --git a/exporter/metrics.go b/exporter/metrics.go index 04ae53a3..d417a6f0 100644 --- a/exporter/metrics.go +++ b/exporter/metrics.go @@ -1,6 +1,7 @@ package exporter import "github.com/prometheus/client_golang/prometheus" +import "strconv" // AddMetrics - Add's all of the metrics to a map, returns the map. func AddMetrics() map[string]*prometheus.Desc { @@ -10,27 +11,27 @@ func AddMetrics() map[string]*prometheus.Desc { APIMetrics["Stars"] = prometheus.NewDesc( prometheus.BuildFQName("github", "repo", "stars"), "Total number of Stars for given repository", - []string{"repo", "user"}, nil, + []string{"repo", "user", "private"}, nil, ) APIMetrics["OpenIssues"] = prometheus.NewDesc( prometheus.BuildFQName("github", "repo", "open_issues"), "Total number of open issues for given repository", - []string{"repo", "user"}, nil, + []string{"repo", "user", "private"}, nil, ) APIMetrics["Watchers"] = prometheus.NewDesc( prometheus.BuildFQName("github", "repo", "watchers"), "Total number of watchers/subscribers for given repository", - []string{"repo", "user"}, nil, + []string{"repo", "user", "private"}, nil, ) APIMetrics["Forks"] = prometheus.NewDesc( prometheus.BuildFQName("github", "repo", "forks"), "Total number of forks for given repository", - []string{"repo", "user"}, nil, + []string{"repo", "user", "private"}, nil, ) APIMetrics["Size"] = prometheus.NewDesc( prometheus.BuildFQName("github", "repo", "size_kb"), "Size in KB for given repository", - []string{"repo", "user"}, nil, + []string{"repo", "user", "private"}, nil, ) APIMetrics["Limit"] = prometheus.NewDesc( prometheus.BuildFQName("github", "rate", "limit"), @@ -57,11 +58,11 @@ func (e *Exporter) processMetrics(data []*APIResponse, rates *RateLimits, ch cha // APIMetrics - range through the data object for _, y := range data { for _, x := range *y { - ch <- prometheus.MustNewConstMetric(e.APIMetrics["Stars"], prometheus.GaugeValue, x.Stars, x.Name, x.Owner.Login) - ch <- prometheus.MustNewConstMetric(e.APIMetrics["Forks"], prometheus.GaugeValue, x.Forks, x.Name, x.Owner.Login) - ch <- prometheus.MustNewConstMetric(e.APIMetrics["OpenIssues"], prometheus.GaugeValue, x.OpenIssues, x.Name, x.Owner.Login) - ch <- prometheus.MustNewConstMetric(e.APIMetrics["Watchers"], prometheus.GaugeValue, x.Watchers, x.Name, x.Owner.Login) - ch <- prometheus.MustNewConstMetric(e.APIMetrics["Size"], prometheus.GaugeValue, x.Size, x.Name, x.Owner.Login) + ch <- prometheus.MustNewConstMetric(e.APIMetrics["Stars"], prometheus.GaugeValue, x.Stars, x.Name, x.Owner.Login, strconv.FormatBool(x.Private)) + ch <- prometheus.MustNewConstMetric(e.APIMetrics["Forks"], prometheus.GaugeValue, x.Forks, x.Name, x.Owner.Login, strconv.FormatBool(x.Private)) + ch <- prometheus.MustNewConstMetric(e.APIMetrics["OpenIssues"], prometheus.GaugeValue, x.OpenIssues, x.Name, x.Owner.Login, strconv.FormatBool(x.Private)) + ch <- prometheus.MustNewConstMetric(e.APIMetrics["Watchers"], prometheus.GaugeValue, x.Watchers, x.Name, x.Owner.Login, strconv.FormatBool(x.Private)) + ch <- prometheus.MustNewConstMetric(e.APIMetrics["Size"], prometheus.GaugeValue, x.Size, x.Name, x.Owner.Login, strconv.FormatBool(x.Private)) } } diff --git a/exporter/structs.go b/exporter/structs.go index e580600a..2cc7d70f 100644 --- a/exporter/structs.go +++ b/exporter/structs.go @@ -19,6 +19,7 @@ type APIResponse []struct { Owner struct { Login string `json:"login"` } `json:"owner"` + Private bool `json:"private"` Forks float64 `json:"forks"` Stars float64 `json:"stargazers_count"` OpenIssues float64 `json:"open_issues"` From 17c64ecb9073e734aa3101ab07d60a25e7fb381d Mon Sep 17 00:00:00 2001 From: Edward Marshall Date: Thu, 27 Apr 2017 14:41:05 +0100 Subject: [PATCH 03/13] Updates following testing --- exporter/gather.go | 50 ++++++++++---- exporter/metrics.go | 15 ++--- exporter/structs.go | 5 +- python/github_exporter.py | 134 -------------------------------------- 4 files changed, 48 insertions(+), 156 deletions(-) delete mode 100644 python/github_exporter.py diff --git a/exporter/gather.go b/exporter/gather.go index dbcd96df..8c6dd85d 100644 --- a/exporter/gather.go +++ b/exporter/gather.go @@ -19,8 +19,9 @@ func (e *Exporter) gatherData(ch chan<- prometheus.Metric) ([]*APIResponse, *Rat for _, u := range e.TargetURLs { - // Create new data slice from Struct + // Create new data slice from Struct for organisation data var d = new(APIResponse) + var da = []*APIResponse{} resp, err := e.getHttpResponse(u) @@ -28,16 +29,31 @@ func (e *Exporter) gatherData(ch chan<- prometheus.Metric) ([]*APIResponse, *Rat log.Errorf("Error requesting http data from API for repository: %s. Got Error: %s", u, err) return nil, nil, err } + body, err := ioutil.ReadAll(resp.Body) + ia := isArray(body) - e.toJSON(resp, &d) + if err != nil { + log.Errorf("Failed to determine API response") + return nil, nil, err + } + + if ia { + log.Info("ARRAY!") + json.Unmarshal(body, &da) + apid = append(apid, da...) + + } else if !ia { + log.Info("NOT ARRAY!") + apid = append(apid, d) + json.Unmarshal(body, &d) + + } if err != nil { log.Errorf("Error gathering JSON data for repository: %s. Got Error: %s", u, err) return nil, nil, err } - apid = append(apid, d) - // Close the response body, the underlying Transport should then close the connection. resp.Body.Close() @@ -53,15 +69,7 @@ func (e *Exporter) gatherData(ch chan<- prometheus.Metric) ([]*APIResponse, *Rat return apid, rates, err } -// toJSON decodes the response to formatted JSON -func (e *Exporter) toJSON(resp *http.Response, target interface{}) { - - //respFormatted := json.NewDecoder(resp.Body).Decode(target) - json.NewDecoder(resp.Body).Decode(target) - - return -} - +// getAuth returns oauth2 token as string for usage in http.request func (e *Exporter) getAuth() (string, error) { if e.APIToken != "" { @@ -144,3 +152,19 @@ func (e *Exporter) getHttpResponse(url string) (*http.Response, error) { return resp, nil } + +func isArray(body []byte) bool { + + isArray := false + + for _, c := range body { + if c == ' ' || c == '\t' || c == '\r' || c == '\n' { + continue + } + isArray = c == '[' + break + } + + return isArray + +} diff --git a/exporter/metrics.go b/exporter/metrics.go index d417a6f0..b9e8a1c6 100644 --- a/exporter/metrics.go +++ b/exporter/metrics.go @@ -56,14 +56,13 @@ func AddMetrics() map[string]*prometheus.Desc { func (e *Exporter) processMetrics(data []*APIResponse, rates *RateLimits, ch chan<- prometheus.Metric) error { // APIMetrics - range through the data object - for _, y := range data { - for _, x := range *y { - ch <- prometheus.MustNewConstMetric(e.APIMetrics["Stars"], prometheus.GaugeValue, x.Stars, x.Name, x.Owner.Login, strconv.FormatBool(x.Private)) - ch <- prometheus.MustNewConstMetric(e.APIMetrics["Forks"], prometheus.GaugeValue, x.Forks, x.Name, x.Owner.Login, strconv.FormatBool(x.Private)) - ch <- prometheus.MustNewConstMetric(e.APIMetrics["OpenIssues"], prometheus.GaugeValue, x.OpenIssues, x.Name, x.Owner.Login, strconv.FormatBool(x.Private)) - ch <- prometheus.MustNewConstMetric(e.APIMetrics["Watchers"], prometheus.GaugeValue, x.Watchers, x.Name, x.Owner.Login, strconv.FormatBool(x.Private)) - ch <- prometheus.MustNewConstMetric(e.APIMetrics["Size"], prometheus.GaugeValue, x.Size, x.Name, x.Owner.Login, strconv.FormatBool(x.Private)) - } + for _, x := range data { + ch <- prometheus.MustNewConstMetric(e.APIMetrics["Stars"], prometheus.GaugeValue, x.Stars, x.Name, x.Owner.Login, strconv.FormatBool(x.Private)) + ch <- prometheus.MustNewConstMetric(e.APIMetrics["Forks"], prometheus.GaugeValue, x.Forks, x.Name, x.Owner.Login, strconv.FormatBool(x.Private)) + ch <- prometheus.MustNewConstMetric(e.APIMetrics["OpenIssues"], prometheus.GaugeValue, x.OpenIssues, x.Name, x.Owner.Login, strconv.FormatBool(x.Private)) + ch <- prometheus.MustNewConstMetric(e.APIMetrics["Watchers"], prometheus.GaugeValue, x.Watchers, x.Name, x.Owner.Login, strconv.FormatBool(x.Private)) + ch <- prometheus.MustNewConstMetric(e.APIMetrics["Size"], prometheus.GaugeValue, x.Size, x.Name, x.Owner.Login, strconv.FormatBool(x.Private)) + } // Rate limit stats diff --git a/exporter/structs.go b/exporter/structs.go index 2cc7d70f..4a04c382 100644 --- a/exporter/structs.go +++ b/exporter/structs.go @@ -14,7 +14,10 @@ type Exporter struct { } // APIResponse is used to store data from all the relevant endpoints in the API -type APIResponse []struct { +type APIResponseArray []APIResponse + +// APIResponse is used to store data from all the relevant endpoints in the API +type APIResponse struct { Name string `json:"name"` Owner struct { Login string `json:"login"` diff --git a/python/github_exporter.py b/python/github_exporter.py deleted file mode 100644 index c3d6f317..00000000 --- a/python/github_exporter.py +++ /dev/null @@ -1,134 +0,0 @@ -from prometheus_client import start_http_server -from prometheus_client.core import CounterMetricFamily, GaugeMetricFamily, REGISTRY - -import json, requests, sys, time, os, ast, signal - -class GitHubCollector(object): - - def collect(self): - - metrics = {'forks': 'forks', - 'stars': 'stargazers_count', - 'open_issues': 'open_issues', - 'watchers': 'subscribers_count', # watchers_count is actually the same as stargazer_count, need subscribers_count - 'has_issues': 'has_issues',} - - METRIC_PREFIX = 'github_repo' - LABELS = ['repo', 'user', 'private'] - gauges = {} - - # Setup metric counters from prometheus_client.core - for metric in metrics: - gauges[metric] = GaugeMetricFamily('%s_%s' % (METRIC_PREFIX, metric), '%s' % metric, value=None, labels=LABELS) - - # Check the API rate limit - self._check_api_limit() - - # loop through specified repositories and organizations and collect metrics - if os.getenv('REPOS'): - repos = os.getenv('REPOS').replace(' ','').split(",") - self._repo_urls = [] - for repo in repos: - print(repo + " added to collection array") - self._repo_urls.extend('https://api.github.com/repos/{0}'.format(repo).split(",")) - self._collect_repo_metrics(gauges, metrics) - print("Metrics collected for individually specified repositories") - - if os.getenv('ORGS'): - orgs = os.getenv('ORGS').replace(' ','').split(",") - self._org_urls = [] - for org in orgs: - print(org + " added to collection array") - self._org_urls.extend('https://api.github.com/orgs/{0}/repos'.format(org).split(",")) - self._collect_org_metrics(gauges, metrics) - print("Metrics collected for repositories listed under specified organization") - - # Yield all metrics returned - for metric in metrics: - yield gauges[metric] - - def _collect_repo_metrics(self, gauges, metrics): - for url in self._repo_urls: - response_json = self._get_json(url) - self._add_metrics(gauges, metrics, response_json) - - def _collect_org_metrics(self, gauges, metrics): - for url in self._org_urls: - response_json = self._get_json(url) - for repo in response_json: - self._add_metrics(gauges, metrics, repo) - - def _get_github_token(self): - if os.getenv('GITHUB_TOKEN'): - return os.getenv('GITHUB_TOKEN') - elif os.getenv('GITHUB_TOKEN_FILE'): - return open(os.getenv('GITHUB_TOKEN_FILE'), 'r').read().rstrip() - else: - return None - - def _get_json(self, url: str): - """ - using github core api - rate limit 5000 per hours - """ - print("Getting JSON Payload for " + url) - gh_token = self._get_github_token() - if gh_token: - payload = {"access_token": gh_token} - r = requests. (url,params=payload) - else: - r = requests.get(url) - result = json.loads(r.content.decode('UTF-8')) - if result is None: - raise ValueError("Github API is broken, try again") - return self._pagination(r, result) - - def _pagination(self, response: requests.Response, result): - if "Link" not in response.headers: - return result - links = dict() - for i in response.headers["Link"].split(","): - url, rel = i.split(";") - rel = rel[6:-1] - url = url[1:-1] - links[rel] = url - if "next" in links: - assert type(result) is list - return result + self._get_json(links["next"]) - else: - return result - - def _check_api_limit(self): - rate_limit_url = "https://api.github.com/rate_limit" - gh_token = self._get_github_token() - if gh_token: - print("Authentication token detected: " + gh_token) - payload = {"access_token": gh_token} - R = requests.get(rate_limit_url,params=payload) - else: - R = requests.get(rate_limit_url) - - limit_js = ast.literal_eval(R.text) - remaining = limit_js["rate"]["remaining"] - print("Requests remaing this hour", remaining) - if not remaining: - print("Rate limit exceeded, try enabling authentication") - - def _add_metrics(self, gauges, metrics, response_json): - for metric, field in metrics.items(): - gauges[metric].add_metric([response_json['name'], response_json['owner']['login'], str(response_json['private']).lower()], value=response_json[field]) - -def sigterm_handler(_signo, _stack_frame): - sys.exit(0) - -if __name__ == '__main__': - # Ensure we have something to export - print("Starting Exporter") - if not (os.getenv('REPOS') or os.getenv('ORGS')): - print("No repositories or organizations specified, exiting") - exit(1) - start_http_server(int(os.getenv('BIND_PORT'))) - REGISTRY.register(GitHubCollector()) - - signal.signal(signal.SIGTERM, sigterm_handler) - while True: time.sleep(1) From 29abf2c674ce53b443831c4e594f8d0136a8cb7a Mon Sep 17 00:00:00 2001 From: Thomas Gray Date: Fri, 28 Apr 2017 08:22:46 +0100 Subject: [PATCH 04/13] Cleaning up app --- config/config.go | 1 + exporter/gather.go | 43 +++++++++++++++---------------------------- 2 files changed, 16 insertions(+), 28 deletions(-) diff --git a/config/config.go b/config/config.go index d950d43a..ba0f7c19 100644 --- a/config/config.go +++ b/config/config.go @@ -31,6 +31,7 @@ func Init() Config { if err != nil { log.Errorf("Error initialising Configuration, Error: %v", err) + panic("Failed to find any valid configuration. Please refer to README.md") } appConfig := Config{ diff --git a/exporter/gather.go b/exporter/gather.go index 8c6dd85d..69e99757 100644 --- a/exporter/gather.go +++ b/exporter/gather.go @@ -15,58 +15,45 @@ import ( // gatherData - Collects the data from thw API, invokes functions to transform that data into metrics func (e *Exporter) gatherData(ch chan<- prometheus.Metric) ([]*APIResponse, *RateLimits, error) { - apid := []*APIResponse{} + APId := []*APIResponse{} for _, u := range e.TargetURLs { - - // Create new data slice from Struct for organisation data - var d = new(APIResponse) - var da = []*APIResponse{} - resp, err := e.getHttpResponse(u) if err != nil { log.Errorf("Error requesting http data from API for repository: %s. Got Error: %s", u, err) return nil, nil, err } + + // Read the body into a string so we can parse it twice (isArray & Unmarshal) body, err := ioutil.ReadAll(resp.Body) - ia := isArray(body) + resp.Body.Close() if err != nil { - log.Errorf("Failed to determine API response") + log.Errorf("Failed to read response body, error: %s", err) return nil, nil, err } - if ia { - log.Info("ARRAY!") - json.Unmarshal(body, &da) - apid = append(apid, da...) - - } else if !ia { - log.Info("NOT ARRAY!") - apid = append(apid, d) - json.Unmarshal(body, &d) - - } - - if err != nil { - log.Errorf("Error gathering JSON data for repository: %s. Got Error: %s", u, err) - return nil, nil, err + if isArray(body) { + dataSlice := []*APIResponse{} + json.Unmarshal(body, &dataSlice) + APId = append(APId, dataSlice...) + } else { + data := new(APIResponse) + json.Unmarshal(body, &data) + APId = append(APId, data) } - // Close the response body, the underlying Transport should then close the connection. - resp.Body.Close() - log.Infof("API data fetched for repository: %s", u) } rates, err := e.getRates(e.APIURL) if err != nil { - return apid, rates, err + return APId, rates, err } - return apid, rates, err + return APId, rates, err } // getAuth returns oauth2 token as string for usage in http.request From d8edb421a195a17f9f7946595bb46c1d9a4d7ca8 Mon Sep 17 00:00:00 2001 From: Edward Marshall Date: Fri, 28 Apr 2017 08:46:04 +0100 Subject: [PATCH 05/13] Removed panic --- config/config.go | 1 - 1 file changed, 1 deletion(-) diff --git a/config/config.go b/config/config.go index ba0f7c19..d950d43a 100644 --- a/config/config.go +++ b/config/config.go @@ -31,7 +31,6 @@ func Init() Config { if err != nil { log.Errorf("Error initialising Configuration, Error: %v", err) - panic("Failed to find any valid configuration. Please refer to README.md") } appConfig := Config{ From 2543f621990eec1d404553284350303340651ac2 Mon Sep 17 00:00:00 2001 From: Edward Marshall Date: Tue, 2 May 2017 08:39:15 +0100 Subject: [PATCH 06/13] Fix error in Dockerfile --- Dockerfile | 4 ++-- config/config.go | 6 ++++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index bba1f22b..dd117ca5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,7 +4,7 @@ LABEL maintainer "Infinity Works" EXPOSE 9173 ENV GOPATH=/go - LISTEN_PORT=9173 +ENV LISTEN_PORT=9173 RUN addgroup exporter \ && adduser -S -G exporter exporter \ @@ -21,4 +21,4 @@ RUN go get \ USER exporter -CMD [ "/bin/main" ] \ No newline at end of file +CMD [ "/bin/main" ] diff --git a/config/config.go b/config/config.go index d950d43a..1e5a2d0b 100644 --- a/config/config.go +++ b/config/config.go @@ -33,6 +33,12 @@ func Init() Config { log.Errorf("Error initialising Configuration, Error: %v", err) } + // I believe I can fly, + // I believe I can touch the sky, + // I think about it every night and day, + // Think about it every night and day, + // Spread my wings and fly awayyyy + appConfig := Config{ &ac, cfg.GetEnv("API_URL", "https://api.github.com"), From 42743225251099e482553bb3457f0f9f697db1b0 Mon Sep 17 00:00:00 2001 From: Edward Marshall Date: Tue, 2 May 2017 08:49:55 +0100 Subject: [PATCH 07/13] Port correction --- Dockerfile | 4 ++-- config/config.go | 6 ------ 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/Dockerfile b/Dockerfile index dd117ca5..49883069 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,10 +1,10 @@ FROM golang:1.8.0-alpine LABEL maintainer "Infinity Works" -EXPOSE 9173 +EXPOSE 9171 ENV GOPATH=/go -ENV LISTEN_PORT=9173 +ENV LISTEN_PORT=9171 RUN addgroup exporter \ && adduser -S -G exporter exporter \ diff --git a/config/config.go b/config/config.go index 1e5a2d0b..d950d43a 100644 --- a/config/config.go +++ b/config/config.go @@ -33,12 +33,6 @@ func Init() Config { log.Errorf("Error initialising Configuration, Error: %v", err) } - // I believe I can fly, - // I believe I can touch the sky, - // I think about it every night and day, - // Think about it every night and day, - // Spread my wings and fly awayyyy - appConfig := Config{ &ac, cfg.GetEnv("API_URL", "https://api.github.com"), From 4d7f525e0e39dca77dafb511715fa78843c03350 Mon Sep 17 00:00:00 2001 From: Edward Marshall Date: Tue, 2 May 2017 11:47:21 +0100 Subject: [PATCH 08/13] Updated code comments --- config/config.go | 7 +++++-- exporter/collect.go | 9 ++++++--- exporter/gather.go | 29 ++++++++++++++++++++--------- exporter/metrics.go | 8 ++++---- exporter/structs.go | 5 ++++- 5 files changed, 39 insertions(+), 19 deletions(-) diff --git a/config/config.go b/config/config.go index d950d43a..439a2193 100644 --- a/config/config.go +++ b/config/config.go @@ -11,7 +11,7 @@ import ( cfg "github.com/infinityworksltd/go-common/config" ) -// Config struct holds all of the confgiguration +// Config struct holds all of the runtime confgiguration for the application type Config struct { *cfg.BaseConfig APIURL string @@ -47,19 +47,21 @@ func Init() Config { } // Init populates the Config struct based on environmental runtime configuration +// All URL's are added to the TargetURL's string array func getScrapeURLs() ([]string, error) { urls := []string{} apiURL := cfg.GetEnv("API_URL", "https://api.github.com") repos := os.Getenv("REPOS") orgs := os.Getenv("ORGS") - opts := "?&per_page=100" + opts := "?&per_page=100" // Used to set the Github API to return 100 results per page (max) // User input validation, check that either repositories or organisations have been passed in if len(repos) == 0 && len(orgs) == 0 { return urls, fmt.Errorf("No organisations or repositories specified") } + // Append repositories to the array if repos != "" { rs := strings.Split(repos, ", ") for _, x := range rs { @@ -68,6 +70,7 @@ func getScrapeURLs() ([]string, error) { } } + // Append github orginisations to the array if orgs != "" { o := strings.Split(orgs, ", ") for _, x := range o { diff --git a/exporter/collect.go b/exporter/collect.go index bf2161b5..893932b3 100644 --- a/exporter/collect.go +++ b/exporter/collect.go @@ -5,18 +5,20 @@ import ( "github.com/prometheus/client_golang/prometheus" ) -// Describe describes all the metrics ever exported by the Exporter +// Describe - loops through the API metrics and passes them to prometheus.Describe func (e *Exporter) Describe(ch chan<- *prometheus.Desc) { - for _, met := range e.APIMetrics { - ch <- met + for _, m := range e.APIMetrics { + ch <- m } } // Collect function, called on by Prometheus Client library +// This function is called when a scrape is peformed on the /metrics page func (e *Exporter) Collect(ch chan<- prometheus.Metric) { + // Scrape the Data from Github var data, rates, err = e.gatherData(ch) if err != nil { @@ -24,6 +26,7 @@ func (e *Exporter) Collect(ch chan<- prometheus.Metric) { return } + // Set prometheus gauge metrics using the data gathered err = e.processMetrics(data, rates, ch) if err != nil { diff --git a/exporter/gather.go b/exporter/gather.go index 69e99757..bfb7eaa7 100644 --- a/exporter/gather.go +++ b/exporter/gather.go @@ -12,13 +12,14 @@ import ( "github.com/prometheus/client_golang/prometheus" ) -// gatherData - Collects the data from thw API, invokes functions to transform that data into metrics +// gatherData - Collects the data from the API and stores into struct func (e *Exporter) gatherData(ch chan<- prometheus.Metric) ([]*APIResponse, *RateLimits, error) { - APId := []*APIResponse{} + aResponses := []*APIResponse{} + // Scrapes are peformed per URL and data is appended to a slice for _, u := range e.TargetURLs { - resp, err := e.getHttpResponse(u) + resp, err := e.getHTTPResponse(u) if err != nil { log.Errorf("Error requesting http data from API for repository: %s. Got Error: %s", u, err) @@ -34,26 +35,29 @@ func (e *Exporter) gatherData(ch chan<- prometheus.Metric) ([]*APIResponse, *Rat return nil, nil, err } + // Github can at times present an array, or an object for the same data set. + // This code checks handles this variation. if isArray(body) { dataSlice := []*APIResponse{} json.Unmarshal(body, &dataSlice) - APId = append(APId, dataSlice...) + aResponses = append(aResponses, dataSlice...) } else { data := new(APIResponse) json.Unmarshal(body, &data) - APId = append(APId, data) + aResponses = append(aResponses, data) } log.Infof("API data fetched for repository: %s", u) } + // Check the API rate data and store as a metric rates, err := e.getRates(e.APIURL) if err != nil { - return APId, rates, err + return aResponses, rates, err } - return APId, rates, err + return aResponses, rates, err } // getAuth returns oauth2 token as string for usage in http.request @@ -73,12 +77,14 @@ func (e *Exporter) getAuth() (string, error) { return "", nil } +// getRates obtains the rate limit data for requests against the github API. +// Especially useful when operating without oauth and the subsequent lower cap. func (e *Exporter) getRates(baseURL string) (*RateLimits, error) { rateEndPoint := ("/rate_limit") url := fmt.Sprintf("%s%s", baseURL, rateEndPoint) - resp, err := e.getHttpResponse(url) + resp, err := e.getHTTPResponse(url) if err != nil { log.Errorf("Error requesting http data from API for repository: %s. Got Error: %s", url, err) @@ -111,22 +117,26 @@ func (e *Exporter) getRates(baseURL string) (*RateLimits, error) { } -func (e *Exporter) getHttpResponse(url string) (*http.Response, error) { +// getHTTPResponse creates a http client, issues a GET and returns the http.Response +func (e *Exporter) getHTTPResponse(url string) (*http.Response, error) { client := &http.Client{} + // (expensive but robus at these volumes) req, err := http.NewRequest("GET", url, nil) if err != nil { return nil, err } + // Obtain auth token from file or environment a, err := e.getAuth() if err != nil { return nil, err } + // If a token is present, add it to the http.request if a != "" { req.Header.Add("Authorization", "token "+a) } @@ -140,6 +150,7 @@ func (e *Exporter) getHttpResponse(url string) (*http.Response, error) { return resp, nil } +// isArray simply looks for key details that determine if the JSON response is an array or not. func isArray(body []byte) bool { isArray := false diff --git a/exporter/metrics.go b/exporter/metrics.go index b9e8a1c6..2c51f0ce 100644 --- a/exporter/metrics.go +++ b/exporter/metrics.go @@ -3,7 +3,7 @@ package exporter import "github.com/prometheus/client_golang/prometheus" import "strconv" -// AddMetrics - Add's all of the metrics to a map, returns the map. +// AddMetrics - Add's all of the metrics to a map of strings, returns the map. func AddMetrics() map[string]*prometheus.Desc { APIMetrics := make(map[string]*prometheus.Desc) @@ -52,10 +52,10 @@ func AddMetrics() map[string]*prometheus.Desc { return APIMetrics } -// processMetrics - Collects the data from the API, returns data object +// processMetrics - processes the response data and sets the metrics using it as a source func (e *Exporter) processMetrics(data []*APIResponse, rates *RateLimits, ch chan<- prometheus.Metric) error { - // APIMetrics - range through the data object + // APIMetrics - range through the data slice for _, x := range data { ch <- prometheus.MustNewConstMetric(e.APIMetrics["Stars"], prometheus.GaugeValue, x.Stars, x.Name, x.Owner.Login, strconv.FormatBool(x.Private)) ch <- prometheus.MustNewConstMetric(e.APIMetrics["Forks"], prometheus.GaugeValue, x.Forks, x.Name, x.Owner.Login, strconv.FormatBool(x.Private)) @@ -65,7 +65,7 @@ func (e *Exporter) processMetrics(data []*APIResponse, rates *RateLimits, ch cha } - // Rate limit stats + // Set Rate limit stats ch <- prometheus.MustNewConstMetric(e.APIMetrics["Limit"], prometheus.GaugeValue, rates.Limit) ch <- prometheus.MustNewConstMetric(e.APIMetrics["Remaining"], prometheus.GaugeValue, rates.Remaining) ch <- prometheus.MustNewConstMetric(e.APIMetrics["Reset"], prometheus.GaugeValue, rates.Reset) diff --git a/exporter/structs.go b/exporter/structs.go index 4a04c382..75979cc8 100644 --- a/exporter/structs.go +++ b/exporter/structs.go @@ -13,7 +13,8 @@ type Exporter struct { config.Config } -// APIResponse is used to store data from all the relevant endpoints in the API +// APIResponseArray is used to store an array of APIResponses. +// This is useful for the JSON array detection type APIResponseArray []APIResponse // APIResponse is used to store data from all the relevant endpoints in the API @@ -30,6 +31,8 @@ type APIResponse struct { Size float64 `json:"size"` } +// RateLimits is used to store rate limit data into a struct +// This data is later represented as a metric, captured at the end of a scrape type RateLimits struct { Limit float64 Remaining float64 From be8037c29bf0b1cbf688584368430104a1811170 Mon Sep 17 00:00:00 2001 From: Edward Marshall Date: Tue, 2 May 2017 12:30:21 +0100 Subject: [PATCH 09/13] Added 404 detection --- exporter/collect.go | 39 --------------------------------------- exporter/gather.go | 6 ++++++ 2 files changed, 6 insertions(+), 39 deletions(-) delete mode 100644 exporter/collect.go diff --git a/exporter/collect.go b/exporter/collect.go deleted file mode 100644 index 893932b3..00000000 --- a/exporter/collect.go +++ /dev/null @@ -1,39 +0,0 @@ -package exporter - -import ( - log "github.com/Sirupsen/logrus" - "github.com/prometheus/client_golang/prometheus" -) - -// Describe - loops through the API metrics and passes them to prometheus.Describe -func (e *Exporter) Describe(ch chan<- *prometheus.Desc) { - - for _, m := range e.APIMetrics { - ch <- m - } - -} - -// Collect function, called on by Prometheus Client library -// This function is called when a scrape is peformed on the /metrics page -func (e *Exporter) Collect(ch chan<- prometheus.Metric) { - - // Scrape the Data from Github - var data, rates, err = e.gatherData(ch) - - if err != nil { - log.Errorf("Error gathering Data from remote API: %v", err) - return - } - - // Set prometheus gauge metrics using the data gathered - err = e.processMetrics(data, rates, ch) - - if err != nil { - log.Error("Error Processing Metrics", err) - return - } - - log.Info("All Metrics successfully collected") - -} diff --git a/exporter/gather.go b/exporter/gather.go index bfb7eaa7..b69c9270 100644 --- a/exporter/gather.go +++ b/exporter/gather.go @@ -19,6 +19,7 @@ func (e *Exporter) gatherData(ch chan<- prometheus.Metric) ([]*APIResponse, *Rat // Scrapes are peformed per URL and data is appended to a slice for _, u := range e.TargetURLs { + resp, err := e.getHTTPResponse(u) if err != nil { @@ -147,6 +148,11 @@ func (e *Exporter) getHTTPResponse(url string) (*http.Response, error) { return resp, err } + // Triggers if a user specifies an invalid or not visible repository + if resp.StatusCode == 404 { + return resp, fmt.Errorf("404 Recieved from GitHub API from URL %s", url) + } + return resp, nil } From 181dad4bfd29873f7e1190c08f0802bbd2fe1c34 Mon Sep 17 00:00:00 2001 From: Edward Marshall Date: Tue, 2 May 2017 12:30:43 +0100 Subject: [PATCH 10/13] Renamed file to make it clearer --- exporter/prometheus.go | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 exporter/prometheus.go diff --git a/exporter/prometheus.go b/exporter/prometheus.go new file mode 100644 index 00000000..893932b3 --- /dev/null +++ b/exporter/prometheus.go @@ -0,0 +1,39 @@ +package exporter + +import ( + log "github.com/Sirupsen/logrus" + "github.com/prometheus/client_golang/prometheus" +) + +// Describe - loops through the API metrics and passes them to prometheus.Describe +func (e *Exporter) Describe(ch chan<- *prometheus.Desc) { + + for _, m := range e.APIMetrics { + ch <- m + } + +} + +// Collect function, called on by Prometheus Client library +// This function is called when a scrape is peformed on the /metrics page +func (e *Exporter) Collect(ch chan<- prometheus.Metric) { + + // Scrape the Data from Github + var data, rates, err = e.gatherData(ch) + + if err != nil { + log.Errorf("Error gathering Data from remote API: %v", err) + return + } + + // Set prometheus gauge metrics using the data gathered + err = e.processMetrics(data, rates, ch) + + if err != nil { + log.Error("Error Processing Metrics", err) + return + } + + log.Info("All Metrics successfully collected") + +} From 26e9b4d7ff8feb82a40ccf864236bb6b1abc7f86 Mon Sep 17 00:00:00 2001 From: Edward Marshall Date: Wed, 3 May 2017 19:00:52 +0100 Subject: [PATCH 11/13] Various updates following testing, async request support added --- exporter/gather.go | 103 +++++++++++------------------------------ exporter/http.go | 101 ++++++++++++++++++++++++++++++++++++++++ exporter/metrics.go | 2 +- exporter/prometheus.go | 2 +- exporter/structs.go | 18 +++++-- 5 files changed, 144 insertions(+), 82 deletions(-) create mode 100644 exporter/http.go diff --git a/exporter/gather.go b/exporter/gather.go index b69c9270..94b58292 100644 --- a/exporter/gather.go +++ b/exporter/gather.go @@ -4,70 +4,57 @@ import ( "encoding/json" "fmt" "io/ioutil" - "net/http" "strconv" log "github.com/Sirupsen/logrus" - - "github.com/prometheus/client_golang/prometheus" ) // gatherData - Collects the data from the API and stores into struct -func (e *Exporter) gatherData(ch chan<- prometheus.Metric) ([]*APIResponse, *RateLimits, error) { - - aResponses := []*APIResponse{} - - // Scrapes are peformed per URL and data is appended to a slice - for _, u := range e.TargetURLs { - - resp, err := e.getHTTPResponse(u) +func (e *Exporter) gatherData() ([]*Datum, *RateLimits, error) { - if err != nil { - log.Errorf("Error requesting http data from API for repository: %s. Got Error: %s", u, err) - return nil, nil, err - } + data := []*Datum{} + responses, err := e.asyncHTTPGets() - // Read the body into a string so we can parse it twice (isArray & Unmarshal) - body, err := ioutil.ReadAll(resp.Body) - resp.Body.Close() + if err != nil { + return data, nil, err + } - if err != nil { - log.Errorf("Failed to read response body, error: %s", err) - return nil, nil, err - } + for _, response := range responses { // Github can at times present an array, or an object for the same data set. // This code checks handles this variation. - if isArray(body) { - dataSlice := []*APIResponse{} - json.Unmarshal(body, &dataSlice) - aResponses = append(aResponses, dataSlice...) + if isArray(response.body) { + ds := []*Datum{} + json.Unmarshal(response.body, &ds) + data = append(data, ds...) } else { - data := new(APIResponse) - json.Unmarshal(body, &data) - aResponses = append(aResponses, data) + d := new(Datum) + json.Unmarshal(response.body, &data) + data = append(data, d) } - log.Infof("API data fetched for repository: %s", u) + log.Infof("API data fetched for repository: %s", response.url) } // Check the API rate data and store as a metric rates, err := e.getRates(e.APIURL) if err != nil { - return aResponses, rates, err + return data, rates, err } - return aResponses, rates, err + //return data, rates, err + return data, rates, nil + } // getAuth returns oauth2 token as string for usage in http.request -func (e *Exporter) getAuth() (string, error) { +func getAuth(token string, tokenFile string) (string, error) { - if e.APIToken != "" { - return e.APIToken, nil - } else if e.APITokenFile != "" { - b, err := ioutil.ReadFile(e.APITokenFile) + if token != "" { + return token, nil + } else if tokenFile != "" { + b, err := ioutil.ReadFile(tokenFile) if err != nil { return "", err } @@ -85,7 +72,9 @@ func (e *Exporter) getRates(baseURL string) (*RateLimits, error) { rateEndPoint := ("/rate_limit") url := fmt.Sprintf("%s%s", baseURL, rateEndPoint) - resp, err := e.getHTTPResponse(url) + resp, err := getHTTPResponse(url, e.APIToken, e.APITokenFile) + + defer resp.Body.Close() if err != nil { log.Errorf("Error requesting http data from API for repository: %s. Got Error: %s", url, err) @@ -118,44 +107,6 @@ func (e *Exporter) getRates(baseURL string) (*RateLimits, error) { } -// getHTTPResponse creates a http client, issues a GET and returns the http.Response -func (e *Exporter) getHTTPResponse(url string) (*http.Response, error) { - - client := &http.Client{} - - // (expensive but robus at these volumes) - req, err := http.NewRequest("GET", url, nil) - - if err != nil { - return nil, err - } - - // Obtain auth token from file or environment - a, err := e.getAuth() - - if err != nil { - return nil, err - } - - // If a token is present, add it to the http.request - if a != "" { - req.Header.Add("Authorization", "token "+a) - } - - resp, err := client.Do(req) - - if err != nil { - return resp, err - } - - // Triggers if a user specifies an invalid or not visible repository - if resp.StatusCode == 404 { - return resp, fmt.Errorf("404 Recieved from GitHub API from URL %s", url) - } - - return resp, nil -} - // isArray simply looks for key details that determine if the JSON response is an array or not. func isArray(body []byte) bool { diff --git a/exporter/http.go b/exporter/http.go new file mode 100644 index 00000000..c43028e8 --- /dev/null +++ b/exporter/http.go @@ -0,0 +1,101 @@ +package exporter + +import ( + "fmt" + "io/ioutil" + "net/http" + "time" + + log "github.com/Sirupsen/logrus" +) + +func (e *Exporter) asyncHTTPGets() ([]*Response, error) { + + // Channels used to enable concurrent requests + ch := make(chan *Response, len(e.TargetURLs)) + + responses := []*Response{} + + for _, url := range e.TargetURLs { + + go func(url string) { + err := e.getResponse(url, ch) + if err != nil { + ch <- &Response{url, nil, []byte{}, err} + } + }(url) + + } + + for { + select { + case r := <-ch: + if r.err != nil { + log.Errorf("Error scraping API, Error: %v", r.err) + break + } + responses = append(responses, r) + + if len(responses) == len(e.TargetURLs) { + return responses, nil + } + } + + } +} + +// getResponse collects an individual http.response and returns a *Response +func (e *Exporter) getResponse(url string, ch chan<- *Response) error { + + log.Infof("Fetching %s \n", url) + + resp, err := getHTTPResponse(url, e.APIToken, e.APITokenFile) + + if err != nil { + return fmt.Errorf("Error converting body to byte array: %v", err) + } + + // Read the body to a byte array so it can be used elsewhere + body, err := ioutil.ReadAll(resp.Body) + + defer resp.Body.Close() + + if err != nil { + return fmt.Errorf("Error converting body to byte array: %v", err) + } + + // Triggers if a user specifies an invalid or not visible repository + if resp.StatusCode == 404 { + return fmt.Errorf("Error: Recieved 404 status from Github API, ensure the repsository URL is correct. If it's a privare repository, also check the oauth token is correct") + } + + ch <- &Response{url, resp, body, err} + + return nil +} + +// getHTTPResponse handles the http client creation, token setting and returns the *http.response +func getHTTPResponse(url string, token string, tokenFile string) (*http.Response, error) { + + client := &http.Client{ + Timeout: time.Second * 10, + } + + req, err := http.NewRequest("GET", url, nil) + + if err != nil { + return nil, err + } + + // Obtain auth token from file or environment + a, err := getAuth(token, tokenFile) + + // If a token is present, add it to the http.request + if a != "" { + req.Header.Add("Authorization", "token "+a) + } + + resp, err := client.Do(req) + + return resp, err +} diff --git a/exporter/metrics.go b/exporter/metrics.go index 2c51f0ce..d0eabd0a 100644 --- a/exporter/metrics.go +++ b/exporter/metrics.go @@ -53,7 +53,7 @@ func AddMetrics() map[string]*prometheus.Desc { } // processMetrics - processes the response data and sets the metrics using it as a source -func (e *Exporter) processMetrics(data []*APIResponse, rates *RateLimits, ch chan<- prometheus.Metric) error { +func (e *Exporter) processMetrics(data []*Datum, rates *RateLimits, ch chan<- prometheus.Metric) error { // APIMetrics - range through the data slice for _, x := range data { diff --git a/exporter/prometheus.go b/exporter/prometheus.go index 893932b3..dac238e3 100644 --- a/exporter/prometheus.go +++ b/exporter/prometheus.go @@ -19,7 +19,7 @@ func (e *Exporter) Describe(ch chan<- *prometheus.Desc) { func (e *Exporter) Collect(ch chan<- prometheus.Metric) { // Scrape the Data from Github - var data, rates, err = e.gatherData(ch) + var data, rates, err = e.gatherData() if err != nil { log.Errorf("Error gathering Data from remote API: %v", err) diff --git a/exporter/structs.go b/exporter/structs.go index 75979cc8..780d9df3 100644 --- a/exporter/structs.go +++ b/exporter/structs.go @@ -1,6 +1,8 @@ package exporter import ( + "net/http" + "github.com/infinityworksltd/github-exporter/config" "github.com/prometheus/client_golang/prometheus" ) @@ -13,12 +15,12 @@ type Exporter struct { config.Config } -// APIResponseArray is used to store an array of APIResponses. +// Data is used to store an array of Datums. // This is useful for the JSON array detection -type APIResponseArray []APIResponse +type Data []Datum -// APIResponse is used to store data from all the relevant endpoints in the API -type APIResponse struct { +// Datum is used to store data from all the relevant endpoints in the API +type Datum struct { Name string `json:"name"` Owner struct { Login string `json:"login"` @@ -38,3 +40,11 @@ type RateLimits struct { Remaining float64 Reset float64 } + +// Response struct is used to store http.Response and associated data +type Response struct { + url string + response *http.Response + body []byte + err error +} From fc3a8276fd90ba814a42e517ca8bff980af4cef5 Mon Sep 17 00:00:00 2001 From: Edward Marshall Date: Wed, 3 May 2017 20:55:44 +0100 Subject: [PATCH 12/13] Fixed bug with single repositories --- exporter/gather.go | 14 +++++++++----- exporter/http.go | 27 ++++++++++++++------------- 2 files changed, 23 insertions(+), 18 deletions(-) diff --git a/exporter/gather.go b/exporter/gather.go index 94b58292..1f613b73 100644 --- a/exporter/gather.go +++ b/exporter/gather.go @@ -13,7 +13,11 @@ import ( func (e *Exporter) gatherData() ([]*Datum, *RateLimits, error) { data := []*Datum{} - responses, err := e.asyncHTTPGets() + + // Obtain auth token from file or environment + token, err := getAuth(e.APIToken, e.APITokenFile) + + responses, err := asyncHTTPGets(e.TargetURLs, token) if err != nil { return data, nil, err @@ -29,7 +33,7 @@ func (e *Exporter) gatherData() ([]*Datum, *RateLimits, error) { data = append(data, ds...) } else { d := new(Datum) - json.Unmarshal(response.body, &data) + json.Unmarshal(response.body, &d) data = append(data, d) } @@ -37,7 +41,7 @@ func (e *Exporter) gatherData() ([]*Datum, *RateLimits, error) { } // Check the API rate data and store as a metric - rates, err := e.getRates(e.APIURL) + rates, err := getRates(e.APIURL, token) if err != nil { return data, rates, err @@ -67,12 +71,12 @@ func getAuth(token string, tokenFile string) (string, error) { // getRates obtains the rate limit data for requests against the github API. // Especially useful when operating without oauth and the subsequent lower cap. -func (e *Exporter) getRates(baseURL string) (*RateLimits, error) { +func getRates(baseURL string, token string) (*RateLimits, error) { rateEndPoint := ("/rate_limit") url := fmt.Sprintf("%s%s", baseURL, rateEndPoint) - resp, err := getHTTPResponse(url, e.APIToken, e.APITokenFile) + resp, err := getHTTPResponse(url, token) defer resp.Body.Close() diff --git a/exporter/http.go b/exporter/http.go index c43028e8..67c78e52 100644 --- a/exporter/http.go +++ b/exporter/http.go @@ -9,17 +9,17 @@ import ( log "github.com/Sirupsen/logrus" ) -func (e *Exporter) asyncHTTPGets() ([]*Response, error) { +func asyncHTTPGets(targets []string, token string) ([]*Response, error) { // Channels used to enable concurrent requests - ch := make(chan *Response, len(e.TargetURLs)) + ch := make(chan *Response, len(targets)) responses := []*Response{} - for _, url := range e.TargetURLs { + for _, url := range targets { go func(url string) { - err := e.getResponse(url, ch) + err := getResponse(url, token, ch) if err != nil { ch <- &Response{url, nil, []byte{}, err} } @@ -36,7 +36,7 @@ func (e *Exporter) asyncHTTPGets() ([]*Response, error) { } responses = append(responses, r) - if len(responses) == len(e.TargetURLs) { + if len(responses) == len(targets) { return responses, nil } } @@ -45,11 +45,11 @@ func (e *Exporter) asyncHTTPGets() ([]*Response, error) { } // getResponse collects an individual http.response and returns a *Response -func (e *Exporter) getResponse(url string, ch chan<- *Response) error { +func getResponse(url string, token string, ch chan<- *Response) error { log.Infof("Fetching %s \n", url) - resp, err := getHTTPResponse(url, e.APIToken, e.APITokenFile) + resp, err := getHTTPResponse(url, token) // do this earlier if err != nil { return fmt.Errorf("Error converting body to byte array: %v", err) @@ -75,7 +75,7 @@ func (e *Exporter) getResponse(url string, ch chan<- *Response) error { } // getHTTPResponse handles the http client creation, token setting and returns the *http.response -func getHTTPResponse(url string, token string, tokenFile string) (*http.Response, error) { +func getHTTPResponse(url string, token string) (*http.Response, error) { client := &http.Client{ Timeout: time.Second * 10, @@ -87,15 +87,16 @@ func getHTTPResponse(url string, token string, tokenFile string) (*http.Response return nil, err } - // Obtain auth token from file or environment - a, err := getAuth(token, tokenFile) - // If a token is present, add it to the http.request - if a != "" { - req.Header.Add("Authorization", "token "+a) + if token != "" { + req.Header.Add("Authorization", "token "+token) } resp, err := client.Do(req) + if err != nil { + return nil, err + } + return resp, err } From cb4145ef14b1ba144d0b4862a13a9a366ea93ade Mon Sep 17 00:00:00 2001 From: Edward Marshall Date: Wed, 3 May 2017 21:08:25 +0100 Subject: [PATCH 13/13] Tidied up config --- config/config.go | 47 ++++++++++++++++++++++++++++++++++------------ exporter/gather.go | 25 ++---------------------- 2 files changed, 37 insertions(+), 35 deletions(-) diff --git a/config/config.go b/config/config.go index 439a2193..0843102f 100644 --- a/config/config.go +++ b/config/config.go @@ -2,6 +2,7 @@ package config import ( "fmt" + "io/ioutil" "strings" log "github.com/Sirupsen/logrus" @@ -17,8 +18,9 @@ type Config struct { APIURL string Repositories string Organisations string - APIToken string + APITokenEnv string APITokenFile string + APIToken string TargetURLs []string } @@ -26,8 +28,13 @@ type Config struct { func Init() Config { ac := cfg.Init() - - scraped, err := getScrapeURLs() + url := cfg.GetEnv("API_URL", "https://api.github.com") + repos := os.Getenv("REPOS") + orgs := os.Getenv("ORGS") + tokenEnv := os.Getenv("GITHUB_TOKEN") + tokenFile := os.Getenv("GITHUB_TOKEN_FILE") + token, err := getAuth(tokenEnv, tokenFile) + scraped, err := getScrapeURLs(url, repos, orgs) if err != nil { log.Errorf("Error initialising Configuration, Error: %v", err) @@ -35,11 +42,12 @@ func Init() Config { appConfig := Config{ &ac, - cfg.GetEnv("API_URL", "https://api.github.com"), - os.Getenv("REPOS"), - os.Getenv("ORGS"), - os.Getenv("GITHUB_TOKEN"), - os.Getenv("GITHUB_TOKEN_FILE"), + url, + repos, + orgs, + tokenEnv, + tokenFile, + token, scraped, } @@ -48,12 +56,10 @@ func Init() Config { // Init populates the Config struct based on environmental runtime configuration // All URL's are added to the TargetURL's string array -func getScrapeURLs() ([]string, error) { +func getScrapeURLs(apiURL string, repos string, orgs string) ([]string, error) { urls := []string{} - apiURL := cfg.GetEnv("API_URL", "https://api.github.com") - repos := os.Getenv("REPOS") - orgs := os.Getenv("ORGS") + opts := "?&per_page=100" // Used to set the Github API to return 100 results per page (max) // User input validation, check that either repositories or organisations have been passed in @@ -81,3 +87,20 @@ func getScrapeURLs() ([]string, error) { return urls, nil } + +// getAuth returns oauth2 token as string for usage in http.request +func getAuth(token string, tokenFile string) (string, error) { + + if token != "" { + return token, nil + } else if tokenFile != "" { + b, err := ioutil.ReadFile(tokenFile) + if err != nil { + return "", err + } + return string(b), err + + } + + return "", nil +} diff --git a/exporter/gather.go b/exporter/gather.go index 1f613b73..a119bd01 100644 --- a/exporter/gather.go +++ b/exporter/gather.go @@ -3,7 +3,6 @@ package exporter import ( "encoding/json" "fmt" - "io/ioutil" "strconv" log "github.com/Sirupsen/logrus" @@ -14,10 +13,7 @@ func (e *Exporter) gatherData() ([]*Datum, *RateLimits, error) { data := []*Datum{} - // Obtain auth token from file or environment - token, err := getAuth(e.APIToken, e.APITokenFile) - - responses, err := asyncHTTPGets(e.TargetURLs, token) + responses, err := asyncHTTPGets(e.TargetURLs, e.APIToken) if err != nil { return data, nil, err @@ -41,7 +37,7 @@ func (e *Exporter) gatherData() ([]*Datum, *RateLimits, error) { } // Check the API rate data and store as a metric - rates, err := getRates(e.APIURL, token) + rates, err := getRates(e.APIURL, e.APIToken) if err != nil { return data, rates, err @@ -52,23 +48,6 @@ func (e *Exporter) gatherData() ([]*Datum, *RateLimits, error) { } -// getAuth returns oauth2 token as string for usage in http.request -func getAuth(token string, tokenFile string) (string, error) { - - if token != "" { - return token, nil - } else if tokenFile != "" { - b, err := ioutil.ReadFile(tokenFile) - if err != nil { - return "", err - } - return string(b), err - - } - - return "", nil -} - // getRates obtains the rate limit data for requests against the github API. // Especially useful when operating without oauth and the subsequent lower cap. func getRates(baseURL string, token string) (*RateLimits, error) {