githubexporter · Rucknar · May 4, 2017 · Apr 25, 2017 · Apr 25, 2017 · Apr 27, 2017
diff --git a/Dockerfile b/Dockerfile
@@ -1,10 +1,24 @@
-FROM python:3.5-alpine
+FROM golang:1.8.0-alpine
+LABEL maintainer "Infinity Works"
 
-RUN pip install prometheus_client requests
+EXPOSE 9171
 
-ENV BIND_PORT 9171
+ENV GOPATH=/go
+ENV LISTEN_PORT=9171
 
-ADD . /usr/src/app
-WORKDIR /usr/src/app
+RUN addgroup exporter \
+     && adduser -S -G exporter exporter \
+     && apk --update add ca-certificates \
+     && apk --update add --virtual build-deps git
 
-CMD ["python", "github_exporter.py"]
+COPY ./ /go/src/github.com/infinityworksltd/github-exporter
+
+WORKDIR /go/src/github.com/infinityworksltd/github-exporter
+
+RUN go get \
+ && go test ./... \
+ && go build -o /bin/main
+
+USER exporter
+
+CMD [ "/bin/main" ]
diff --git a/METRICS.md b/METRICS.md
@@ -0,0 +1,38 @@
+# Metrics
+
+Below are an example of the metrics as exposed by this exporter. 
+
+```
+# HELP github_repo_open_issues open_issues
+# TYPE github_repo_open_issues gauge
+github_repo_open_issues{repo="docker-hub-exporter",user="infinityworksltd"} 1.0
+github_repo_open_issues{repo="prometheus-rancher-exporter",user="infinityworksltd"} 2.0
+# HELP github_repo_watchers watchers
+# TYPE github_repo_watchers gauge
+github_repo_watchers{repo="docker-hub-exporter",user="infinityworksltd"} 1.0
+github_repo_watchers{repo="prometheus-rancher-exporter",user="infinityworksltd"} 6.0
+# HELP github_repo_stars stars
+# TYPE github_repo_stars gauge
+github_repo_stars{repo="docker-hub-exporter",user="infinityworksltd"} 1.0
+github_repo_stars{repo="prometheus-rancher-exporter",user="infinityworksltd"} 6.0
+# HELP github_repo_forks forks
+# TYPE github_repo_forks gauge
+github_repo_forks{repo="docker-hub-exporter",user="infinityworksltd"} 0.0
+github_repo_forks{repo="prometheus-rancher-exporter",user="infinityworksltd"} 9.0
+# HELP github_repo_size_kb Size in KB for given repository
+# TYPE github_repo_size_kb gauge
+github_repo_size_kb{repo="docker-hub-exporter",user="infinityworksltd"} 44
+github_repo_size_kb{repo="prometheus-rancher-exporter",user="infinityworksltd"} 7242
+# HELP github_rate_limit Number of API queries allowed in a 60 minute window
+# TYPE github_rate_limit gauge
+github_rate_limit 60
+# HELP github_rate_remaining Number of API queries remaining in the current window
+# TYPE github_rate_remaining gauge
+github_rate_remaining 38
+# HELP github_rate_reset The time at which the current rate limit window resets in UTC epoch seconds
+# TYPE github_rate_reset gauge
+github_rate_reset 1.493139756e+09
+# HELP github_size_kb Size in KB for given repository
+# TYPE github_size_kb gauge
+github_size_kb{repo="CRUST",user="infinityworksltd"} 44
+```
diff --git a/README.md b/README.md
@@ -5,11 +5,20 @@ Exposes basic metrics for your repositories from the GitHub API, to a Prometheus
 ## Configuration
 
 This exporter is setup to take input from environment variables:
-* `BIND_PORT` The port you wish to run the container on, defaults to `9171`
+
+### Required
+* `ORGS` If supplied, the exporter will enumerate all repositories for that organization. Expected in the format "org1, org2". Can be ommited if using `REPOS` and not required
+* `REPOS` If supplied, The images you wish to monitor, expected in the format "user/repo1, user/repo2". Can be across different Github users/orgs. Can be ommited if using `ORGS` and not required
+
+
+### Optional
 * `GITHUB_TOKEN` If supplied, enables the user to supply a github authentication token that allows the API to be queried more often. Optional, but recommended.
 * `GITHUB_TOKEN_FILE` If supplied _instead of_ `GITHUB_TOKEN`, enables the user to supply a path to a file containing a github authentication token that allows the API to be queried more often. Optional, but recommended.
-* `ORGS` If supplied, the exporter will enumerate all repositories for that organization.
-* `REPOS` If supplied, The images you wish to monitor, expected in the format "user/repo1, user/repo2". Can be across different Github users/orgs.
+* `API_URL` Github API URL, shouldn't need to change this. Defaults to `https://api.github.com`
+* `LISTEN_PORT` The port you wish to run the container on, the Dockerfile defaults this to `9171`
+* `METRICS_PATH` the metrics URL path you wish to use, defaults to `/metrics`
+* `LOG_LEVEL` The level of logging the exporter will run with, defaults to `debug`
+
 
 ## Install and deploy
 
@@ -38,29 +47,7 @@ github-exporter:
 ## Metrics
 
 Metrics will be made available on port 9171 by default
-
-```
-# HELP github_repo_open_issues open_issues
-# TYPE github_repo_open_issues gauge
-github_repo_open_issues{repo="docker-hub-exporter",user="infinityworksltd"} 1.0
-github_repo_open_issues{repo="prometheus-rancher-exporter",user="infinityworksltd"} 2.0
-# HELP github_repo_watchers watchers
-# TYPE github_repo_watchers gauge
-github_repo_watchers{repo="docker-hub-exporter",user="infinityworksltd"} 1.0
-github_repo_watchers{repo="prometheus-rancher-exporter",user="infinityworksltd"} 6.0
-# HELP github_repo_stars stars
-# TYPE github_repo_stars gauge
-github_repo_stars{repo="docker-hub-exporter",user="infinityworksltd"} 1.0
-github_repo_stars{repo="prometheus-rancher-exporter",user="infinityworksltd"} 6.0
-# HELP github_repo_forks forks
-# TYPE github_repo_forks gauge
-github_repo_forks{repo="docker-hub-exporter",user="infinityworksltd"} 0.0
-github_repo_forks{repo="prometheus-rancher-exporter",user="infinityworksltd"} 9.0
-# HELP github_repo_has_issues has_issues
-# TYPE github_repo_has_issues gauge
-github_repo_has_issues{repo="docker-hub-exporter",user="infinityworksltd"} 1.0
-github_repo_has_issues{repo="prometheus-rancher-exporter",user="infinityworksltd"} 1.0
-```
+An example of these metrics can be found in the `METRICS.md` markdown file in the root of this repository
 
 ## Metadata
 [![](https://images.microbadger.com/badges/image/infinityworks/github-exporter.svg)](http://microbadger.com/images/infinityworks/github-exporter "Get your own image badge on microbadger.com") [![](https://images.microbadger.com/badges/version/infinityworks/github-exporter.svg)](http://microbadger.com/images/infinityworks/github-exporter "Get your own version badge on microbadger.com")
diff --git a/config/config.go b/config/config.go
@@ -0,0 +1,106 @@
+package config
+
+import (
+	"fmt"
+	"io/ioutil"
+	"strings"
+
+	log "github.com/Sirupsen/logrus"
+
+	"os"
+
+	cfg "github.com/infinityworksltd/go-common/config"
+)
+
+// Config struct holds all of the runtime confgiguration for the application
+type Config struct {
+	*cfg.BaseConfig
+	APIURL        string
+	Repositories  string
+	Organisations string
+	APITokenEnv   string
+	APITokenFile  string
+	APIToken      string
+	TargetURLs    []string
+}
+
+// Init populates the Config struct based on environmental runtime configuration
+func Init() Config {
+
+	ac := cfg.Init()
+	url := cfg.GetEnv("API_URL", "https://api.github.com")
+	repos := os.Getenv("REPOS")
+	orgs := os.Getenv("ORGS")
+	tokenEnv := os.Getenv("GITHUB_TOKEN")
+	tokenFile := os.Getenv("GITHUB_TOKEN_FILE")
+	token, err := getAuth(tokenEnv, tokenFile)
+	scraped, err := getScrapeURLs(url, repos, orgs)
+
+	if err != nil {
+		log.Errorf("Error initialising Configuration, Error: %v", err)
+	}
+
+	appConfig := Config{
+		&ac,
+		url,
+		repos,
+		orgs,
+		tokenEnv,
+		tokenFile,
+		token,
+		scraped,
+	}
+
+	return appConfig
+}
+
+// Init populates the Config struct based on environmental runtime configuration
+// All URL's are added to the TargetURL's string array
+func getScrapeURLs(apiURL string, repos string, orgs string) ([]string, error) {
+
+	urls := []string{}
+
+	opts := "?&per_page=100" // Used to set the Github API to return 100 results per page (max)
+
+	// User input validation, check that either repositories or organisations have been passed in
+	if len(repos) == 0 && len(orgs) == 0 {
+		return urls, fmt.Errorf("No organisations or repositories specified")
+	}
+
+	// Append repositories to the array
+	if repos != "" {
+		rs := strings.Split(repos, ", ")
+		for _, x := range rs {
+			y := fmt.Sprintf("%s/repos/%s%s", apiURL, x, opts)
+			urls = append(urls, y)
+		}
+	}
+
+	// Append github orginisations to the array
+	if orgs != "" {
+		o := strings.Split(orgs, ", ")
+		for _, x := range o {
+			y := fmt.Sprintf("%s/orgs/%s/repos%s", apiURL, x, opts)
+			urls = append(urls, y)
+		}
+	}
+
+	return urls, nil
+}
+
+// getAuth returns oauth2 token as string for usage in http.request
+func getAuth(token string, tokenFile string) (string, error) {
+
+	if token != "" {
+		return token, nil
+	} else if tokenFile != "" {
+		b, err := ioutil.ReadFile(tokenFile)
+		if err != nil {
+			return "", err
+		}
+		return string(b), err
+
+	}
+
+	return "", nil
+}
diff --git a/exporter/gather.go b/exporter/gather.go
@@ -0,0 +1,108 @@
+package exporter
+
+import (
+	"encoding/json"
+	"fmt"
+	"strconv"
+
+	log "github.com/Sirupsen/logrus"
+)
+
+// gatherData - Collects the data from the API and stores into struct
+func (e *Exporter) gatherData() ([]*Datum, *RateLimits, error) {
+
+	data := []*Datum{}
+
+	responses, err := asyncHTTPGets(e.TargetURLs, e.APIToken)
+
+	if err != nil {
+		return data, nil, err
+	}
+
+	for _, response := range responses {
+
+		// Github can at times present an array, or an object for the same data set.
+		// This code checks handles this variation.
+		if isArray(response.body) {
+			ds := []*Datum{}
+			json.Unmarshal(response.body, &ds)
+			data = append(data, ds...)
+		} else {
+			d := new(Datum)
+			json.Unmarshal(response.body, &d)
+			data = append(data, d)
+		}
+
+		log.Infof("API data fetched for repository: %s", response.url)
+	}
+
+	// Check the API rate data and store as a metric
+	rates, err := getRates(e.APIURL, e.APIToken)
+
+	if err != nil {
+		return data, rates, err
+	}
+
+	//return data, rates, err
+	return data, rates, nil
+
+}
+
+// getRates obtains the rate limit data for requests against the github API.
+// Especially useful when operating without oauth and the subsequent lower cap.
+func getRates(baseURL string, token string) (*RateLimits, error) {
+
+	rateEndPoint := ("/rate_limit")
+	url := fmt.Sprintf("%s%s", baseURL, rateEndPoint)
+
+	resp, err := getHTTPResponse(url, token)
+
+	defer resp.Body.Close()
+
+	if err != nil {
+		log.Errorf("Error requesting http data from API for repository: %s. Got Error: %s", url, err)
+		return &RateLimits{}, err
+	}
+
+	limit, err := strconv.ParseFloat(resp.Header.Get("X-RateLimit-Limit"), 64)
+
+	if err != nil {
+		return &RateLimits{}, err
+	}
+
+	rem, err := strconv.ParseFloat(resp.Header.Get("X-RateLimit-Remaining"), 64)
+
+	if err != nil {
+		return &RateLimits{}, err
+	}
+
+	reset, err := strconv.ParseFloat(resp.Header.Get("X-RateLimit-Reset"), 64)
+
+	if err != nil {
+		return &RateLimits{}, err
+	}
+
+	return &RateLimits{
+		Limit:     limit,
+		Remaining: rem,
+		Reset:     reset,
+	}, err
+
+}
+
+// isArray simply looks for key details that determine if the JSON response is an array or not.
+func isArray(body []byte) bool {
+
+	isArray := false
+
+	for _, c := range body {
+		if c == ' ' || c == '\t' || c == '\r' || c == '\n' {
+			continue
+		}
+		isArray = c == '['
+		break
+	}
+
+	return isArray
+
+}