githubexporter · dgeorgievski · Feb 25, 2021
diff --git a/.gitignore b/.gitignore
@@ -1 +1,6 @@
-.sequence
+.sequence
+.vscode/
+build/
+/github-exporter
+
+.DS_Store
diff --git a/Makefile b/Makefile
@@ -1,10 +1,22 @@
-.PHONY: install test build
+BINARY_NAME := github-exporter
+DOCKER_REPO ?= infinityworks/github-exporter
+ARCH ?= darwin
+GOBUILD_VERSION_ARGS := ""
+TAG := $(shell cat VERSION)
+
+
+.PHONY: install test build docker
+
+all: build
 
 install:
 	@go mod download
 
 test:
 	@go test -v -race ./...
 
-build:
-	@go build ./...
+build: *.go
+	@go build -v -o build/bin/$(ARCH)/$(BINARY_NAME) $(GOBUILD_VERSION_ARGS)
+
+docker:
+	docker build -t ${DOCKER_REPO}:$(TAG) .
diff --git a/README.md b/README.md
@@ -10,6 +10,8 @@ This exporter is setup to take input from environment variables. All variables a
 
 * `ORGS` If supplied, the exporter will enumerate all repositories for that organization. Expected in the format "org1, org2".
 * `REPOS` If supplied, The repos you wish to monitor, expected in the format "user/repo1, user/repo2". Can be across different Github users/orgs.
+* `PR_QUERY_OPTIONS` if supplied, added as [query options](https://docs.github.com/en/rest/reference/pulls#list-pull-requests) to list pulls request.
+* `PR_LONG_RUNNING_TIME_DIFF` Time threshold in hours beyond which PRs are considered long running. Default is 2 weeks ie 336 hours.
 * `USERS` If supplied, the exporter will enumerate all repositories for that users. Expected in
 the format "user1, user2".
 * `GITHUB_TOKEN` If supplied, enables the user to supply a github authentication token that allows the API to be queried more often. Optional, but recommended.
@@ -50,6 +52,8 @@ github-exporter:
 
 ```
 
+## Kubernetes deployment
+Follow the instructions provided in [README.md](helm/github-exporter/README.md)
 ## Metrics
 
 Metrics will be made available on port 9171 by default

diff --git a/config/config.go b/config/config.go
@@ -3,6 +3,7 @@ package config
 import (
 	"fmt"
 	"io/ioutil"
+	"strconv"
 	"strings"
 
 	log "github.com/sirupsen/logrus"
@@ -12,17 +13,24 @@ import (
 	cfg "github.com/infinityworks/go-common/config"
 )
 
+var (
+	prQueryOptionDefault string = "per_page=100&sort=long-running&direction=descending"
+)
+
 // Config struct holds all of the runtime confgiguration for the application
 type Config struct {
 	*cfg.BaseConfig
-	APIURL        string
-	Repositories  string
-	Organisations string
-	Users         string
-	APITokenEnv   string
-	APITokenFile  string
-	APIToken      string
-	TargetURLs    []string
+	APIURL                string
+	Repositories          string
+	Organisations         string
+	Users                 string
+	APITokenEnv           string
+	APITokenFile          string
+	APIToken              string
+	TargetURLs            []string
+	PRLongRunningTimeDiff float64 //  Now - PR created time in hours; Above this PR considered to be long running
+	PRQueryOptions        string  // API query options for /pulls
+
 }
 
 // Init populates the Config struct based on environmental runtime configuration
@@ -39,6 +47,9 @@ func Init() Config {
 	tokenFile := os.Getenv("GITHUB_TOKEN_FILE")
 	token, err := getAuth(tokenEnv, tokenFile)
 	scraped, err := getScrapeURLs(url, repos, orgs, users)
+	// default for long running PRs is 2 weeks
+	prLongRunningTimeDiff, err := strconv.ParseFloat(cfg.GetEnv("PR_LONG_RUNNING_TIME_DIFF", "336"), 64)
+	prQueryOptions := cfg.GetEnv("PR_QUERY_OPTIONS", prQueryOptionDefault)
 
 	if err != nil {
 		log.Errorf("Error initialising Configuration, Error: %v", err)
@@ -54,6 +65,8 @@ func Init() Config {
 		tokenFile,
 		token,
 		scraped,
+		prLongRunningTimeDiff,
+		prQueryOptions,
 	}
 
 	return appConfig

diff --git a/exporter/gather.go b/exporter/gather.go
@@ -111,14 +111,27 @@ func getReleases(e *Exporter, url string, data *[]Release) {
 func getPRs(e *Exporter, url string, data *[]Pull) {
 	i := strings.Index(url, "?")
 	baseURL := url[:i]
-	pullsURL := baseURL + "/pulls"
-	pullsResponse, err := asyncHTTPGets([]string{pullsURL}, e.APIToken)
+	pullsURL := strings.Builder{}
+	pullsURL.WriteString(baseURL)
+	pullsURL.WriteString("/pulls")
+	if len(strings.TrimSpace(e.PRQueryOptions)) > 0 {
+		if !strings.HasPrefix(e.PRQueryOptions, "?") {
+			pullsURL.WriteString("?")
+		}
+		pullsURL.WriteString(e.PRQueryOptions)
+	}
+
+	pullsResponse, err := asyncHTTPGets([]string{pullsURL.String()}, e.APIToken)
 
 	if err != nil {
 		log.Errorf("Unable to obtain pull requests from API, Error: %s", err)
 	}
 
-	json.Unmarshal(pullsResponse[0].body, &data)
+	for _, resp := range pullsResponse {
+		var dtPull []Pull
+		json.Unmarshal(resp.body, &dtPull)
+		*data = append(*data, dtPull...)
+	}
 }
 
 // isArray simply looks for key details that determine if the JSON response is an array or not.

diff --git a/exporter/metrics.go b/exporter/metrics.go
@@ -1,7 +1,11 @@
 package exporter
 
-import "github.com/prometheus/client_golang/prometheus"
-import "strconv"
+import (
+	"strconv"
+	"time"
+
+	"github.com/prometheus/client_golang/prometheus"
+)
 
 // AddMetrics - Add's all of the metrics to a map of strings, returns the map.
 func AddMetrics() map[string]*prometheus.Desc {
@@ -18,11 +22,19 @@ func AddMetrics() map[string]*prometheus.Desc {
 		"Total number of open issues for given repository",
 		[]string{"repo", "user", "private", "fork", "archived", "license", "language"}, nil,
 	)
+
 	APIMetrics["PullRequestCount"] = prometheus.NewDesc(
 		prometheus.BuildFQName("github", "repo", "pull_request_count"),
 		"Total number of pull requests for given repository",
-		[]string{"repo"}, nil,
+		[]string{"repo", "user", "language"}, nil,
+	)
+
+	APIMetrics["PullRequestLongRunningCount"] = prometheus.NewDesc(
+		prometheus.BuildFQName("github", "repo", "pull_request_long_running_count"),
+		"Total number of long running pull requests for given repository",
+		[]string{"repo", "user", "language"}, nil,
 	)
+
 	APIMetrics["Watchers"] = prometheus.NewDesc(
 		prometheus.BuildFQName("github", "repo", "watchers"),
 		"Total number of watchers/subscribers for given repository",
@@ -65,6 +77,9 @@ func AddMetrics() map[string]*prometheus.Desc {
 // processMetrics - processes the response data and sets the metrics using it as a source
 func (e *Exporter) processMetrics(data []*Datum, rates *RateLimits, ch chan<- prometheus.Metric) error {
 
+	currentTime := time.Now()
+	longRunningPRsTimeDiff := e.Config.PRLongRunningTimeDiff
+
 	// APIMetrics - range through the data slice
 	for _, x := range data {
 		ch <- prometheus.MustNewConstMetric(e.APIMetrics["Stars"], prometheus.GaugeValue, x.Stars, x.Name, x.Owner.Login, strconv.FormatBool(x.Private), strconv.FormatBool(x.Fork), strconv.FormatBool(x.Archived), x.License.Key, x.Language)
@@ -77,15 +92,44 @@ func (e *Exporter) processMetrics(data []*Datum, rates *RateLimits, ch chan<- pr
 				ch <- prometheus.MustNewConstMetric(e.APIMetrics["ReleaseDownloads"], prometheus.GaugeValue, float64(asset.Downloads), x.Name, x.Owner.Login, release.Name, asset.Name, asset.CreatedAt)
 			}
 		}
+
 		prCount := 0
-		for range x.Pulls {
-			prCount += 1
+		prLongRunning := 0
+		for _, pull := range x.Pulls {
+			prCount++
+			if currentTime.Sub(pull.CreaatedAt).Hours() > longRunningPRsTimeDiff {
+				prLongRunning++
+			}
 		}
+		// fmt.Printf("prCount: %d | prLongRunning: %d ", prCount, prLongRunning)
+
 		// issueCount = x.OpenIssue - prCount
-		ch <- prometheus.MustNewConstMetric(e.APIMetrics["OpenIssues"], prometheus.GaugeValue, (x.OpenIssues - float64(prCount)), x.Name, x.Owner.Login, strconv.FormatBool(x.Private), strconv.FormatBool(x.Fork), strconv.FormatBool(x.Archived), x.License.Key, x.Language)
+		ch <- prometheus.MustNewConstMetric(e.APIMetrics["OpenIssues"],
+			prometheus.GaugeValue,
+			x.OpenIssues,
+			x.Name,
+			x.Owner.Login,
+			strconv.FormatBool(x.Private),
+			strconv.FormatBool(x.Fork),
+			strconv.FormatBool(x.Archived),
+			x.License.Key,
+			x.Language)
+
+		// prCount
+		ch <- prometheus.MustNewConstMetric(e.APIMetrics["PullRequestCount"],
+			prometheus.GaugeValue,
+			float64(prCount),
+			x.Name,
+			x.Owner.Login,
+			x.Language)
 
 		// prCount
-		ch <- prometheus.MustNewConstMetric(e.APIMetrics["PullRequestCount"], prometheus.GaugeValue, float64(prCount), x.Name)
+		ch <- prometheus.MustNewConstMetric(e.APIMetrics["PullRequestLongRunningCount"],
+			prometheus.GaugeValue,
+			float64(prLongRunning),
+			x.Name,
+			x.Owner.Login,
+			x.Language)
 	}
 
 	// Set Rate limit stats

diff --git a/exporter/structs.go b/exporter/structs.go
@@ -2,6 +2,7 @@ package exporter
 
 import (
 	"net/http"
+	"time"
 
 	"github.com/infinityworks/github-exporter/config"
 	"github.com/prometheus/client_golang/prometheus"
@@ -46,11 +47,16 @@ type Release struct {
 	Assets []Asset `json:"assets"`
 }
 
+// Pull define PR request stats
 type Pull struct {
 	Url  string `json:"url"`
 	User struct {
 		Login string `json:"login"`
 	} `json:"user"`
+	Number     int       `json:"number"`
+	State      string    `json:"state"`
+	CreaatedAt time.Time `json:"created_at"`
+	UpdateddAt time.Time `json:"updated_at"`
 }
 
 type Asset struct {

diff --git a/helm/github-exporter/.helmignore b/helm/github-exporter/.helmignore
@@ -0,0 +1,23 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*.orig
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/
diff --git a/helm/github-exporter/Chart.yaml b/helm/github-exporter/Chart.yaml
@@ -0,0 +1,13 @@
+apiVersion: v2
+name: github-exporter
+description: GitHub metrics exporter for Prometheus
+
+maintainers: 
+  - name: Dimitar Georgievski
+    email: dgeorgievski@gmail.com
+
+type: application
+
+version: 0.1.0
+
+appVersion: 1.0.2
diff --git a/helm/github-exporter/README.md b/helm/github-exporter/README.md
@@ -0,0 +1,14 @@
+# github-exporter helm chart
+
+# Deployment
+```
+cd github-exporter/helm
+
+helm upgrade \
+github-exporter github-exporter/ \
+--install \
+--namespace=exporters \
+-f helm_vars/nonprod/values.yaml \
+--debug --dry-run
+
+```
diff --git a/helm/github-exporter/templates/NOTES.txt b/helm/github-exporter/templates/NOTES.txt
@@ -0,0 +1,21 @@
+1. Get the application URL by running these commands:
+{{- if .Values.ingress.enabled }}
+{{- range $host := .Values.ingress.hosts }}
+  {{- range .paths }}
+  http{{ if $.Values.ingress.tls }}s{{ end }}://{{ $host.host }}{{ . }}
+  {{- end }}
+{{- end }}
+{{- else if contains "NodePort" .Values.service.type }}
+  export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "github-exporter.fullname" . }})
+  export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}")
+  echo http://$NODE_IP:$NODE_PORT
+{{- else if contains "LoadBalancer" .Values.service.type }}
+     NOTE: It may take a few minutes for the LoadBalancer IP to be available.
+           You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "github-exporter.fullname" . }}'
+  export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "github-exporter.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}")
+  echo http://$SERVICE_IP:{{ .Values.service.port }}
+{{- else if contains "ClusterIP" .Values.service.type }}
+  export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "github-exporter.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}")
+  echo "Visit http://127.0.0.1:8080 to use your application"
+  kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:80
+{{- end }}