From 7af55333d200cd870a8635ee42558ad7c815801e Mon Sep 17 00:00:00 2001 From: Henry McConville Date: Sat, 18 Oct 2025 13:39:20 +0100 Subject: [PATCH 01/14] [v2] Simplify init and config --- config/config.go | 296 +++++++++++------------------------ config/config_test.go | 118 ++++++++++++++ exporter/gather.go | 22 ++- exporter/prometheus.go | 11 +- go.mod | 4 +- go.sum | 4 +- http/server.go | 6 +- main.go | 27 ++-- test/github_exporter_test.go | 10 +- 9 files changed, 255 insertions(+), 243 deletions(-) create mode 100644 config/config_test.go diff --git a/config/config.go b/config/config.go index c81f63df..f55e6ca3 100644 --- a/config/config.go +++ b/config/config.go @@ -2,238 +2,101 @@ package config import ( "context" + "fmt" + "github.com/bradleyfalzon/ghinstallation/v2" + log "github.com/sirupsen/logrus" "net/http" "net/url" "os" "path" - "strconv" "strings" - "github.com/bradleyfalzon/ghinstallation/v2" - cfg "github.com/infinityworks/go-common/config" - log "github.com/sirupsen/logrus" + "github.com/kelseyhightower/envconfig" ) -// Config struct holds all the runtime configuration for the application +// Config struct holds runtime configuration required for the application type Config struct { - *cfg.BaseConfig - apiUrl *url.URL - repositories []string - organisations []string - users []string - apiToken string - targetURLs []string - gitHubApp bool - gitHubAppKeyPath string - gitHubAppId int64 - gitHubAppInstallationId int64 - gitHubRateLimit float64 + MetricsPath string `envconfig:"METRICS_PATH" required:"false" default:"/metrics"` + ListenPort string `envconfig:"LISTEN_PORT" required:"false" default:"9171"` + LogLevel string `envconfig:"LOG_LEVEL" required:"false" default:"INFO"` + AppName string `envconfig:"APP_NAME" required:"false" default:"github-exporter"` + ApiUrl *url.URL `envconfig:"API_URL" required:"false" default:"https://api.github.com"` + Repositories []string `envconfig:"REPOS" required:"false"` + Organisations []string `envconfig:"ORGS" required:"false"` + Users []string `envconfig:"USERS" required:"false"` + GithubToken string `envconfig:"GITHUB_TOKEN" required:"false"` + GithubTokenFile string `envconfig:"GITHUB_TOKEN_FILE" required:"false"` + GitHubApp bool `envconfig:"GITHUB_APP" required:"false" default:"false"` + *GitHubAppConfig `ignored:"true"` +} + +type GitHubAppConfig struct { + GitHubAppKeyPath string `envconfig:"GITHUB_APP_KEY_PATH" required:"false" default:""` + GitHubAppId int64 `envconfig:"GITHUB_APP_ID" required:"false" default:""` + GitHubAppInstallationId int64 `envconfig:"GITHUB_APP_INSTALLATION_ID" required:"false" default:""` + GitHubRateLimit float64 `envconfig:"GITHUB_RATE_LIMIT" required:"false" default:"15000"` } // Init populates the Config struct based on environmental runtime configuration -func Init() Config { - - listenPort := cfg.GetEnv("LISTEN_PORT", "9171") - os.Setenv("LISTEN_PORT", listenPort) - ac := cfg.Init() +func Init() (*Config, error) { - appConfig := Config{ - &ac, - nil, - nil, - nil, - nil, - "", - nil, - false, - "", - 0, - 0, - 15000, + var cfg Config + if err := envconfig.Process("", &cfg); err != nil { + return nil, fmt.Errorf("processing envconfig: %v", err) } - err := appConfig.SetAPIURL(cfg.GetEnv("API_URL", "https://api.github.com")) + // Parse and set log level + level, err := log.ParseLevel(cfg.LogLevel) if err != nil { - log.Errorf("Error initialising Configuration. Unable to parse API URL. Error: %v", err) + return nil, fmt.Errorf("parsing log level: %v", err) } - repos := os.Getenv("REPOS") - if repos != "" { - appConfig.SetRepositories(strings.Split(repos, ", ")) - } - orgs := os.Getenv("ORGS") - if orgs != "" { - appConfig.SetOrganisations(strings.Split(orgs, ", ")) - } - users := os.Getenv("USERS") - if users != "" { - appConfig.SetUsers(strings.Split(users, ", ")) - } - - gitHubApp := strings.ToLower(os.Getenv("GITHUB_APP")) - if gitHubApp == "true" { - gitHubAppKeyPath := os.Getenv("GITHUB_APP_KEY_PATH") - gitHubAppId, _ := strconv.ParseInt(os.Getenv("GITHUB_APP_ID"), 10, 64) - gitHubAppInstallationId, _ := strconv.ParseInt(os.Getenv("GITHUB_APP_INSTALLATION_ID"), 10, 64) - gitHubRateLimit, _ := strconv.ParseFloat(cfg.GetEnv("GITHUB_RATE_LIMIT", "15000"), 64) - appConfig.SetGitHubApp(true) - appConfig.SetGitHubAppKeyPath(gitHubAppKeyPath) - appConfig.SetGitHubAppId(gitHubAppId) - appConfig.SetGitHubAppInstallationId(gitHubAppInstallationId) - appConfig.SetGitHubRateLimit(gitHubRateLimit) - err = appConfig.SetAPITokenFromGitHubApp() + log.SetLevel(level) + + // Trim whitespace from repositories, organisations, and users + cfg.Repositories = mapSlice(cfg.Repositories, strings.TrimSpace) + cfg.Organisations = mapSlice(cfg.Organisations, strings.TrimSpace) + cfg.Users = mapSlice(cfg.Users, strings.TrimSpace) + + // Process GitHub App config if enabled + if cfg.GitHubApp { + var appCfg GitHubAppConfig + if err := envconfig.Process("", &appCfg); err != nil { + return nil, fmt.Errorf("processing GitHub App envconfig: %v", err) + } + cfg.GitHubAppConfig = &appCfg + token, err := cfg.APITokenFromGitHubApp() if err != nil { - log.Errorf("Error initializing Configuration, Error: %v", err) + return nil, fmt.Errorf("generating API token from GitHub App config: %v", err) } + cfg.GithubToken = token } - tokenEnv := os.Getenv("GITHUB_TOKEN") - tokenFile := os.Getenv("GITHUB_TOKEN_FILE") - if tokenEnv != "" { - appConfig.SetAPIToken(tokenEnv) - } else if tokenFile != "" { - err = appConfig.SetAPITokenFromFile(tokenFile) + // Read token from file if not set directly + if cfg.GithubToken == "" && cfg.GithubTokenFile != "" { + tokenBytes, err := os.ReadFile(cfg.GithubTokenFile) if err != nil { - log.Errorf("Error initialising Configuration, Error: %v", err) + return nil, fmt.Errorf("reading GitHub token from file: %v", err) } + cfg.GithubToken = strings.TrimSpace(string(tokenBytes)) } - return appConfig -} -// Returns the base APIURL -func (c *Config) APIURL() *url.URL { - return c.apiUrl + return &cfg, nil } -// Returns a list of all object URLs to scrape func (c *Config) TargetURLs() []string { - return c.targetURLs -} -// Returns the oauth2 token for usage in http.request -func (c *Config) APIToken() string { - return c.apiToken -} - -// Returns the GitHub App authentication value -func (c *Config) GitHubApp() bool { - return c.gitHubApp -} - -// Returns the GitHub app private key path -func (c *Config) GitHubAppKeyPath() string { - return c.gitHubAppKeyPath -} - -// Returns the GitHub app id -func (c *Config) GitHubAppId() int64 { - return c.gitHubAppId -} - -// Returns the GitHub app installation id -func (c *Config) GitHubAppInstallationId() int64 { - return c.gitHubAppInstallationId -} - -// Returns the GitHub RateLimit -func (c *Config) GitHubRateLimit() float64 { - return c.gitHubRateLimit -} - -// Sets the base API URL returning an error if the supplied string is not a valid URL -func (c *Config) SetAPIURL(u string) error { - ur, err := url.Parse(u) - c.apiUrl = ur - return err -} - -// Overrides the entire list of repositories -func (c *Config) SetRepositories(repos []string) { - c.repositories = repos - c.setScrapeURLs() -} - -// Overrides the entire list of organisations -func (c *Config) SetOrganisations(orgs []string) { - c.organisations = orgs - c.setScrapeURLs() -} - -// Overrides the entire list of users -func (c *Config) SetUsers(users []string) { - c.users = users - c.setScrapeURLs() -} - -// SetAPIToken accepts a string oauth2 token for usage in http.request -func (c *Config) SetAPIToken(token string) { - c.apiToken = token -} - -// SetAPITokenFromFile accepts a file containing an oauth2 token for usage in http.request -func (c *Config) SetAPITokenFromFile(tokenFile string) error { - b, err := os.ReadFile(tokenFile) - if err != nil { - return err - } - c.apiToken = strings.TrimSpace(string(b)) - return nil -} - -// SetGitHubApp accepts a boolean for GitHub app authentication -func (c *Config) SetGitHubApp(githubApp bool) { - c.gitHubApp = githubApp -} - -// SetGitHubAppKeyPath accepts a string for GitHub app private key path -func (c *Config) SetGitHubAppKeyPath(gitHubAppKeyPath string) { - c.gitHubAppKeyPath = gitHubAppKeyPath -} - -// SetGitHubAppId accepts a string for GitHub app id -func (c *Config) SetGitHubAppId(gitHubAppId int64) { - c.gitHubAppId = gitHubAppId -} - -// SetGitHubAppInstallationId accepts a string for GitHub app installation id -func (c *Config) SetGitHubAppInstallationId(gitHubAppInstallationId int64) { - c.gitHubAppInstallationId = gitHubAppInstallationId -} - -// SetGitHubAppRateLimit accepts a string for GitHub RateLimit -func (c *Config) SetGitHubRateLimit(gitHubRateLimit float64) { - c.gitHubRateLimit = gitHubRateLimit -} - -// SetAPITokenFromGitHubApp generating api token from github app configuration. -func (c *Config) SetAPITokenFromGitHubApp() error { - itr, err := ghinstallation.NewKeyFromFile(http.DefaultTransport, c.gitHubAppId, c.gitHubAppInstallationId, c.gitHubAppKeyPath) - if err != nil { - return err - } - strToken, err := itr.Token(context.Background()) - if err != nil { - return err - } - c.SetAPIToken(strToken) - return nil -} - -// Init populates the Config struct based on environmental runtime configuration -// All URL's are added to the TargetURL's string array -func (c *Config) setScrapeURLs() error { - - urls := []string{} + var urls []string opts := map[string]string{"per_page": "100"} // Used to set the Github API to return 100 results per page (max) - if len(c.repositories) == 0 && len(c.organisations) == 0 && len(c.users) == 0 { + if len(c.Repositories) == 0 && len(c.Organisations) == 0 && len(c.Users) == 0 { log.Info("No targets specified. Only rate limit endpoint will be scraped") } // Append repositories to the array - if len(c.repositories) > 0 { - for _, x := range c.repositories { - y := *c.apiUrl + if len(c.Repositories) > 0 { + for _, x := range c.Repositories { + y := *c.ApiUrl y.Path = path.Join(y.Path, "repos", x) q := y.Query() for k, v := range opts { @@ -244,11 +107,10 @@ func (c *Config) setScrapeURLs() error { } } - // Append github orginisations to the array - - if len(c.organisations) > 0 { - for _, x := range c.organisations { - y := *c.apiUrl + // Append GitHub organisations to the array + if len(c.Organisations) > 0 { + for _, x := range c.Organisations { + y := *c.ApiUrl y.Path = path.Join(y.Path, "orgs", x, "repos") q := y.Query() for k, v := range opts { @@ -259,9 +121,9 @@ func (c *Config) setScrapeURLs() error { } } - if len(c.users) > 0 { - for _, x := range c.users { - y := *c.apiUrl + if len(c.Users) > 0 { + for _, x := range c.Users { + y := *c.ApiUrl y.Path = path.Join(y.Path, "users", x, "repos") q := y.Query() for k, v := range opts { @@ -272,7 +134,29 @@ func (c *Config) setScrapeURLs() error { } } - c.targetURLs = urls + return urls +} - return nil +// APITokenFromGitHubApp generating api token from github app configuration. +func (c *Config) APITokenFromGitHubApp() (string, error) { + itr, err := ghinstallation.NewKeyFromFile(http.DefaultTransport, c.GitHubAppId, c.GitHubAppInstallationId, c.GitHubAppKeyPath) + if err != nil { + return "", err + } + + strToken, err := itr.Token(context.Background()) + if err != nil { + return "", err + } + + return strToken, nil +} + +// mapSlice applies a function to each element of a slice and returns a new slice with the results. +func mapSlice[T any, M any](input []T, f func(T) M) []M { + result := make([]M, len(input)) + for i, e := range input { + result[i] = f(e) + } + return result } diff --git a/config/config_test.go b/config/config_test.go new file mode 100644 index 00000000..867662ad --- /dev/null +++ b/config/config_test.go @@ -0,0 +1,118 @@ +package config + +import ( + "errors" + "net/url" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestConfig(t *testing.T) { + tests := []struct { + name string + envVars map[string]string + expectedCfg *Config + expectedErr error + }{ + { + name: "default config", + expectedCfg: &Config{ + MetricsPath: "/metrics", + ListenPort: "9171", + LogLevel: "INFO", + AppName: "github-exporter", + ApiUrl: &url.URL{ + Scheme: "https", + Host: "api.github.com", + }, + Repositories: []string{}, + Organisations: []string{}, + Users: []string{}, + GithubToken: "", + GithubTokenFile: "", + GitHubApp: false, + GitHubAppConfig: nil, + }, + expectedErr: nil, + }, + { + name: "non-default config", + envVars: map[string]string{ + "METRICS_PATH": "/otherendpoint", + "LISTEN_PORT": "1111", + "LOG_LEVEL": "DEBUG", + "APP_NAME": "other-app-name", + "API_URL": "https://example.com", + "REPOS": "repo1, repo2", + "ORGS": "org1,org2 ", + "USERS": " user1, user2 ", + "GITHUB_TOKEN": "token", + }, + expectedCfg: &Config{ + MetricsPath: "/otherendpoint", + ListenPort: "1111", + LogLevel: "DEBUG", + AppName: "other-app-name", + ApiUrl: &url.URL{ + Scheme: "https", + Host: "example.com", + }, + Repositories: []string{ + "repo1", + "repo2", + }, + Organisations: []string{ + "org1", + "org2", + }, + Users: []string{ + "user1", + "user2", + }, + GithubToken: "token", + GithubTokenFile: "", + GitHubApp: false, + GitHubAppConfig: nil, + }, + expectedErr: nil, + }, + { + name: "invalid url", + expectedCfg: nil, + envVars: map[string]string{ + "API_URL": "://invalid-url", + }, + expectedErr: errors.New("processing envconfig: envconfig.Process: assigning API_URL to ApiUrl: converting '://invalid-url' to type url.URL. details: parse \"://invalid-url\": missing protocol scheme"), + }, + { + name: "invalid github app id", + expectedCfg: nil, + envVars: map[string]string{ + "GITHUB_APP": "true", + "GITHUB_APP_ID": "not-an-integer", + }, + expectedErr: errors.New("processing GitHub App envconfig: envconfig.Process: assigning GITHUB_APP_ID to GitHubAppId: converting 'not-an-integer' to type int64. details: strconv.ParseInt: parsing \"not-an-integer\": invalid syntax"), + }, + { + name: "github token file not found", + envVars: map[string]string{ + "GITHUB_TOKEN_FILE": "/non/existent/file", + }, + expectedCfg: nil, + expectedErr: errors.New("reading GitHub token from file: open /non/existent/file: no such file or directory"), + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + for k, v := range tt.envVars { + t.Setenv(k, v) + } + + cfg, err := Init() + + assert.Equal(t, tt.expectedErr, err) + assert.Equal(t, tt.expectedCfg, cfg) + }) + } +} diff --git a/exporter/gather.go b/exporter/gather.go index 7a72db2d..4c41a1bf 100644 --- a/exporter/gather.go +++ b/exporter/gather.go @@ -15,7 +15,7 @@ func (e *Exporter) gatherData() ([]*Datum, error) { data := []*Datum{} - responses, err := asyncHTTPGets(e.TargetURLs(), e.APIToken()) + responses, err := asyncHTTPGets(e.TargetURLs(), e.GithubToken) if err != nil { return data, err @@ -55,10 +55,10 @@ func (e *Exporter) gatherData() ([]*Datum, error) { // getRates obtains the rate limit data for requests against the github API. // Especially useful when operating without oauth and the subsequent lower cap. func (e *Exporter) getRates() (*RateLimits, error) { - u := *e.APIURL() + u := *e.ApiUrl u.Path = path.Join(u.Path, "rate_limit") - resp, err := getHTTPResponse(u.String(), e.APIToken()) + resp, err := getHTTPResponse(u.String(), e.GithubToken) if err != nil { return &RateLimits{}, err } @@ -99,26 +99,34 @@ func getReleases(e *Exporter, url string, data *[]Release) { i := strings.Index(url, "?") baseURL := url[:i] releasesURL := baseURL + "/releases" - releasesResponse, err := asyncHTTPGets([]string{releasesURL}, e.APIToken()) + releasesResponse, err := asyncHTTPGets([]string{releasesURL}, e.GithubToken) if err != nil { log.Errorf("Unable to obtain releases from API, Error: %s", err) + return } - json.Unmarshal(releasesResponse[0].body, &data) + err = json.Unmarshal(releasesResponse[0].body, &data) + if err != nil { + log.Errorf("Unable to unmarshal releases from API response, Error: %s", err) + } } func getPRs(e *Exporter, url string, data *[]Pull) { i := strings.Index(url, "?") baseURL := url[:i] pullsURL := baseURL + "/pulls" - pullsResponse, err := asyncHTTPGets([]string{pullsURL}, e.APIToken()) + pullsResponse, err := asyncHTTPGets([]string{pullsURL}, e.GithubToken) if err != nil { log.Errorf("Unable to obtain pull requests from API, Error: %s", err) + return } - json.Unmarshal(pullsResponse[0].body, &data) + err = json.Unmarshal(pullsResponse[0].body, &data) + if err != nil { + log.Errorf("Unable to unmarshal pulls from API response, Error: %s", err) + } } // isArray simply looks for key details that determine if the JSON response is an array or not. diff --git a/exporter/prometheus.go b/exporter/prometheus.go index 4023e1cc..fff5dd4a 100644 --- a/exporter/prometheus.go +++ b/exporter/prometheus.go @@ -23,17 +23,18 @@ func (e *Exporter) Collect(ch chan<- prometheus.Metric) { data := []*Datum{} var err error - if e.Config.GitHubApp() { + if e.Config.GitHubApp { needReAuth, err := e.isTokenExpired() if err != nil { log.Errorf("Error checking token expiration status: %v", err) return } if needReAuth { - err = e.Config.SetAPITokenFromGitHubApp() + token, err := e.Config.APITokenFromGitHubApp() if err != nil { log.Errorf("Error authenticating with GitHub app: %v", err) } + e.GithubToken = token } } // Scrape the Data from Github @@ -64,10 +65,10 @@ func (e *Exporter) Collect(ch chan<- prometheus.Metric) { } func (e *Exporter) isTokenExpired() (bool, error) { - u := *e.APIURL() + u := *e.ApiUrl u.Path = path.Join(u.Path, "rate_limit") - resp, err := getHTTPResponse(u.String(), e.APIToken()) + resp, err := getHTTPResponse(u.String(), e.GithubToken) if err != nil { return false, err @@ -84,7 +85,7 @@ func (e *Exporter) isTokenExpired() (bool, error) { return false, err } - defaultRateLimit := e.Config.GitHubRateLimit() + defaultRateLimit := e.Config.GitHubRateLimit if limit < defaultRateLimit { return true, nil } diff --git a/go.mod b/go.mod index fdd6079e..ea23caff 100644 --- a/go.mod +++ b/go.mod @@ -4,10 +4,11 @@ go 1.22 require ( github.com/bradleyfalzon/ghinstallation/v2 v2.11.0 - github.com/infinityworks/go-common v0.0.0-20170820165359-7f20a140fd37 + github.com/kelseyhightower/envconfig v1.4.0 github.com/prometheus/client_golang v1.20.4 github.com/sirupsen/logrus v1.9.3 github.com/steinfletcher/apitest v1.3.8 + github.com/stretchr/testify v1.9.0 github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 ) @@ -25,7 +26,6 @@ require ( github.com/prometheus/client_model v0.6.1 // indirect github.com/prometheus/common v0.59.1 // indirect github.com/prometheus/procfs v0.15.1 // indirect - github.com/stretchr/testify v1.9.0 // indirect golang.org/x/sys v0.25.0 // indirect google.golang.org/protobuf v1.34.2 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/go.sum b/go.sum index ff4841e9..d0c8c62a 100644 --- a/go.sum +++ b/go.sum @@ -17,8 +17,8 @@ github.com/google/go-github/v62 v62.0.0 h1:/6mGCaRywZz9MuHyw9gD1CwsbmBX8GWsbFkwM github.com/google/go-github/v62 v62.0.0/go.mod h1:EMxeUqGJq2xRu9DYBMwel/mr7kZrzUOfQmmpYrZn2a4= github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= -github.com/infinityworks/go-common v0.0.0-20170820165359-7f20a140fd37 h1:Lm6kyC3JBiJQvJrus66He0E4viqDc/m5BdiFNSkIFfU= -github.com/infinityworks/go-common v0.0.0-20170820165359-7f20a140fd37/go.mod h1:+OaHNKQvQ9oOCr+DgkF95PkiDx20fLHpzMp8SmRPQTg= +github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dvMUtDTo2cv8= +github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg= github.com/klauspost/compress v1.17.10 h1:oXAz+Vh0PMUvJczoi+flxpnBEPxoER1IaAnU/NMPtT0= github.com/klauspost/compress v1.17.10/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= diff --git a/http/server.go b/http/server.go index 274d0f21..4cceddb9 100644 --- a/http/server.go +++ b/http/server.go @@ -21,14 +21,14 @@ func NewServer(exporter exporter.Exporter) *Server { // This invokes the Collect method through the prometheus client libraries. prometheus.MustRegister(&exporter) - r.Handle(exporter.MetricsPath(), promhttp.Handler()) + r.Handle(exporter.MetricsPath, promhttp.Handler()) r.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { _, _ = w.Write([]byte(` Github Exporter

GitHub Prometheus Metrics Exporter

For more information, visit GitHub

-

Metrics

+

Metrics

`)) @@ -38,5 +38,5 @@ func NewServer(exporter exporter.Exporter) *Server { } func (s *Server) Start() { - log.Fatal(http.ListenAndServe(":"+s.exporter.ListenPort(), s.Handler)) + log.Fatal(http.ListenAndServe(":"+s.exporter.ListenPort, s.Handler)) } diff --git a/main.go b/main.go index 35a54677..d4bbe997 100644 --- a/main.go +++ b/main.go @@ -4,29 +4,22 @@ import ( conf "github.com/githubexporter/github-exporter/config" "github.com/githubexporter/github-exporter/exporter" "github.com/githubexporter/github-exporter/http" - "github.com/infinityworks/go-common/logger" - "github.com/prometheus/client_golang/prometheus" - "github.com/sirupsen/logrus" + log "github.com/sirupsen/logrus" ) -var ( - log *logrus.Logger - applicationCfg conf.Config - mets map[string]*prometheus.Desc -) - -func init() { - applicationCfg = conf.Init() - mets = exporter.AddMetrics() - log = logger.Start(&applicationCfg) -} - func main() { log.Info("Starting Exporter") + applicationCfg, err := conf.Init() + if err != nil { + log.Fatalf("Error initializing configuration: %v", err) + } + + metrics := exporter.AddMetrics() + exp := exporter.Exporter{ - APIMetrics: mets, - Config: applicationCfg, + APIMetrics: metrics, + Config: *applicationCfg, } http.NewServer(exp).Start() diff --git a/test/github_exporter_test.go b/test/github_exporter_test.go index 534e4fb9..c8a0ad17 100644 --- a/test/github_exporter_test.go +++ b/test/github_exporter_test.go @@ -63,7 +63,10 @@ func TestGithubExporterHttpErrorHandling(t *testing.T) { // Ideally a new gauge should be added to keep track of scrape errors // following prometheus exporter guidelines test.Mocks( + githubRepos(), + githubReleases(), githubPullsError(), + githubRateLimit(), ). Get("/metrics"). Expect(t). @@ -86,7 +89,11 @@ func apiTest(conf config.Config) (*apitest.APITest, exporter.Exporter) { func withConfig(repos string) config.Config { _ = os.Setenv("REPOS", repos) _ = os.Setenv("GITHUB_TOKEN", "12345") - return config.Init() + cfg, err := config.Init() + if err != nil { + panic(err) + } + return *cfg } func githubRepos() *apitest.Mock { @@ -140,6 +147,7 @@ func githubPullsError() *apitest.Mock { Get("https://api.github.com/repos/myOrg/myRepo/pulls"). Header("Authorization", "token 12345"). RespondWith(). + Times(2). Status(http.StatusBadRequest). End() } From a9b203ad507e341108df52f794ae3b17ce4af096 Mon Sep 17 00:00:00 2001 From: Henry McConville Date: Sat, 18 Oct 2025 15:09:03 +0100 Subject: [PATCH 02/14] [v2] Use github client library instead of direct API requests --- config/config.go | 27 +++- exporter/gather.go | 147 ----------------- exporter/http.go | 160 ------------------ exporter/metrics.go | 10 +- exporter/prometheus.go | 216 +++++++++++++++++++------ exporter/structs.go | 39 +++-- go.mod | 3 +- go.sum | 5 +- http/server.go | 1 + main.go | 7 +- test/github_exporter_test.go | 26 ++- test/testdata/rate_limit_response.json | 46 ++++++ 12 files changed, 288 insertions(+), 399 deletions(-) delete mode 100644 exporter/gather.go delete mode 100644 exporter/http.go create mode 100644 test/testdata/rate_limit_response.json diff --git a/config/config.go b/config/config.go index f55e6ca3..1b9686d0 100644 --- a/config/config.go +++ b/config/config.go @@ -3,15 +3,16 @@ package config import ( "context" "fmt" - "github.com/bradleyfalzon/ghinstallation/v2" - log "github.com/sirupsen/logrus" + "github.com/google/go-github/v76/github" "net/http" "net/url" "os" "path" "strings" + "github.com/bradleyfalzon/ghinstallation/v2" "github.com/kelseyhightower/envconfig" + log "github.com/sirupsen/logrus" ) // Config struct holds runtime configuration required for the application @@ -152,6 +153,28 @@ func (c *Config) APITokenFromGitHubApp() (string, error) { return strToken, nil } +func (c *Config) GetClient() (*github.Client, error) { + var httpClient *http.Client + + // Add custom transport for GitHub App authentication if enabled + if c.GitHubApp { + itr, err := ghinstallation.NewKeyFromFile(http.DefaultTransport, c.GitHubAppId, c.GitHubAppInstallationId, c.GitHubAppKeyPath) + if err != nil { + return nil, fmt.Errorf("creating GitHub App installation transport: %v", err) + } + + httpClient.Transport = itr + } + + client := github.NewClient(httpClient) + + if c.GithubToken != "" { + client = client.WithAuthToken(c.GithubToken) + } + + return client, nil +} + // mapSlice applies a function to each element of a slice and returns a new slice with the results. func mapSlice[T any, M any](input []T, f func(T) M) []M { result := make([]M, len(input)) diff --git a/exporter/gather.go b/exporter/gather.go deleted file mode 100644 index 4c41a1bf..00000000 --- a/exporter/gather.go +++ /dev/null @@ -1,147 +0,0 @@ -package exporter - -import ( - "encoding/json" - "fmt" - "path" - "strconv" - "strings" - - log "github.com/sirupsen/logrus" -) - -// gatherData - Collects the data from the API and stores into struct -func (e *Exporter) gatherData() ([]*Datum, error) { - - data := []*Datum{} - - responses, err := asyncHTTPGets(e.TargetURLs(), e.GithubToken) - - if err != nil { - return data, err - } - - for _, response := range responses { - - // Github can at times present an array, or an object for the same data set. - // This code checks handles this variation. - if isArray(response.body) { - ds := []*Datum{} - json.Unmarshal(response.body, &ds) - data = append(data, ds...) - } else { - d := new(Datum) - - // Get releases - if strings.Contains(response.url, "/repos/") { - getReleases(e, response.url, &d.Releases) - } - // Get PRs - if strings.Contains(response.url, "/repos/") { - getPRs(e, response.url, &d.Pulls) - } - json.Unmarshal(response.body, &d) - data = append(data, d) - } - - log.Infof("API data fetched for repository: %s", response.url) - } - - //return data, rates, err - return data, nil - -} - -// getRates obtains the rate limit data for requests against the github API. -// Especially useful when operating without oauth and the subsequent lower cap. -func (e *Exporter) getRates() (*RateLimits, error) { - u := *e.ApiUrl - u.Path = path.Join(u.Path, "rate_limit") - - resp, err := getHTTPResponse(u.String(), e.GithubToken) - if err != nil { - return &RateLimits{}, err - } - defer resp.Body.Close() - - // Triggers if rate-limiting isn't enabled on private Github Enterprise installations - if resp.StatusCode == 404 { - return &RateLimits{}, fmt.Errorf("Rate Limiting not enabled in GitHub API") - } - - limit, err := strconv.ParseFloat(resp.Header.Get("X-RateLimit-Limit"), 64) - - if err != nil { - return &RateLimits{}, err - } - - rem, err := strconv.ParseFloat(resp.Header.Get("X-RateLimit-Remaining"), 64) - - if err != nil { - return &RateLimits{}, err - } - - reset, err := strconv.ParseFloat(resp.Header.Get("X-RateLimit-Reset"), 64) - - if err != nil { - return &RateLimits{}, err - } - - return &RateLimits{ - Limit: limit, - Remaining: rem, - Reset: reset, - }, err - -} - -func getReleases(e *Exporter, url string, data *[]Release) { - i := strings.Index(url, "?") - baseURL := url[:i] - releasesURL := baseURL + "/releases" - releasesResponse, err := asyncHTTPGets([]string{releasesURL}, e.GithubToken) - - if err != nil { - log.Errorf("Unable to obtain releases from API, Error: %s", err) - return - } - - err = json.Unmarshal(releasesResponse[0].body, &data) - if err != nil { - log.Errorf("Unable to unmarshal releases from API response, Error: %s", err) - } -} - -func getPRs(e *Exporter, url string, data *[]Pull) { - i := strings.Index(url, "?") - baseURL := url[:i] - pullsURL := baseURL + "/pulls" - pullsResponse, err := asyncHTTPGets([]string{pullsURL}, e.GithubToken) - - if err != nil { - log.Errorf("Unable to obtain pull requests from API, Error: %s", err) - return - } - - err = json.Unmarshal(pullsResponse[0].body, &data) - if err != nil { - log.Errorf("Unable to unmarshal pulls from API response, Error: %s", err) - } -} - -// isArray simply looks for key details that determine if the JSON response is an array or not. -func isArray(body []byte) bool { - - isArray := false - - for _, c := range body { - if c == ' ' || c == '\t' || c == '\r' || c == '\n' { - continue - } - isArray = c == '[' - break - } - - return isArray - -} diff --git a/exporter/http.go b/exporter/http.go deleted file mode 100644 index 273f63f4..00000000 --- a/exporter/http.go +++ /dev/null @@ -1,160 +0,0 @@ -package exporter - -import ( - "fmt" - "io" - "net/http" - neturl "net/url" - "strconv" - "time" - - log "github.com/sirupsen/logrus" - "github.com/tomnomnom/linkheader" -) - -// RateLimitExceededStatus is the status response from github when the rate limit is exceeded. -const RateLimitExceededStatus = "403 rate limit exceeded" - -func asyncHTTPGets(targets []string, token string) ([]*Response, error) { - // Expand targets by following GitHub pagination links - targets = paginateTargets(targets, token) - - // Channels used to enable concurrent requests - ch := make(chan *Response, len(targets)) - - responses := []*Response{} - - for _, url := range targets { - - go func(url string) { - err := getResponse(url, token, ch) - if err != nil { - ch <- &Response{url, nil, []byte{}, err} - } - }(url) - - } - - for { - select { - case r := <-ch: - if r.err != nil { - log.Errorf("Error scraping API, Error: %v", r.err) - return nil, r.err - } - responses = append(responses, r) - - if len(responses) == len(targets) { - return responses, nil - } - } - - } -} - -// paginateTargets returns all pages for the provided targets -func paginateTargets(targets []string, token string) []string { - - paginated := targets - - for _, url := range targets { - - // make a request to the original target to get link header if it exists - resp, err := getHTTPResponse(url, token) - if err != nil { - log.Errorf("Error retrieving Link headers, Error: %s", err) - continue - } - - if resp.Header["Link"] != nil { - links := linkheader.Parse(resp.Header["Link"][0]) - - for _, link := range links { - if link.Rel == "last" { - - u, err := neturl.Parse(link.URL) - if err != nil { - log.Errorf("Unable to parse page URL, Error: %s", err) - } - - q := u.Query() - - lastPage, err := strconv.Atoi(q.Get("page")) - if err != nil { - log.Errorf("Unable to convert page substring to int, Error: %s", err) - } - - // add all pages to the slice of targets to return - for page := 2; page <= lastPage; page++ { - q.Set("page", strconv.Itoa(page)) - u.RawQuery = q.Encode() - paginated = append(paginated, u.String()) - } - - break - } - } - } - } - return paginated -} - -// getResponse collects an individual http.response and returns a *Response -func getResponse(url string, token string, ch chan<- *Response) error { - - log.Infof("Fetching %s \n", url) - - resp, err := getHTTPResponse(url, token) // do this earlier - if err != nil { - return fmt.Errorf("Error fetching http response: %v", err) - } - defer resp.Body.Close() - - // Read the body to a byte array so it can be used elsewhere - body, err := io.ReadAll(resp.Body) - if err != nil { - return fmt.Errorf("Error converting body to byte array: %v", err) - } - - // Triggers if a user specifies an invalid or not visible repository - if resp.StatusCode == 404 { - return fmt.Errorf("Error: Received 404 status from Github API, ensure the repository URL is correct. If it's a private repository, also check the oauth token is correct") - } - - ch <- &Response{url, resp, body, err} - - return nil -} - -// getHTTPResponse handles the http client creation, token setting and returns the *http.response -func getHTTPResponse(url string, token string) (*http.Response, error) { - - client := &http.Client{ - Timeout: time.Second * 10, - } - - req, err := http.NewRequest("GET", url, nil) - - if err != nil { - return nil, err - } - - // If a token is present, add it to the http.request - if token != "" { - req.Header.Add("Authorization", "token "+token) - } - - resp, err := client.Do(req) - - if err != nil { - return nil, err - } - - // check rate limit exceeded. - if resp.Status == RateLimitExceededStatus { - resp.Body.Close() - return nil, fmt.Errorf("%s", resp.Status) - } - - return resp, err -} diff --git a/exporter/metrics.go b/exporter/metrics.go index 32618825..b6a98239 100644 --- a/exporter/metrics.go +++ b/exporter/metrics.go @@ -70,10 +70,10 @@ func (e *Exporter) processMetrics(data []*Datum, rates *RateLimits, ch chan<- pr // APIMetrics - range through the data slice for _, x := range data { - ch <- prometheus.MustNewConstMetric(e.APIMetrics["Stars"], prometheus.GaugeValue, x.Stars, x.Name, x.Owner.Login, strconv.FormatBool(x.Private), strconv.FormatBool(x.Fork), strconv.FormatBool(x.Archived), x.License.Key, x.Language) - ch <- prometheus.MustNewConstMetric(e.APIMetrics["Forks"], prometheus.GaugeValue, x.Forks, x.Name, x.Owner.Login, strconv.FormatBool(x.Private), strconv.FormatBool(x.Fork), strconv.FormatBool(x.Archived), x.License.Key, x.Language) - ch <- prometheus.MustNewConstMetric(e.APIMetrics["Watchers"], prometheus.GaugeValue, x.Watchers, x.Name, x.Owner.Login, strconv.FormatBool(x.Private), strconv.FormatBool(x.Fork), strconv.FormatBool(x.Archived), x.License.Key, x.Language) - ch <- prometheus.MustNewConstMetric(e.APIMetrics["Size"], prometheus.GaugeValue, x.Size, x.Name, x.Owner.Login, strconv.FormatBool(x.Private), strconv.FormatBool(x.Fork), strconv.FormatBool(x.Archived), x.License.Key, x.Language) + ch <- prometheus.MustNewConstMetric(e.APIMetrics["Stars"], prometheus.GaugeValue, float64(x.Stars), x.Name, x.Owner.Login, strconv.FormatBool(x.Private), strconv.FormatBool(x.Fork), strconv.FormatBool(x.Archived), x.License.Key, x.Language) + ch <- prometheus.MustNewConstMetric(e.APIMetrics["Forks"], prometheus.GaugeValue, float64(x.Forks), x.Name, x.Owner.Login, strconv.FormatBool(x.Private), strconv.FormatBool(x.Fork), strconv.FormatBool(x.Archived), x.License.Key, x.Language) + ch <- prometheus.MustNewConstMetric(e.APIMetrics["Watchers"], prometheus.GaugeValue, float64(x.Watchers), x.Name, x.Owner.Login, strconv.FormatBool(x.Private), strconv.FormatBool(x.Fork), strconv.FormatBool(x.Archived), x.License.Key, x.Language) + ch <- prometheus.MustNewConstMetric(e.APIMetrics["Size"], prometheus.GaugeValue, float64(x.Size), x.Name, x.Owner.Login, strconv.FormatBool(x.Private), strconv.FormatBool(x.Fork), strconv.FormatBool(x.Archived), x.License.Key, x.Language) for _, release := range x.Releases { for _, asset := range release.Assets { @@ -85,7 +85,7 @@ func (e *Exporter) processMetrics(data []*Datum, rates *RateLimits, ch chan<- pr prCount += 1 } // issueCount = x.OpenIssue - prCount - ch <- prometheus.MustNewConstMetric(e.APIMetrics["OpenIssues"], prometheus.GaugeValue, (x.OpenIssues - float64(prCount)), x.Name, x.Owner.Login, strconv.FormatBool(x.Private), strconv.FormatBool(x.Fork), strconv.FormatBool(x.Archived), x.License.Key, x.Language) + ch <- prometheus.MustNewConstMetric(e.APIMetrics["OpenIssues"], prometheus.GaugeValue, float64(x.OpenIssues-prCount), x.Name, x.Owner.Login, strconv.FormatBool(x.Private), strconv.FormatBool(x.Fork), strconv.FormatBool(x.Archived), x.License.Key, x.Language) // prCount ch <- prometheus.MustNewConstMetric(e.APIMetrics["PullRequestCount"], prometheus.GaugeValue, float64(prCount), x.Name, x.Owner.Login) diff --git a/exporter/prometheus.go b/exporter/prometheus.go index fff5dd4a..38caef25 100644 --- a/exporter/prometheus.go +++ b/exporter/prometheus.go @@ -1,94 +1,212 @@ package exporter import ( - "path" - "strconv" + "context" + "fmt" + "strings" + "time" + "github.com/google/go-github/v76/github" "github.com/prometheus/client_golang/prometheus" log "github.com/sirupsen/logrus" ) // Describe - loops through the API metrics and passes them to prometheus.Describe func (e *Exporter) Describe(ch chan<- *prometheus.Desc) { - for _, m := range e.APIMetrics { ch <- m } - } // Collect function, called on by Prometheus Client library -// This function is called when a scrape is peformed on the /metrics page +// This function is called when a scrape is performed on the /metrics endpoint func (e *Exporter) Collect(ch chan<- prometheus.Metric) { - data := []*Datum{} - var err error + ctx := context.Background() + + log.Info("collecting metrics") + var data []*Datum + + repoMetrics, err := e.getRepoMetrics(ctx) + if err != nil { + log.Errorf("Error fetching repository metrics: %v", err) + return + } + + data = append(data, repoMetrics...) + + // TODO - get all rate limits, not just core + rates, _, err := e.Client.RateLimit.Get(ctx) + if err != nil { + log.Errorf("Error fetching rate limits: %v", err) + return + } + + r := &RateLimits{ + Limit: float64(rates.Core.Limit), + Remaining: float64(rates.Core.Remaining), + Reset: float64(rates.Core.Reset.Unix()), + } - if e.Config.GitHubApp { - needReAuth, err := e.isTokenExpired() + // Set prometheus gauge metrics using the data gathered + err = e.processMetrics(data, r, ch) +} + +// getRateLimits fetches the current rate limits from the GitHub API +func (e *Exporter) getRateLimits(ctx context.Context) (*RateLimits, error) { + rates, _, err := e.Client.RateLimit.Get(ctx) + if err != nil { + log.Errorf("Error fetching rate limits: %v", err) + return nil, err + } + + r := &RateLimits{ + Limit: float64(rates.Core.Limit), + Remaining: float64(rates.Core.Remaining), + Reset: float64(rates.Core.Reset.Unix()), + } + + return r, nil +} + +// getOrganisationMetrics fetches metrics for the configured organisations +func (e *Exporter) getOrganisationMetrics(ctx context.Context) ([]*Datum, error) { + var data []*Datum + for _, o := range e.Config.Organisations { + repos, _, err := e.Client.Repositories.ListByOrg(ctx, o, nil) if err != nil { - log.Errorf("Error checking token expiration status: %v", err) - return + log.Errorf("Error fetching organisation repositories: %v", err) + continue } - if needReAuth { - token, err := e.Config.APITokenFromGitHubApp() + for _, repo := range repos { + d, err := e.parseRepo(ctx, *repo) if err != nil { - log.Errorf("Error authenticating with GitHub app: %v", err) + log.Errorf("Error parsing organisation data: %v", err) + continue } - e.GithubToken = token + data = append(data, d) } } - // Scrape the Data from Github - if len(e.TargetURLs()) > 0 { - data, err = e.gatherData() + + return data, nil +} + +// getUserMetrics fetches metrics for the configured users +func (e *Exporter) getUserMetrics(ctx context.Context) ([]*Datum, error) { + var data []*Datum + for _, u := range e.Config.Users { + repos, _, err := e.Client.Repositories.ListByUser(ctx, u, nil) if err != nil { - log.Errorf("Error gathering Data from remote API: %v", err) - return + log.Errorf("Error fetching user data: %v", err) + continue } - } - rates, err := e.getRates() - if err != nil { - log.Errorf("Error gathering Rates from remote API: %v", err) - return + for _, repo := range repos { + d, err := e.parseRepo(ctx, *repo) + if err != nil { + log.Errorf("Error parsing user data: %v", err) + continue + } + data = append(data, d) + } } - // Set prometheus gauge metrics using the data gathered - err = e.processMetrics(data, rates, ch) + return data, nil +} - if err != nil { - log.Error("Error Processing Metrics", err) - return - } +// getRepoMetrics fetches metrics for the configured repositories +func (e *Exporter) getRepoMetrics(ctx context.Context) ([]*Datum, error) { + var data []*Datum + for _, m := range e.Config.Repositories { + // Split the repository string into owner and name + parts := strings.Split(m, "/") + if len(parts) != 2 { + log.Errorf("Invalid repository format: %s", m) + continue + } - log.Info("All Metrics successfully collected") + repo, _, err := e.Client.Repositories.Get(ctx, parts[0], parts[1]) + if err != nil { + log.Errorf("Error fetching repository data: %v", err) + continue + } -} + d, err := e.parseRepo(ctx, *repo) + if err != nil { + log.Errorf("Error parsing repository data: %v", err) + continue + } -func (e *Exporter) isTokenExpired() (bool, error) { - u := *e.ApiUrl - u.Path = path.Join(u.Path, "rate_limit") + data = append(data, d) + } - resp, err := getHTTPResponse(u.String(), e.GithubToken) + return data, nil +} +func (e *Exporter) parseRepo(ctx context.Context, repo github.Repository) (*Datum, error) { + repoOwner := repo.GetOwner().GetLogin() + repoName := repo.GetName() + + rel, _, err := e.Client.Repositories.ListReleases(ctx, repoOwner, repoName, nil) if err != nil { - return false, err - } - defer resp.Body.Close() - // Triggers if rate-limiting isn't enabled on private Github Enterprise installations - if resp.StatusCode == 404 { - return false, nil + return nil, fmt.Errorf("listing releases: %w", err) } - limit, err := strconv.ParseFloat(resp.Header.Get("X-RateLimit-Limit"), 64) + var releases []Release + for _, release := range rel { + var assets []Asset + for _, asset := range release.Assets { + a := Asset{ + Name: asset.GetName(), + Size: asset.GetSize(), + Downloads: asset.GetDownloadCount(), + CreatedAt: asset.GetCreatedAt().Format(time.RFC3339), + } + assets = append(assets, a) + } + + r := Release{ + Name: release.GetName(), + Tag: release.GetTagName(), + Assets: assets, + } + releases = append(releases, r) + } + pullRequests, _, err := e.Client.PullRequests.List(ctx, repoOwner, repoName, nil) if err != nil { - return false, err + return nil, fmt.Errorf("fetching pull requests: %w", err) + } + var pulls []Pull + for _, pr := range pullRequests { + p := Pull{ + Url: pr.GetURL(), + User: User{ + Login: pr.GetUser().GetLogin(), + }, + } + pulls = append(pulls, p) } - defaultRateLimit := e.Config.GitHubRateLimit - if limit < defaultRateLimit { - return true, nil + d := &Datum{ + Name: repo.GetName(), + Owner: User{ + Login: repo.GetOwner().GetLogin(), + }, + License: License{ + Key: repo.GetLicense().GetKey(), + }, + Language: repo.GetLanguage(), + Archived: repo.GetArchived(), + Private: repo.GetPrivate(), + Fork: repo.GetFork(), + Forks: repo.GetForksCount(), + Stars: repo.GetStargazersCount(), + OpenIssues: repo.GetOpenIssuesCount(), + Watchers: repo.GetSubscribersCount(), + Size: repo.GetSize(), + Releases: releases, + Pulls: pulls, } - return false, nil + return d, nil } diff --git a/exporter/structs.go b/exporter/structs.go index 858b876f..54524ba6 100644 --- a/exporter/structs.go +++ b/exporter/structs.go @@ -4,6 +4,8 @@ import ( "net/http" "github.com/githubexporter/github-exporter/config" + + "github.com/google/go-github/v76/github" "github.com/prometheus/client_golang/prometheus" ) @@ -12,6 +14,7 @@ import ( // user defined runtime configuration when the Collect method is called. type Exporter struct { APIMetrics map[string]*prometheus.Desc + Client *github.Client config.Config } @@ -21,26 +24,26 @@ type Data []Datum // Datum is used to store data from all the relevant endpoints in the API type Datum struct { - Name string `json:"name"` - Owner struct { - Login string `json:"login"` - } `json:"owner"` - License struct { - Key string `json:"key"` - } `json:"license"` + Name string `json:"name"` + Owner User `json:"owner"` + License License `json:"license"` Language string `json:"language"` Archived bool `json:"archived"` Private bool `json:"private"` Fork bool `json:"fork"` - Forks float64 `json:"forks"` - Stars float64 `json:"stargazers_count"` - OpenIssues float64 `json:"open_issues"` - Watchers float64 `json:"subscribers_count"` - Size float64 `json:"size"` + Forks int `json:"forks"` + Stars int `json:"stargazers_count"` + OpenIssues int `json:"open_issues"` + Watchers int `json:"subscribers_count"` + Size int `json:"size"` Releases []Release Pulls []Pull } +type License struct { + Key string `json:"key"` +} + type Release struct { Name string `json:"name"` Assets []Asset `json:"assets"` @@ -49,15 +52,17 @@ type Release struct { type Pull struct { Url string `json:"url"` - User struct { - Login string `json:"login"` - } `json:"user"` + User User +} + +type User struct { + Login string `json:"login"` } type Asset struct { Name string `json:"name"` - Size int64 `json:"size"` - Downloads int32 `json:"download_count"` + Size int `json:"size"` + Downloads int `json:"download_count"` CreatedAt string `json:"created_at"` } diff --git a/go.mod b/go.mod index ea23caff..7e3957cd 100644 --- a/go.mod +++ b/go.mod @@ -1,9 +1,10 @@ module github.com/githubexporter/github-exporter -go 1.22 +go 1.24.0 require ( github.com/bradleyfalzon/ghinstallation/v2 v2.11.0 + github.com/google/go-github/v76 v76.0.0 github.com/kelseyhightower/envconfig v1.4.0 github.com/prometheus/client_golang v1.20.4 github.com/sirupsen/logrus v1.9.3 diff --git a/go.sum b/go.sum index d0c8c62a..43222a34 100644 --- a/go.sum +++ b/go.sum @@ -11,10 +11,13 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg= github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/go-github/v62 v62.0.0 h1:/6mGCaRywZz9MuHyw9gD1CwsbmBX8GWsbFkwMmHdhl4= github.com/google/go-github/v62 v62.0.0/go.mod h1:EMxeUqGJq2xRu9DYBMwel/mr7kZrzUOfQmmpYrZn2a4= +github.com/google/go-github/v76 v76.0.0 h1:MCa9VQn+VG5GG7Y7BAkBvSRUN3o+QpaEOuZwFPJmdFA= +github.com/google/go-github/v76 v76.0.0/go.mod h1:38+d/8pYDO4fBLYfBhXF5EKO0wA3UkXBjfmQapFsNCQ= github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dvMUtDTo2cv8= diff --git a/http/server.go b/http/server.go index 4cceddb9..6fbd212b 100644 --- a/http/server.go +++ b/http/server.go @@ -5,6 +5,7 @@ import ( "net/http" "github.com/githubexporter/github-exporter/exporter" + "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" ) diff --git a/main.go b/main.go index d4bbe997..4a904b9c 100644 --- a/main.go +++ b/main.go @@ -1,16 +1,18 @@ package main import ( - conf "github.com/githubexporter/github-exporter/config" + "github.com/githubexporter/github-exporter/config" "github.com/githubexporter/github-exporter/exporter" "github.com/githubexporter/github-exporter/http" + + "github.com/google/go-github/v76/github" log "github.com/sirupsen/logrus" ) func main() { log.Info("Starting Exporter") - applicationCfg, err := conf.Init() + applicationCfg, err := config.Init() if err != nil { log.Fatalf("Error initializing configuration: %v", err) } @@ -20,6 +22,7 @@ func main() { exp := exporter.Exporter{ APIMetrics: metrics, Config: *applicationCfg, + Client: github.NewClient(nil), } http.NewServer(exp).Start() diff --git a/test/github_exporter_test.go b/test/github_exporter_test.go index c8a0ad17..8f6f4a56 100644 --- a/test/github_exporter_test.go +++ b/test/github_exporter_test.go @@ -11,6 +11,8 @@ import ( "github.com/githubexporter/github-exporter/config" "github.com/githubexporter/github-exporter/exporter" web "github.com/githubexporter/github-exporter/http" + + "github.com/google/go-github/v76/github" "github.com/prometheus/client_golang/prometheus" "github.com/steinfletcher/apitest" ) @@ -40,7 +42,7 @@ func TestGithubExporter(t *testing.T) { Expect(t). Assert(bodyContains(`github_rate_limit 60`)). Assert(bodyContains(`github_rate_remaining 60`)). - Assert(bodyContains(`github_rate_reset 1.566853865e+09`)). + Assert(bodyContains(`github_rate_reset 3e+09`)). Assert(bodyContains(`github_repo_forks{archived="false",fork="false",language="Go",license="mit",private="false",repo="myRepo",user="myOrg"} 10`)). Assert(bodyContains(`github_repo_pull_request_count{repo="myRepo",user="myOrg"} 3`)). Assert(bodyContains(`github_repo_open_issues{archived="false",fork="false",language="Go",license="mit",private="false",repo="myRepo",user="myOrg"} 2`)). @@ -78,6 +80,7 @@ func apiTest(conf config.Config) (*apitest.APITest, exporter.Exporter) { exp := exporter.Exporter{ APIMetrics: exporter.AddMetrics(), Config: conf, + Client: github.NewClient(nil), } server := web.NewServer(exp) @@ -99,10 +102,8 @@ func withConfig(repos string) config.Config { func githubRepos() *apitest.Mock { return apitest.NewMock(). Get("https://api.github.com/repos/myOrg/myRepo"). - Header("Authorization", "token 12345"). - Query("per_page", "100"). RespondWith(). - Times(2). + Times(1). Body(readFile("testdata/my_repo_response.json")). Status(200). End() @@ -111,11 +112,9 @@ func githubRepos() *apitest.Mock { func githubRateLimit() *apitest.Mock { return apitest.NewMock(). Get("https://api.github.com/rate_limit"). - Header("Authorization", "token 12345"). RespondWith(). - Header("X-RateLimit-Limit", "60"). - Header("X-RateLimit-Remaining", "60"). - Header("X-RateLimit-Reset", "1566853865"). + Times(1). + Body(readFile("testdata/rate_limit_response.json")). Status(http.StatusOK). End() } @@ -123,9 +122,8 @@ func githubRateLimit() *apitest.Mock { func githubReleases() *apitest.Mock { return apitest.NewMock(). Get("https://api.github.com/repos/myOrg/myRepo/releases"). - Header("Authorization", "token 12345"). RespondWith(). - Times(2). + Times(1). Body(readFile("testdata/releases_response.json")). Status(http.StatusOK). End() @@ -134,9 +132,8 @@ func githubReleases() *apitest.Mock { func githubPulls() *apitest.Mock { return apitest.NewMock(). Get("https://api.github.com/repos/myOrg/myRepo/pulls"). - Header("Authorization", "token 12345"). RespondWith(). - Times(2). + Times(1). Body(readFile("testdata/pulls_response.json")). Status(http.StatusOK). End() @@ -145,9 +142,8 @@ func githubPulls() *apitest.Mock { func githubPullsError() *apitest.Mock { return apitest.NewMock(). Get("https://api.github.com/repos/myOrg/myRepo/pulls"). - Header("Authorization", "token 12345"). RespondWith(). - Times(2). + Times(1). Status(http.StatusBadRequest). End() } @@ -168,7 +164,7 @@ func bodyContains(substr string) func(*http.Response, *http.Request) error { } response := string(bytes) if !strings.Contains(response, substr) { - return fmt.Errorf("response did not contain substring '%s'", substr) + return fmt.Errorf("response did not contain substring '%s'", response) } return nil } diff --git a/test/testdata/rate_limit_response.json b/test/testdata/rate_limit_response.json new file mode 100644 index 00000000..5d86feb3 --- /dev/null +++ b/test/testdata/rate_limit_response.json @@ -0,0 +1,46 @@ +{ + "resources": { + "code_search": { + "limit": 60, + "remaining": 60, + "reset": 3000000000, + "used": 6, + "resource": "code_search" + }, + "core": { + "limit": 60, + "remaining": 60, + "reset": 3000000000, + "used": 6, + "resource": "core" + }, + "graphql": { + "limit": 0, + "remaining": 0, + "reset": 3000000000, + "used": 0, + "resource": "graphql" + }, + "integration_manifest": { + "limit": 5000, + "remaining": 5000, + "reset": 3000000000, + "used": 0, + "resource": "integration_manifest" + }, + "search": { + "limit": 10, + "remaining": 10, + "reset": 3000000000, + "used": 0, + "resource": "search" + } + }, + "rate": { + "limit": 60, + "remaining": 60, + "reset": 3000000000, + "used": 6, + "resource": "core" + } +} \ No newline at end of file From bf2f73d2a1d60d8461fa50733cef0b94f9194ee2 Mon Sep 17 00:00:00 2001 From: Henry McConville Date: Sat, 18 Oct 2025 15:19:42 +0100 Subject: [PATCH 03/14] [v2] Upgrade golang to 1.25.2 --- Dockerfile | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 226b4a50..5e1f14fc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,9 +1,7 @@ -ARG GOLANG_VERSION=1.22.7 +ARG GOLANG_VERSION=1.25.2 FROM golang:${GOLANG_VERSION} AS build LABEL maintainer="githubexporter" -ENV GO111MODULE=on - COPY ./ /go/src/github.com/githubexporter/github-exporter WORKDIR /go/src/github.com/githubexporter/github-exporter From 34f65c0b5bd7fea4ce433ef868e96eba4d5de77e Mon Sep 17 00:00:00 2001 From: Henry McConville Date: Sat, 18 Oct 2025 15:23:58 +0100 Subject: [PATCH 04/14] [v2] Remove unused code --- config/config.go | 9 +++--- exporter/prometheus.go | 63 ++---------------------------------- exporter/structs.go | 10 ------ main.go | 2 +- test/github_exporter_test.go | 2 +- 5 files changed, 8 insertions(+), 78 deletions(-) diff --git a/config/config.go b/config/config.go index 1b9686d0..1346b9d4 100644 --- a/config/config.go +++ b/config/config.go @@ -3,7 +3,6 @@ package config import ( "context" "fmt" - "github.com/google/go-github/v76/github" "net/http" "net/url" "os" @@ -11,6 +10,7 @@ import ( "strings" "github.com/bradleyfalzon/ghinstallation/v2" + "github.com/google/go-github/v76/github" "github.com/kelseyhightower/envconfig" log "github.com/sirupsen/logrus" ) @@ -32,10 +32,9 @@ type Config struct { } type GitHubAppConfig struct { - GitHubAppKeyPath string `envconfig:"GITHUB_APP_KEY_PATH" required:"false" default:""` - GitHubAppId int64 `envconfig:"GITHUB_APP_ID" required:"false" default:""` - GitHubAppInstallationId int64 `envconfig:"GITHUB_APP_INSTALLATION_ID" required:"false" default:""` - GitHubRateLimit float64 `envconfig:"GITHUB_RATE_LIMIT" required:"false" default:"15000"` + GitHubAppKeyPath string `envconfig:"GITHUB_APP_KEY_PATH" required:"false" default:""` + GitHubAppId int64 `envconfig:"GITHUB_APP_ID" required:"false" default:""` + GitHubAppInstallationId int64 `envconfig:"GITHUB_APP_INSTALLATION_ID" required:"false" default:""` } // Init populates the Config struct based on environmental runtime configuration diff --git a/exporter/prometheus.go b/exporter/prometheus.go index 38caef25..9305e833 100644 --- a/exporter/prometheus.go +++ b/exporter/prometheus.go @@ -49,74 +49,15 @@ func (e *Exporter) Collect(ch chan<- prometheus.Metric) { // Set prometheus gauge metrics using the data gathered err = e.processMetrics(data, r, ch) -} - -// getRateLimits fetches the current rate limits from the GitHub API -func (e *Exporter) getRateLimits(ctx context.Context) (*RateLimits, error) { - rates, _, err := e.Client.RateLimit.Get(ctx) if err != nil { - log.Errorf("Error fetching rate limits: %v", err) - return nil, err - } - - r := &RateLimits{ - Limit: float64(rates.Core.Limit), - Remaining: float64(rates.Core.Remaining), - Reset: float64(rates.Core.Reset.Unix()), - } - - return r, nil -} - -// getOrganisationMetrics fetches metrics for the configured organisations -func (e *Exporter) getOrganisationMetrics(ctx context.Context) ([]*Datum, error) { - var data []*Datum - for _, o := range e.Config.Organisations { - repos, _, err := e.Client.Repositories.ListByOrg(ctx, o, nil) - if err != nil { - log.Errorf("Error fetching organisation repositories: %v", err) - continue - } - for _, repo := range repos { - d, err := e.parseRepo(ctx, *repo) - if err != nil { - log.Errorf("Error parsing organisation data: %v", err) - continue - } - data = append(data, d) - } + log.Errorf("Error processing metrics: %v", err) } - - return data, nil -} - -// getUserMetrics fetches metrics for the configured users -func (e *Exporter) getUserMetrics(ctx context.Context) ([]*Datum, error) { - var data []*Datum - for _, u := range e.Config.Users { - repos, _, err := e.Client.Repositories.ListByUser(ctx, u, nil) - if err != nil { - log.Errorf("Error fetching user data: %v", err) - continue - } - - for _, repo := range repos { - d, err := e.parseRepo(ctx, *repo) - if err != nil { - log.Errorf("Error parsing user data: %v", err) - continue - } - data = append(data, d) - } - } - - return data, nil } // getRepoMetrics fetches metrics for the configured repositories func (e *Exporter) getRepoMetrics(ctx context.Context) ([]*Datum, error) { var data []*Datum - for _, m := range e.Config.Repositories { + for _, m := range e.Repositories { // Split the repository string into owner and name parts := strings.Split(m, "/") if len(parts) != 2 { diff --git a/exporter/structs.go b/exporter/structs.go index 54524ba6..2ea4dab8 100644 --- a/exporter/structs.go +++ b/exporter/structs.go @@ -1,8 +1,6 @@ package exporter import ( - "net/http" - "github.com/githubexporter/github-exporter/config" "github.com/google/go-github/v76/github" @@ -73,11 +71,3 @@ type RateLimits struct { Remaining float64 Reset float64 } - -// Response struct is used to store http.Response and associated data -type Response struct { - url string - response *http.Response - body []byte - err error -} diff --git a/main.go b/main.go index 4a904b9c..5f346e5e 100644 --- a/main.go +++ b/main.go @@ -4,7 +4,7 @@ import ( "github.com/githubexporter/github-exporter/config" "github.com/githubexporter/github-exporter/exporter" "github.com/githubexporter/github-exporter/http" - + "github.com/google/go-github/v76/github" log "github.com/sirupsen/logrus" ) diff --git a/test/github_exporter_test.go b/test/github_exporter_test.go index 8f6f4a56..6d89a609 100644 --- a/test/github_exporter_test.go +++ b/test/github_exporter_test.go @@ -11,7 +11,7 @@ import ( "github.com/githubexporter/github-exporter/config" "github.com/githubexporter/github-exporter/exporter" web "github.com/githubexporter/github-exporter/http" - + "github.com/google/go-github/v76/github" "github.com/prometheus/client_golang/prometheus" "github.com/steinfletcher/apitest" From 649520854ab77a5e13cd51ffdf471464cca8305f Mon Sep 17 00:00:00 2001 From: Henry McConville Date: Sat, 18 Oct 2025 15:28:03 +0100 Subject: [PATCH 05/14] [v2] Update go dependencies --- go.mod | 25 +++++++++++++------------ go.sum | 51 ++++++++++++++++++++++++++------------------------- 2 files changed, 39 insertions(+), 37 deletions(-) diff --git a/go.mod b/go.mod index 7e3957cd..7666334b 100644 --- a/go.mod +++ b/go.mod @@ -2,32 +2,33 @@ module github.com/githubexporter/github-exporter go 1.24.0 +toolchain go1.25.2 + require ( - github.com/bradleyfalzon/ghinstallation/v2 v2.11.0 + github.com/bradleyfalzon/ghinstallation/v2 v2.17.0 github.com/google/go-github/v76 v76.0.0 github.com/kelseyhightower/envconfig v1.4.0 - github.com/prometheus/client_golang v1.20.4 + github.com/prometheus/client_golang v1.23.2 github.com/sirupsen/logrus v1.9.3 github.com/steinfletcher/apitest v1.3.8 - github.com/stretchr/testify v1.9.0 - github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 + github.com/stretchr/testify v1.11.1 ) require ( github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect - github.com/golang-jwt/jwt/v4 v4.5.0 // indirect - github.com/google/go-github/v62 v62.0.0 // indirect + github.com/golang-jwt/jwt/v4 v4.5.2 // indirect + github.com/google/go-github/v75 v75.0.0 // indirect github.com/google/go-querystring v1.1.0 // indirect - github.com/klauspost/compress v1.17.10 // indirect github.com/kr/text v0.2.0 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/prometheus/client_model v0.6.1 // indirect - github.com/prometheus/common v0.59.1 // indirect - github.com/prometheus/procfs v0.15.1 // indirect - golang.org/x/sys v0.25.0 // indirect - google.golang.org/protobuf v1.34.2 // indirect + github.com/prometheus/client_model v0.6.2 // indirect + github.com/prometheus/common v0.67.1 // indirect + github.com/prometheus/procfs v0.17.0 // indirect + go.yaml.in/yaml/v2 v2.4.3 // indirect + golang.org/x/sys v0.37.0 // indirect + google.golang.org/protobuf v1.36.10 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 43222a34..8a61eb22 100644 --- a/go.sum +++ b/go.sum @@ -1,29 +1,28 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/bradleyfalzon/ghinstallation/v2 v2.11.0 h1:R9d0v+iobRHSaE4wKUnXFiZp53AL4ED5MzgEMwGTZag= -github.com/bradleyfalzon/ghinstallation/v2 v2.11.0/go.mod h1:0LWKQwOHewXO/1acI6TtyE0Xc4ObDb2rFN7eHBAG71M= +github.com/bradleyfalzon/ghinstallation/v2 v2.17.0 h1:SmbUK/GxpAspRjSQbB6ARvH+ArzlNzTtHydNyXUQ6zg= +github.com/bradleyfalzon/ghinstallation/v2 v2.17.0/go.mod h1:vuD/xvJT9Y+ZVZRv4HQ42cMyPFIYqpc7AbB4Gvt/DlY= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg= -github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= +github.com/golang-jwt/jwt/v4 v4.5.2 h1:YtQM7lnr8iZ+j5q71MGKkNw9Mn7AjHM68uc9g5fXeUI= +github.com/golang-jwt/jwt/v4 v4.5.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= -github.com/google/go-github/v62 v62.0.0 h1:/6mGCaRywZz9MuHyw9gD1CwsbmBX8GWsbFkwMmHdhl4= -github.com/google/go-github/v62 v62.0.0/go.mod h1:EMxeUqGJq2xRu9DYBMwel/mr7kZrzUOfQmmpYrZn2a4= +github.com/google/go-github/v75 v75.0.0 h1:k7q8Bvg+W5KxRl9Tjq16a9XEgVY1pwuiG5sIL7435Ic= +github.com/google/go-github/v75 v75.0.0/go.mod h1:H3LUJEA1TCrzuUqtdAQniBNwuKiQIqdGKgBo1/M/uqI= github.com/google/go-github/v76 v76.0.0 h1:MCa9VQn+VG5GG7Y7BAkBvSRUN3o+QpaEOuZwFPJmdFA= github.com/google/go-github/v76 v76.0.0/go.mod h1:38+d/8pYDO4fBLYfBhXF5EKO0wA3UkXBjfmQapFsNCQ= github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dvMUtDTo2cv8= github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg= -github.com/klauspost/compress v1.17.10 h1:oXAz+Vh0PMUvJczoi+flxpnBEPxoER1IaAnU/NMPtT0= -github.com/klauspost/compress v1.17.10/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= +github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= +github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= @@ -34,14 +33,14 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/client_golang v1.20.4 h1:Tgh3Yr67PaOv/uTqloMsCEdeuFTatm5zIq5+qNN23vI= -github.com/prometheus/client_golang v1.20.4/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE= -github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= -github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= -github.com/prometheus/common v0.59.1 h1:LXb1quJHWm1P6wq/U824uxYi4Sg0oGvNeUm1z5dJoX0= -github.com/prometheus/common v0.59.1/go.mod h1:GpWM7dewqmVYcd7SmRaiWVe9SSqjf0UrwnYnpEZNuT0= -github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= -github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= +github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= +github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= +github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= +github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= +github.com/prometheus/common v0.67.1 h1:OTSON1P4DNxzTg4hmKCc37o4ZAZDv0cfXLkOt0oEowI= +github.com/prometheus/common v0.67.1/go.mod h1:RpmT9v35q2Y+lsieQsdOh5sXZ6ajUGC8NjZAmr8vb0Q= +github.com/prometheus/procfs v0.17.0 h1:FuLQ+05u4ZI+SS/w9+BWEM2TXiHKsUQ9TADiRH7DuK0= +github.com/prometheus/procfs v0.17.0/go.mod h1:oPQLaDAMRbA+u8H5Pbfq+dl3VDAvHxMUOVhe0wYB2zw= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= @@ -51,16 +50,18 @@ github.com/steinfletcher/apitest v1.3.8/go.mod h1:LOVbGzWvWCiiVE4PZByfhRnA5L00l5 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= -github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 h1:nrZ3ySNYwJbSpD6ce9duiP+QkD3JuLCcWkdaehUS/3Y= -github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80/go.mod h1:iFyPdL66DjUD96XmzVL3ZntbzcflLnznH0fr99w5VqE= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0= +go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= -golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.37.0 h1:fdNQudmxPjkdUTPnLn5mdQv7Zwvbvpaxqs831goi9kQ= +golang.org/x/sys v0.37.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= -google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= +google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE= +google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= From f914129953f763d2b22c2918d1ee9fd87d1b72f3 Mon Sep 17 00:00:00 2001 From: Henry McConville Date: Sat, 18 Oct 2025 16:14:38 +0100 Subject: [PATCH 06/14] [v2] Add option to disable github rate limit metric collection --- README.md | 4 +-- config/config.go | 25 ++++++++------- config/config_test.go | 43 +++++++++++++------------ exporter/metrics.go | 61 ++++++++++++++++++++++++------------ exporter/prometheus.go | 29 ++++++++++++----- main.go | 12 +++---- test/github_exporter_test.go | 2 +- 7 files changed, 105 insertions(+), 71 deletions(-) diff --git a/README.md b/README.md index af554734..82eb15f4 100644 --- a/README.md +++ b/README.md @@ -16,11 +16,11 @@ This exporter is setup to take input from environment variables. All variables a * `GITHUB_APP_ID` The APP ID of the GitHub App. * `GITHUB_APP_INSTALLATION_ID` The INSTALLATION ID of the GitHub App. * `GITHUB_APP_KEY_PATH` The path to the github private key. -* `GITHUB_RATE_LIMIT` The RATE LIMIT that suppose to be for github app (default is 15,000). If the exporter sees the value is below this variable it generating new token for the app. +* `GITHUB_RATE_LIMIT_ENABLED` Whether to fetch Rate Limit metrics. Defaults to `true` * `API_URL` Github API URL, shouldn't need to change this. Defaults to `https://api.github.com` * `LISTEN_PORT` The port you wish to run the container on, the Dockerfile defaults this to `9171` * `METRICS_PATH` the metrics URL path you wish to use, defaults to `/metrics` -* `LOG_LEVEL` The level of logging the exporter will run with, defaults to `debug` +* `LOG_LEVEL` The level of logging the exporter will run with, defaults to `info` ## Install and deploy diff --git a/config/config.go b/config/config.go index 1346b9d4..6337ee8e 100644 --- a/config/config.go +++ b/config/config.go @@ -17,18 +17,19 @@ import ( // Config struct holds runtime configuration required for the application type Config struct { - MetricsPath string `envconfig:"METRICS_PATH" required:"false" default:"/metrics"` - ListenPort string `envconfig:"LISTEN_PORT" required:"false" default:"9171"` - LogLevel string `envconfig:"LOG_LEVEL" required:"false" default:"INFO"` - AppName string `envconfig:"APP_NAME" required:"false" default:"github-exporter"` - ApiUrl *url.URL `envconfig:"API_URL" required:"false" default:"https://api.github.com"` - Repositories []string `envconfig:"REPOS" required:"false"` - Organisations []string `envconfig:"ORGS" required:"false"` - Users []string `envconfig:"USERS" required:"false"` - GithubToken string `envconfig:"GITHUB_TOKEN" required:"false"` - GithubTokenFile string `envconfig:"GITHUB_TOKEN_FILE" required:"false"` - GitHubApp bool `envconfig:"GITHUB_APP" required:"false" default:"false"` - *GitHubAppConfig `ignored:"true"` + MetricsPath string `envconfig:"METRICS_PATH" required:"false" default:"/metrics"` + ListenPort string `envconfig:"LISTEN_PORT" required:"false" default:"9171"` + LogLevel string `envconfig:"LOG_LEVEL" required:"false" default:"INFO"` + AppName string `envconfig:"APP_NAME" required:"false" default:"github-exporter"` + ApiUrl *url.URL `envconfig:"API_URL" required:"false" default:"https://api.github.com"` + Repositories []string `envconfig:"REPOS" required:"false"` + Organisations []string `envconfig:"ORGS" required:"false"` + Users []string `envconfig:"USERS" required:"false"` + GithubToken string `envconfig:"GITHUB_TOKEN" required:"false"` + GithubTokenFile string `envconfig:"GITHUB_TOKEN_FILE" required:"false"` + GitHubApp bool `envconfig:"GITHUB_APP" required:"false" default:"false"` + GitHubRateLimitEnabled bool `envconfig:"GITHUB_RATE_LIMIT_ENABLED" required:"false" default:"true"` + *GitHubAppConfig `ignored:"true"` } type GitHubAppConfig struct { diff --git a/config/config_test.go b/config/config_test.go index 867662ad..2cfe2759 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -26,28 +26,30 @@ func TestConfig(t *testing.T) { Scheme: "https", Host: "api.github.com", }, - Repositories: []string{}, - Organisations: []string{}, - Users: []string{}, - GithubToken: "", - GithubTokenFile: "", - GitHubApp: false, - GitHubAppConfig: nil, + Repositories: []string{}, + Organisations: []string{}, + Users: []string{}, + GithubToken: "", + GithubTokenFile: "", + GitHubApp: false, + GitHubAppConfig: nil, + GitHubRateLimitEnabled: true, }, expectedErr: nil, }, { name: "non-default config", envVars: map[string]string{ - "METRICS_PATH": "/otherendpoint", - "LISTEN_PORT": "1111", - "LOG_LEVEL": "DEBUG", - "APP_NAME": "other-app-name", - "API_URL": "https://example.com", - "REPOS": "repo1, repo2", - "ORGS": "org1,org2 ", - "USERS": " user1, user2 ", - "GITHUB_TOKEN": "token", + "METRICS_PATH": "/otherendpoint", + "LISTEN_PORT": "1111", + "LOG_LEVEL": "DEBUG", + "APP_NAME": "other-app-name", + "API_URL": "https://example.com", + "REPOS": "repo1, repo2", + "ORGS": "org1,org2 ", + "USERS": " user1, user2 ", + "GITHUB_TOKEN": "token", + "GITHUB_RATE_LIMIT_ENABLED": "false", }, expectedCfg: &Config{ MetricsPath: "/otherendpoint", @@ -70,10 +72,11 @@ func TestConfig(t *testing.T) { "user1", "user2", }, - GithubToken: "token", - GithubTokenFile: "", - GitHubApp: false, - GitHubAppConfig: nil, + GithubToken: "token", + GithubTokenFile: "", + GitHubApp: false, + GitHubAppConfig: nil, + GitHubRateLimitEnabled: false, }, expectedErr: nil, }, diff --git a/exporter/metrics.go b/exporter/metrics.go index b6a98239..0cf16553 100644 --- a/exporter/metrics.go +++ b/exporter/metrics.go @@ -1,13 +1,29 @@ package exporter import ( + "fmt" "strconv" + "github.com/githubexporter/github-exporter/config" + "github.com/prometheus/client_golang/prometheus" ) -// AddMetrics - Add's all of the metrics to a map of strings, returns the map. -func AddMetrics() map[string]*prometheus.Desc { +func NewExporter(cfg *config.Config) (*Exporter, error) { + client, err := cfg.GetClient() + if err != nil { + return nil, fmt.Errorf("getting client: %w", err) + } + + return &Exporter{ + APIMetrics: AddMetrics(cfg), + Config: *cfg, + Client: client, + }, nil +} + +// AddMetrics - Adds all the metrics to a map of strings, returns the map. +func AddMetrics(cfg *config.Config) map[string]*prometheus.Desc { APIMetrics := make(map[string]*prometheus.Desc) @@ -46,21 +62,24 @@ func AddMetrics() map[string]*prometheus.Desc { "Download count for a given release", []string{"repo", "user", "release", "name", "tag", "created_at"}, nil, ) - APIMetrics["Limit"] = prometheus.NewDesc( - prometheus.BuildFQName("github", "rate", "limit"), - "Number of API queries allowed in a 60 minute window", - []string{}, nil, - ) - APIMetrics["Remaining"] = prometheus.NewDesc( - prometheus.BuildFQName("github", "rate", "remaining"), - "Number of API queries remaining in the current window", - []string{}, nil, - ) - APIMetrics["Reset"] = prometheus.NewDesc( - prometheus.BuildFQName("github", "rate", "reset"), - "The time at which the current rate limit window resets in UTC epoch seconds", - []string{}, nil, - ) + + if cfg.GitHubRateLimitEnabled { + APIMetrics["Limit"] = prometheus.NewDesc( + prometheus.BuildFQName("github", "rate", "limit"), + "Number of API queries allowed in a 60 minute window", + []string{}, nil, + ) + APIMetrics["Remaining"] = prometheus.NewDesc( + prometheus.BuildFQName("github", "rate", "remaining"), + "Number of API queries remaining in the current window", + []string{}, nil, + ) + APIMetrics["Reset"] = prometheus.NewDesc( + prometheus.BuildFQName("github", "rate", "reset"), + "The time at which the current rate limit window resets in UTC epoch seconds", + []string{}, nil, + ) + } return APIMetrics } @@ -92,9 +111,11 @@ func (e *Exporter) processMetrics(data []*Datum, rates *RateLimits, ch chan<- pr } // Set Rate limit stats - ch <- prometheus.MustNewConstMetric(e.APIMetrics["Limit"], prometheus.GaugeValue, rates.Limit) - ch <- prometheus.MustNewConstMetric(e.APIMetrics["Remaining"], prometheus.GaugeValue, rates.Remaining) - ch <- prometheus.MustNewConstMetric(e.APIMetrics["Reset"], prometheus.GaugeValue, rates.Reset) + if e.GitHubRateLimitEnabled && rates != nil { + ch <- prometheus.MustNewConstMetric(e.APIMetrics["Limit"], prometheus.GaugeValue, rates.Limit) + ch <- prometheus.MustNewConstMetric(e.APIMetrics["Remaining"], prometheus.GaugeValue, rates.Remaining) + ch <- prometheus.MustNewConstMetric(e.APIMetrics["Reset"], prometheus.GaugeValue, rates.Reset) + } return nil } diff --git a/exporter/prometheus.go b/exporter/prometheus.go index 9305e833..a6f95003 100644 --- a/exporter/prometheus.go +++ b/exporter/prometheus.go @@ -28,17 +28,34 @@ func (e *Exporter) Collect(ch chan<- prometheus.Metric) { repoMetrics, err := e.getRepoMetrics(ctx) if err != nil { - log.Errorf("Error fetching repository metrics: %v", err) + log.Errorf("getting repository metrics: %v", err) return } data = append(data, repoMetrics...) + r, err := e.getRateLimits(ctx) + if err != nil { + log.Errorf("getting rate limit metrics: %v", err) + return + } + + // Set prometheus gauge metrics using the data gathered + err = e.processMetrics(data, r, ch) + if err != nil { + log.Errorf("processing metrics: %v", err) + } +} + +func (e *Exporter) getRateLimits(ctx context.Context) (*RateLimits, error) { + if !e.GitHubRateLimitEnabled { + return nil, nil + } + // TODO - get all rate limits, not just core rates, _, err := e.Client.RateLimit.Get(ctx) if err != nil { - log.Errorf("Error fetching rate limits: %v", err) - return + return nil, fmt.Errorf("fetching rate limits: %w", err) } r := &RateLimits{ @@ -47,11 +64,7 @@ func (e *Exporter) Collect(ch chan<- prometheus.Metric) { Reset: float64(rates.Core.Reset.Unix()), } - // Set prometheus gauge metrics using the data gathered - err = e.processMetrics(data, r, ch) - if err != nil { - log.Errorf("Error processing metrics: %v", err) - } + return r, nil } // getRepoMetrics fetches metrics for the configured repositories diff --git a/main.go b/main.go index 5f346e5e..16e5d606 100644 --- a/main.go +++ b/main.go @@ -5,7 +5,6 @@ import ( "github.com/githubexporter/github-exporter/exporter" "github.com/githubexporter/github-exporter/http" - "github.com/google/go-github/v76/github" log "github.com/sirupsen/logrus" ) @@ -17,13 +16,10 @@ func main() { log.Fatalf("Error initializing configuration: %v", err) } - metrics := exporter.AddMetrics() - - exp := exporter.Exporter{ - APIMetrics: metrics, - Config: *applicationCfg, - Client: github.NewClient(nil), + exp, err := exporter.NewExporter(applicationCfg) + if err != nil { + log.Fatalf("Error initializing exporter: %v", err) } - http.NewServer(exp).Start() + http.NewServer(*exp).Start() } diff --git a/test/github_exporter_test.go b/test/github_exporter_test.go index 6d89a609..72e8ca34 100644 --- a/test/github_exporter_test.go +++ b/test/github_exporter_test.go @@ -78,7 +78,7 @@ func TestGithubExporterHttpErrorHandling(t *testing.T) { func apiTest(conf config.Config) (*apitest.APITest, exporter.Exporter) { exp := exporter.Exporter{ - APIMetrics: exporter.AddMetrics(), + APIMetrics: exporter.AddMetrics(&conf), Config: conf, Client: github.NewClient(nil), } From e42039cbecf9c636631be4ab6c985ca884fbe105 Mon Sep 17 00:00:00 2001 From: Henry McConville Date: Sun, 19 Oct 2025 10:27:17 +0100 Subject: [PATCH 07/14] [v2] Fix app auth --- config/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/config.go b/config/config.go index 6337ee8e..206bea67 100644 --- a/config/config.go +++ b/config/config.go @@ -163,7 +163,7 @@ func (c *Config) GetClient() (*github.Client, error) { return nil, fmt.Errorf("creating GitHub App installation transport: %v", err) } - httpClient.Transport = itr + httpClient = &http.Client{Transport: itr} } client := github.NewClient(httpClient) From 22bbfe531411ed3796a4fd1c670af486f42c7e4d Mon Sep 17 00:00:00 2001 From: Henry McConville Date: Mon, 20 Oct 2025 08:47:18 +0100 Subject: [PATCH 08/14] [v2] Add extra rate limit types --- exporter/metrics.go | 17 ++++++++++------- exporter/prometheus.go | 37 +++++++++++++++++++++++++++++------- exporter/structs.go | 5 +++-- test/github_exporter_test.go | 18 +++++++++++++++--- 4 files changed, 58 insertions(+), 19 deletions(-) diff --git a/exporter/metrics.go b/exporter/metrics.go index 0cf16553..a886d465 100644 --- a/exporter/metrics.go +++ b/exporter/metrics.go @@ -64,20 +64,21 @@ func AddMetrics(cfg *config.Config) map[string]*prometheus.Desc { ) if cfg.GitHubRateLimitEnabled { + rateLimitLabels := []string{"resource"} APIMetrics["Limit"] = prometheus.NewDesc( prometheus.BuildFQName("github", "rate", "limit"), "Number of API queries allowed in a 60 minute window", - []string{}, nil, + rateLimitLabels, nil, ) APIMetrics["Remaining"] = prometheus.NewDesc( prometheus.BuildFQName("github", "rate", "remaining"), "Number of API queries remaining in the current window", - []string{}, nil, + rateLimitLabels, nil, ) APIMetrics["Reset"] = prometheus.NewDesc( prometheus.BuildFQName("github", "rate", "reset"), "The time at which the current rate limit window resets in UTC epoch seconds", - []string{}, nil, + rateLimitLabels, nil, ) } @@ -85,7 +86,7 @@ func AddMetrics(cfg *config.Config) map[string]*prometheus.Desc { } // processMetrics - processes the response data and sets the metrics using it as a source -func (e *Exporter) processMetrics(data []*Datum, rates *RateLimits, ch chan<- prometheus.Metric) error { +func (e *Exporter) processMetrics(data []*Datum, rates *[]RateLimit, ch chan<- prometheus.Metric) error { // APIMetrics - range through the data slice for _, x := range data { @@ -112,9 +113,11 @@ func (e *Exporter) processMetrics(data []*Datum, rates *RateLimits, ch chan<- pr // Set Rate limit stats if e.GitHubRateLimitEnabled && rates != nil { - ch <- prometheus.MustNewConstMetric(e.APIMetrics["Limit"], prometheus.GaugeValue, rates.Limit) - ch <- prometheus.MustNewConstMetric(e.APIMetrics["Remaining"], prometheus.GaugeValue, rates.Remaining) - ch <- prometheus.MustNewConstMetric(e.APIMetrics["Reset"], prometheus.GaugeValue, rates.Reset) + for _, r := range *rates { + ch <- prometheus.MustNewConstMetric(e.APIMetrics["Limit"], prometheus.GaugeValue, r.Limit, r.Resource) + ch <- prometheus.MustNewConstMetric(e.APIMetrics["Remaining"], prometheus.GaugeValue, r.Remaining, r.Resource) + ch <- prometheus.MustNewConstMetric(e.APIMetrics["Reset"], prometheus.GaugeValue, r.Reset, r.Resource) + } } return nil diff --git a/exporter/prometheus.go b/exporter/prometheus.go index a6f95003..82e91851 100644 --- a/exporter/prometheus.go +++ b/exporter/prometheus.go @@ -47,24 +47,46 @@ func (e *Exporter) Collect(ch chan<- prometheus.Metric) { } } -func (e *Exporter) getRateLimits(ctx context.Context) (*RateLimits, error) { +func (e *Exporter) getRateLimits(ctx context.Context) (*[]RateLimit, error) { if !e.GitHubRateLimitEnabled { return nil, nil } - // TODO - get all rate limits, not just core rates, _, err := e.Client.RateLimit.Get(ctx) if err != nil { return nil, fmt.Errorf("fetching rate limits: %w", err) } - r := &RateLimits{ - Limit: float64(rates.Core.Limit), - Remaining: float64(rates.Core.Remaining), - Reset: float64(rates.Core.Reset.Unix()), + rateLimits := []*github.Rate{ + rates.ActionsRunnerRegistration, + rates.AuditLog, + rates.CodeScanningUpload, + rates.CodeSearch, + rates.Core, + rates.DependencySnapshots, + rates.GraphQL, + rates.IntegrationManifest, + rates.SCIM, + rates.Search, + rates.SourceImport, } - return r, nil + var rls []RateLimit + + for _, rate := range rateLimits { + if rate == nil { + continue + } + r := RateLimit{ + Resource: rate.Resource, + Limit: float64(rate.Limit), + Remaining: float64(rate.Remaining), + Reset: float64(rate.Reset.Unix()), + } + rls = append(rls, r) + } + + return &rls, nil } // getRepoMetrics fetches metrics for the configured repositories @@ -100,6 +122,7 @@ func (e *Exporter) parseRepo(ctx context.Context, repo github.Repository) (*Datu repoOwner := repo.GetOwner().GetLogin() repoName := repo.GetName() + // TODO - check pagination rel, _, err := e.Client.Repositories.ListReleases(ctx, repoOwner, repoName, nil) if err != nil { return nil, fmt.Errorf("listing releases: %w", err) diff --git a/exporter/structs.go b/exporter/structs.go index 2ea4dab8..5f9c91eb 100644 --- a/exporter/structs.go +++ b/exporter/structs.go @@ -64,9 +64,10 @@ type Asset struct { CreatedAt string `json:"created_at"` } -// RateLimits is used to store rate limit data into a struct +// RateLimit is used to store rate limit data into a struct // This data is later represented as a metric, captured at the end of a scrape -type RateLimits struct { +type RateLimit struct { + Resource string Limit float64 Remaining float64 Reset float64 diff --git a/test/github_exporter_test.go b/test/github_exporter_test.go index 72e8ca34..ad55001b 100644 --- a/test/github_exporter_test.go +++ b/test/github_exporter_test.go @@ -40,9 +40,21 @@ func TestGithubExporter(t *testing.T) { ). Get("/metrics"). Expect(t). - Assert(bodyContains(`github_rate_limit 60`)). - Assert(bodyContains(`github_rate_remaining 60`)). - Assert(bodyContains(`github_rate_reset 3e+09`)). + Assert(bodyContains(`github_rate_limit{resource="code_search"} 60`)). + Assert(bodyContains(`github_rate_limit{resource="core"} 60`)). + Assert(bodyContains(`github_rate_limit{resource="graphql"} 0`)). + Assert(bodyContains(`github_rate_limit{resource="integration_manifest"} 5000`)). + Assert(bodyContains(`github_rate_limit{resource="search"} 10`)). + Assert(bodyContains(`github_rate_remaining{resource="code_search"} 60`)). + Assert(bodyContains(`github_rate_remaining{resource="core"} 60`)). + Assert(bodyContains(`github_rate_remaining{resource="graphql"} 0`)). + Assert(bodyContains(`github_rate_remaining{resource="integration_manifest"} 5000`)). + Assert(bodyContains(`github_rate_remaining{resource="search"} 10`)). + Assert(bodyContains(`github_rate_reset{resource="code_search"} 3e+09`)). + Assert(bodyContains(`github_rate_reset{resource="core"} 3e+09`)). + Assert(bodyContains(`github_rate_reset{resource="graphql"} 3e+09`)). + Assert(bodyContains(`github_rate_reset{resource="integration_manifest"} 3e+09`)). + Assert(bodyContains(`github_rate_reset{resource="search"} 3e+09`)). Assert(bodyContains(`github_repo_forks{archived="false",fork="false",language="Go",license="mit",private="false",repo="myRepo",user="myOrg"} 10`)). Assert(bodyContains(`github_repo_pull_request_count{repo="myRepo",user="myOrg"} 3`)). Assert(bodyContains(`github_repo_open_issues{archived="false",fork="false",language="Go",license="mit",private="false",repo="myRepo",user="myOrg"} 2`)). From f1bfadf47f187dfdf74e683b2d35bc13d4dcfc0d Mon Sep 17 00:00:00 2001 From: Henry McConville Date: Mon, 20 Oct 2025 09:09:01 +0100 Subject: [PATCH 09/14] [v2] Fix app rate limit handling --- config/config.go | 76 ------------------------------------------ exporter/prometheus.go | 28 ++++++++-------- 2 files changed, 14 insertions(+), 90 deletions(-) diff --git a/config/config.go b/config/config.go index 206bea67..8f92ea41 100644 --- a/config/config.go +++ b/config/config.go @@ -1,12 +1,10 @@ package config import ( - "context" "fmt" "net/http" "net/url" "os" - "path" "strings" "github.com/bradleyfalzon/ghinstallation/v2" @@ -65,11 +63,6 @@ func Init() (*Config, error) { return nil, fmt.Errorf("processing GitHub App envconfig: %v", err) } cfg.GitHubAppConfig = &appCfg - token, err := cfg.APITokenFromGitHubApp() - if err != nil { - return nil, fmt.Errorf("generating API token from GitHub App config: %v", err) - } - cfg.GithubToken = token } // Read token from file if not set directly @@ -84,75 +77,6 @@ func Init() (*Config, error) { return &cfg, nil } -func (c *Config) TargetURLs() []string { - - var urls []string - - opts := map[string]string{"per_page": "100"} // Used to set the Github API to return 100 results per page (max) - - if len(c.Repositories) == 0 && len(c.Organisations) == 0 && len(c.Users) == 0 { - log.Info("No targets specified. Only rate limit endpoint will be scraped") - } - - // Append repositories to the array - if len(c.Repositories) > 0 { - for _, x := range c.Repositories { - y := *c.ApiUrl - y.Path = path.Join(y.Path, "repos", x) - q := y.Query() - for k, v := range opts { - q.Add(k, v) - } - y.RawQuery = q.Encode() - urls = append(urls, y.String()) - } - } - - // Append GitHub organisations to the array - if len(c.Organisations) > 0 { - for _, x := range c.Organisations { - y := *c.ApiUrl - y.Path = path.Join(y.Path, "orgs", x, "repos") - q := y.Query() - for k, v := range opts { - q.Add(k, v) - } - y.RawQuery = q.Encode() - urls = append(urls, y.String()) - } - } - - if len(c.Users) > 0 { - for _, x := range c.Users { - y := *c.ApiUrl - y.Path = path.Join(y.Path, "users", x, "repos") - q := y.Query() - for k, v := range opts { - q.Add(k, v) - } - y.RawQuery = q.Encode() - urls = append(urls, y.String()) - } - } - - return urls -} - -// APITokenFromGitHubApp generating api token from github app configuration. -func (c *Config) APITokenFromGitHubApp() (string, error) { - itr, err := ghinstallation.NewKeyFromFile(http.DefaultTransport, c.GitHubAppId, c.GitHubAppInstallationId, c.GitHubAppKeyPath) - if err != nil { - return "", err - } - - strToken, err := itr.Token(context.Background()) - if err != nil { - return "", err - } - - return strToken, nil -} - func (c *Config) GetClient() (*github.Client, error) { var httpClient *http.Client diff --git a/exporter/prometheus.go b/exporter/prometheus.go index 82e91851..19095017 100644 --- a/exporter/prometheus.go +++ b/exporter/prometheus.go @@ -57,28 +57,28 @@ func (e *Exporter) getRateLimits(ctx context.Context) (*[]RateLimit, error) { return nil, fmt.Errorf("fetching rate limits: %w", err) } - rateLimits := []*github.Rate{ - rates.ActionsRunnerRegistration, - rates.AuditLog, - rates.CodeScanningUpload, - rates.CodeSearch, - rates.Core, - rates.DependencySnapshots, - rates.GraphQL, - rates.IntegrationManifest, - rates.SCIM, - rates.Search, - rates.SourceImport, + rateLimits := map[string]*github.Rate{ + "actions_runner_registration": rates.ActionsRunnerRegistration, + "audit_log": rates.AuditLog, + "code_scanning_upload": rates.CodeScanningUpload, + "code_search": rates.CodeSearch, + "core": rates.Core, + "dependency_snapshots": rates.DependencySnapshots, + "graphql": rates.GraphQL, + "integration_manifest": rates.IntegrationManifest, + "scim": rates.SCIM, + "search": rates.Search, + "source_import": rates.SourceImport, } var rls []RateLimit - for _, rate := range rateLimits { + for resource, rate := range rateLimits { if rate == nil { continue } r := RateLimit{ - Resource: rate.Resource, + Resource: resource, Limit: float64(rate.Limit), Remaining: float64(rate.Remaining), Reset: float64(rate.Reset.Unix()), From 83a2b4edc280d5528d247764588c026d235898ce Mon Sep 17 00:00:00 2001 From: Henry McConville Date: Mon, 20 Oct 2025 10:00:22 +0100 Subject: [PATCH 10/14] [v2] Improve README --- METRICS.md | 37 ++++++++++++++++++++++------------ README.md | 46 ++++++++++++++++++++++++++----------------- config/config.go | 1 - config/config_test.go | 3 --- 4 files changed, 52 insertions(+), 35 deletions(-) diff --git a/METRICS.md b/METRICS.md index 60b4c9fa..a0e4a1dd 100644 --- a/METRICS.md +++ b/METRICS.md @@ -5,34 +5,46 @@ Below are an example of the metrics as exposed by this exporter. ``` # HELP github_rate_limit Number of API queries allowed in a 60 minute window # TYPE github_rate_limit gauge -github_rate_limit 5000 +github_rate_limit{resource="code_search"} 60 +github_rate_limit{resource="core"} 60 +github_rate_limit{resource="graphql"} 0 +github_rate_limit{resource="integration_manifest"} 5000 +github_rate_limit{resource="search"} 10 # HELP github_rate_remaining Number of API queries remaining in the current window # TYPE github_rate_remaining gauge -github_rate_remaining 2801 +github_rate_remaining{resource="code_search"} 56 +github_rate_remaining{resource="core"} 56 +github_rate_remaining{resource="graphql"} 0 +github_rate_remaining{resource="integration_manifest"} 5000 +github_rate_remaining{resource="search"} 10 # HELP github_rate_reset The time at which the current rate limit window resets in UTC epoch seconds # TYPE github_rate_reset gauge -github_rate_reset 1.527709029e+09 +github_rate_reset{resource="code_search"} 1.760954346e+09 +github_rate_reset{resource="core"} 1.760954346e+09 +github_rate_reset{resource="graphql"} 1.760954347e+09 +github_rate_reset{resource="integration_manifest"} 1.760954347e+09 +github_rate_reset{resource="search"} 1.760950807e+09 # HELP github_repo_forks Total number of forks for given repository # TYPE github_repo_forks gauge -github_repo_forks{archived="false",fork="false",language="Go",license="mit",private="false",repo="github-exporter",user="infinityworks"} 19 +github_repo_forks{archived="false",fork="false",language="Go",license="mit",private="false",repo="github-exporter",user="githubexporter"} 129 # HELP github_repo_open_issues Total number of open issues for given repository # TYPE github_repo_open_issues gauge -github_repo_open_issues{archived="false",fork="false",language="Go",license="mit",private="false",repo="github-exporter",user="infinityworks"} 7 +github_repo_open_issues{archived="false",fork="false",language="Go",license="mit",private="false",repo="github-exporter",user="githubexporter"} 28 +# HELP github_repo_pull_request_count Total number of pull requests for given repository +# TYPE github_repo_pull_request_count gauge +github_repo_pull_request_count{repo="github-exporter",user="githubexporter"} 7 # HELP github_repo_size_kb Size in KB for given repository # TYPE github_repo_size_kb gauge -github_repo_size_kb{archived="false",fork="false",language="Go",license="mit",private="false",repo="github-exporter",user="infinityworks"} 41 +github_repo_size_kb{archived="false",fork="false",language="Go",license="mit",private="false",repo="github-exporter",user="githubexporter"} 2185 # HELP github_repo_stars Total number of Stars for given repository # TYPE github_repo_stars gauge -github_repo_stars{archived="false",fork="false",language="Go",license="mit",private="false",repo="github-exporter",user="infinityworks"} 64 +github_repo_stars{archived="false",fork="false",language="Go",license="mit",private="false",repo="github-exporter",user="githubexporter"} 441 # HELP github_repo_watchers Total number of watchers/subscribers for given repository # TYPE github_repo_watchers gauge -github_repo_watchers{archived="false",fork="false",language="Go",license="mit",private="false",repo="github-exporter",user="infinityworks"} 10 -# TYPE github_repo_release_downloads gauge -github_repo_release_downloads{name="release1.0.0",repo="github-exporter",user="infinityworks"} 3500 +github_repo_watchers{archived="false",fork="false",language="Go",license="mit",private="false",repo="github-exporter",user="githubexporter"} 10``` ``` + --> \ No newline at end of file diff --git a/README.md b/README.md index 82eb15f4..493770fb 100644 --- a/README.md +++ b/README.md @@ -4,24 +4,34 @@ Exposes basic metrics for your repositories from the GitHub API, to a Prometheus ## Configuration -This exporter is setup to take input from environment variables. All variables are optional: - -* `ORGS` If supplied, the exporter will enumerate all repositories for that organization. Expected in the format "org1, org2". -* `REPOS` If supplied, The repos you wish to monitor, expected in the format "user/repo1, user/repo2". Can be across different Github users/orgs. -* `USERS` If supplied, the exporter will enumerate all repositories for that users. Expected in - the format "user1, user2". -* `GITHUB_TOKEN` If supplied, enables the user to supply a github authentication token that allows the API to be queried more often. Optional, but recommended. -* `GITHUB_TOKEN_FILE` If supplied _instead of_ `GITHUB_TOKEN`, enables the user to supply a path to a file containing a github authentication token that allows the API to be queried more often. Optional, but recommended. -* `GITHUB_APP` If true , authenticates ass GitHub app to the API. -* `GITHUB_APP_ID` The APP ID of the GitHub App. -* `GITHUB_APP_INSTALLATION_ID` The INSTALLATION ID of the GitHub App. -* `GITHUB_APP_KEY_PATH` The path to the github private key. -* `GITHUB_RATE_LIMIT_ENABLED` Whether to fetch Rate Limit metrics. Defaults to `true` -* `API_URL` Github API URL, shouldn't need to change this. Defaults to `https://api.github.com` -* `LISTEN_PORT` The port you wish to run the container on, the Dockerfile defaults this to `9171` -* `METRICS_PATH` the metrics URL path you wish to use, defaults to `/metrics` -* `LOG_LEVEL` The level of logging the exporter will run with, defaults to `info` - +This exporter is configured via environment variables. All variables are optional unless otherwise stated. Below is a list of supported configuration values: + +| Variable | Description | Default | +|-------------------------------|---------------------------------------------------------------------------------------------------------------|------------------------------| +| `ORGS` | Comma-separated list of GitHub organizations to monitor (e.g. `org1,org2`). | | +| `REPOS` | Comma-separated list of repositories to monitor (e.g. `user/repo1,user/repo2`). | | +| `USERS` | Comma-separated list of GitHub users to monitor (e.g. `user1,user2`). | | +| `GITHUB_TOKEN` | GitHub personal access token for API authentication. | | +| `GITHUB_TOKEN_FILE` | Path to a file containing a GitHub personal access token. | | +| `GITHUB_APP` | Set to `true` to authenticate as a GitHub App. | `false` | +| `GITHUB_APP_ID` | The App ID of the GitHub App. Required if `GITHUB_APP` is `true`. | | +| `GITHUB_APP_INSTALLATION_ID` | The Installation ID of the GitHub App. Required if `GITHUB_APP` is `true`. | | +| `GITHUB_APP_KEY_PATH` | Path to the GitHub App private key file. Required if `GITHUB_APP` is `true`. | | +| `GITHUB_RATE_LIMIT_ENABLED` | Whether to fetch GitHub API rate limit metrics (`true` or `false`). | `true` | +| `API_URL` | GitHub API URL. You should not need to change this unless using GitHub Enterprise. | `https://api.github.com` | +| `LISTEN_PORT` | The port the exporter will listen on. | `9171` | +| `METRICS_PATH` | The HTTP path to expose Prometheus metrics. | `/metrics` | +| `LOG_LEVEL` | Logging level (`debug`, `info`, `warn`, `error`). | `info` | + +### Credential Precedence + +When authenticating with the GitHub API, the exporter uses credentials in the following order of precedence: + +1. **GitHub App credentials** (`GITHUB_APP=true` with `GITHUB_APP_ID`, `GITHUB_APP_INSTALLATION_ID`, and `GITHUB_APP_KEY_PATH`): If enabled, the exporter authenticates as a GitHub App and ignores any personal access token or token file. +2. **Token file** (`GITHUB_TOKEN_FILE`): If a token file is provided (and GitHub App is not enabled), the exporter reads the token from the specified file. +3. **Direct token** (`GITHUB_TOKEN`): If neither GitHub App nor token file is provided, the exporter uses the token supplied directly via the environment variable. + +If none of these credentials are provided, the exporter will make unauthenticated requests, which are subject to very strict rate limits. ## Install and deploy diff --git a/config/config.go b/config/config.go index 8f92ea41..94d4c55a 100644 --- a/config/config.go +++ b/config/config.go @@ -18,7 +18,6 @@ type Config struct { MetricsPath string `envconfig:"METRICS_PATH" required:"false" default:"/metrics"` ListenPort string `envconfig:"LISTEN_PORT" required:"false" default:"9171"` LogLevel string `envconfig:"LOG_LEVEL" required:"false" default:"INFO"` - AppName string `envconfig:"APP_NAME" required:"false" default:"github-exporter"` ApiUrl *url.URL `envconfig:"API_URL" required:"false" default:"https://api.github.com"` Repositories []string `envconfig:"REPOS" required:"false"` Organisations []string `envconfig:"ORGS" required:"false"` diff --git a/config/config_test.go b/config/config_test.go index 2cfe2759..ab354500 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -21,7 +21,6 @@ func TestConfig(t *testing.T) { MetricsPath: "/metrics", ListenPort: "9171", LogLevel: "INFO", - AppName: "github-exporter", ApiUrl: &url.URL{ Scheme: "https", Host: "api.github.com", @@ -43,7 +42,6 @@ func TestConfig(t *testing.T) { "METRICS_PATH": "/otherendpoint", "LISTEN_PORT": "1111", "LOG_LEVEL": "DEBUG", - "APP_NAME": "other-app-name", "API_URL": "https://example.com", "REPOS": "repo1, repo2", "ORGS": "org1,org2 ", @@ -55,7 +53,6 @@ func TestConfig(t *testing.T) { MetricsPath: "/otherendpoint", ListenPort: "1111", LogLevel: "DEBUG", - AppName: "other-app-name", ApiUrl: &url.URL{ Scheme: "https", Host: "example.com", From f1e3ef1049fcc56f546eef4ba539c108fc24346d Mon Sep 17 00:00:00 2001 From: Henry McConville Date: Mon, 20 Oct 2025 13:19:47 +0100 Subject: [PATCH 11/14] [v2] Pagination support --- README.md | 33 +++++++++++++++++---------------- config/config.go | 13 +++++++++---- config/config_test.go | 11 +++++++++++ exporter/prometheus.go | 1 - go.mod | 1 + go.sum | 2 ++ 6 files changed, 40 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index 493770fb..4e2d9d4c 100644 --- a/README.md +++ b/README.md @@ -6,22 +6,23 @@ Exposes basic metrics for your repositories from the GitHub API, to a Prometheus This exporter is configured via environment variables. All variables are optional unless otherwise stated. Below is a list of supported configuration values: -| Variable | Description | Default | -|-------------------------------|---------------------------------------------------------------------------------------------------------------|------------------------------| -| `ORGS` | Comma-separated list of GitHub organizations to monitor (e.g. `org1,org2`). | | -| `REPOS` | Comma-separated list of repositories to monitor (e.g. `user/repo1,user/repo2`). | | -| `USERS` | Comma-separated list of GitHub users to monitor (e.g. `user1,user2`). | | -| `GITHUB_TOKEN` | GitHub personal access token for API authentication. | | -| `GITHUB_TOKEN_FILE` | Path to a file containing a GitHub personal access token. | | -| `GITHUB_APP` | Set to `true` to authenticate as a GitHub App. | `false` | -| `GITHUB_APP_ID` | The App ID of the GitHub App. Required if `GITHUB_APP` is `true`. | | -| `GITHUB_APP_INSTALLATION_ID` | The Installation ID of the GitHub App. Required if `GITHUB_APP` is `true`. | | -| `GITHUB_APP_KEY_PATH` | Path to the GitHub App private key file. Required if `GITHUB_APP` is `true`. | | -| `GITHUB_RATE_LIMIT_ENABLED` | Whether to fetch GitHub API rate limit metrics (`true` or `false`). | `true` | -| `API_URL` | GitHub API URL. You should not need to change this unless using GitHub Enterprise. | `https://api.github.com` | -| `LISTEN_PORT` | The port the exporter will listen on. | `9171` | -| `METRICS_PATH` | The HTTP path to expose Prometheus metrics. | `/metrics` | -| `LOG_LEVEL` | Logging level (`debug`, `info`, `warn`, `error`). | `info` | +| Variable | Description | Default | +|------------------------------|------------------------------------------------------------------------------------|--------------------------| +| `ORGS` | Comma-separated list of GitHub organizations to monitor (e.g. `org1,org2`). | | +| `REPOS` | Comma-separated list of repositories to monitor (e.g. `user/repo1,user/repo2`). | | +| `USERS` | Comma-separated list of GitHub users to monitor (e.g. `user1,user2`). | | +| `GITHUB_TOKEN` | GitHub personal access token for API authentication. | | +| `GITHUB_TOKEN_FILE` | Path to a file containing a GitHub personal access token. | | +| `GITHUB_APP` | Set to `true` to authenticate as a GitHub App. | `false` | +| `GITHUB_APP_ID` | The App ID of the GitHub App. Required if `GITHUB_APP` is `true`. | | +| `GITHUB_APP_INSTALLATION_ID` | The Installation ID of the GitHub App. Required if `GITHUB_APP` is `true`. | | +| `GITHUB_APP_KEY_PATH` | Path to the GitHub App private key file. Required if `GITHUB_APP` is `true`. | | +| `GITHUB_RATE_LIMIT_ENABLED` | Whether to fetch GitHub API rate limit metrics (`true` or `false`). | `true` | +| `GITHUB_RESULTS_PER_PAGE` | Number of results to request per page from the GitHub API (max 100). | `100` | +| `API_URL` | GitHub API URL. You should not need to change this unless using GitHub Enterprise. | `https://api.github.com` | +| `LISTEN_PORT` | The port the exporter will listen on. | `9171` | +| `METRICS_PATH` | The HTTP path to expose Prometheus metrics. | `/metrics` | +| `LOG_LEVEL` | Logging level (`debug`, `info`, `warn`, `error`). | `info` | ### Credential Precedence diff --git a/config/config.go b/config/config.go index 94d4c55a..08050d26 100644 --- a/config/config.go +++ b/config/config.go @@ -8,6 +8,7 @@ import ( "strings" "github.com/bradleyfalzon/ghinstallation/v2" + "github.com/gofri/go-github-pagination/githubpagination" "github.com/google/go-github/v76/github" "github.com/kelseyhightower/envconfig" log "github.com/sirupsen/logrus" @@ -22,6 +23,7 @@ type Config struct { Repositories []string `envconfig:"REPOS" required:"false"` Organisations []string `envconfig:"ORGS" required:"false"` Users []string `envconfig:"USERS" required:"false"` + GitHubResultsPerPage int `envconfig:"GITHUB_RESULTS_PER_PAGE" required:"false" default:"100"` GithubToken string `envconfig:"GITHUB_TOKEN" required:"false"` GithubTokenFile string `envconfig:"GITHUB_TOKEN_FILE" required:"false"` GitHubApp bool `envconfig:"GITHUB_APP" required:"false" default:"false"` @@ -77,7 +79,7 @@ func Init() (*Config, error) { } func (c *Config) GetClient() (*github.Client, error) { - var httpClient *http.Client + transport := http.DefaultTransport // Add custom transport for GitHub App authentication if enabled if c.GitHubApp { @@ -85,11 +87,14 @@ func (c *Config) GetClient() (*github.Client, error) { if err != nil { return nil, fmt.Errorf("creating GitHub App installation transport: %v", err) } - - httpClient = &http.Client{Transport: itr} + transport = itr } - client := github.NewClient(httpClient) + paginator := githubpagination.NewClient(transport, + githubpagination.WithPerPage(c.GitHubResultsPerPage), + ) + + client := github.NewClient(paginator) if c.GithubToken != "" { client = client.WithAuthToken(c.GithubToken) diff --git a/config/config_test.go b/config/config_test.go index ab354500..30727986 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -28,6 +28,7 @@ func TestConfig(t *testing.T) { Repositories: []string{}, Organisations: []string{}, Users: []string{}, + GitHubResultsPerPage: 100, GithubToken: "", GithubTokenFile: "", GitHubApp: false, @@ -46,6 +47,7 @@ func TestConfig(t *testing.T) { "REPOS": "repo1, repo2", "ORGS": "org1,org2 ", "USERS": " user1, user2 ", + "GITHUB_RESULTS_PER_PAGE": "50", "GITHUB_TOKEN": "token", "GITHUB_RATE_LIMIT_ENABLED": "false", }, @@ -69,6 +71,7 @@ func TestConfig(t *testing.T) { "user1", "user2", }, + GitHubResultsPerPage: 50, GithubToken: "token", GithubTokenFile: "", GitHubApp: false, @@ -94,6 +97,14 @@ func TestConfig(t *testing.T) { }, expectedErr: errors.New("processing GitHub App envconfig: envconfig.Process: assigning GITHUB_APP_ID to GitHubAppId: converting 'not-an-integer' to type int64. details: strconv.ParseInt: parsing \"not-an-integer\": invalid syntax"), }, + { + name: "invalid log level", + expectedCfg: nil, + envVars: map[string]string{ + "LOG_LEVEL": "boop", + }, + expectedErr: errors.New("parsing log level: not a valid logrus Level: \"boop\""), + }, { name: "github token file not found", envVars: map[string]string{ diff --git a/exporter/prometheus.go b/exporter/prometheus.go index 19095017..34a9a3b4 100644 --- a/exporter/prometheus.go +++ b/exporter/prometheus.go @@ -122,7 +122,6 @@ func (e *Exporter) parseRepo(ctx context.Context, repo github.Repository) (*Datu repoOwner := repo.GetOwner().GetLogin() repoName := repo.GetName() - // TODO - check pagination rel, _, err := e.Client.Repositories.ListReleases(ctx, repoOwner, repoName, nil) if err != nil { return nil, fmt.Errorf("listing releases: %w", err) diff --git a/go.mod b/go.mod index 7666334b..3d761059 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ toolchain go1.25.2 require ( github.com/bradleyfalzon/ghinstallation/v2 v2.17.0 + github.com/gofri/go-github-pagination v1.0.1 github.com/google/go-github/v76 v76.0.0 github.com/kelseyhightower/envconfig v1.4.0 github.com/prometheus/client_golang v1.23.2 diff --git a/go.sum b/go.sum index 8a61eb22..3940cae1 100644 --- a/go.sum +++ b/go.sum @@ -8,6 +8,8 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ3 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/gofri/go-github-pagination v1.0.1 h1:j5uJRx65i/Ta2M0QSgiPcyokY69JnCQglt4n9pspFhY= +github.com/gofri/go-github-pagination v1.0.1/go.mod h1:Qij55Fb4fNPjam3SB+8cLnqp4pgR8RGMyIspYXcyHX0= github.com/golang-jwt/jwt/v4 v4.5.2 h1:YtQM7lnr8iZ+j5q71MGKkNw9Mn7AjHM68uc9g5fXeUI= github.com/golang-jwt/jwt/v4 v4.5.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= From 5bfac650bdebc5d7811139896d46725e9e53aed2 Mon Sep 17 00:00:00 2001 From: Henry McConville Date: Mon, 20 Oct 2025 14:45:03 +0100 Subject: [PATCH 12/14] [v2] Support multiple architectures in Dockerfile --- Dockerfile | 11 +++++++---- go.mod | 2 +- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index 5e1f14fc..3083ed78 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,18 +1,21 @@ -ARG GOLANG_VERSION=1.25.2 -FROM golang:${GOLANG_VERSION} AS build +ARG GOLANG_VERSION=1.25.3 +FROM --platform=$BUILDPLATFORM golang:${GOLANG_VERSION} AS build LABEL maintainer="githubexporter" +ARG TARGETOS +ARG TARGETARCH + COPY ./ /go/src/github.com/githubexporter/github-exporter WORKDIR /go/src/github.com/githubexporter/github-exporter RUN go mod download \ && go test ./... \ - && CGO_ENABLED=0 GOOS=linux go build -o /bin/main + && CGO_ENABLED=0 GOOS=$TARGETOS GOARCH=$TARGETARCH go build -o /bin/main FROM gcr.io/distroless/static AS runtime ADD VERSION . COPY --from=build /bin/main /bin/main ENV LISTEN_PORT=9171 -EXPOSE 9171 +EXPOSE $LISTEN_PORT ENTRYPOINT [ "/bin/main" ] diff --git a/go.mod b/go.mod index 3d761059..ea673588 100644 --- a/go.mod +++ b/go.mod @@ -2,7 +2,7 @@ module github.com/githubexporter/github-exporter go 1.24.0 -toolchain go1.25.2 +toolchain go1.25.3 require ( github.com/bradleyfalzon/ghinstallation/v2 v2.17.0 From 2682008ed48a0b31991aee12946ea53b96dd6825 Mon Sep 17 00:00:00 2001 From: Henry McConville Date: Mon, 20 Oct 2025 15:08:15 +0100 Subject: [PATCH 13/14] [v2] Update version tag --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 3a3cd8cc..359a5b95 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.3.1 +2.0.0 \ No newline at end of file From 1cb399c6e69a6e266ada5d4e0e2591da07571340 Mon Sep 17 00:00:00 2001 From: Henry McConville Date: Mon, 20 Oct 2025 15:19:30 +0100 Subject: [PATCH 14/14] [v2] Add arm64 to docker platform list --- release-version.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release-version.sh b/release-version.sh index bdf98c0d..1aa95c19 100755 --- a/release-version.sh +++ b/release-version.sh @@ -21,4 +21,4 @@ if ! [[ "$version" =~ ^[0-9.]+$ ]]; then exit 1 fi -docker buildx build --platform linux/amd64 -t githubexporter/github-exporter:latest -t githubexporter/github-exporter:$version --push . +docker buildx build --platform linux/amd64,linux/arm64 -t githubexporter/github-exporter:$version --push .