diff --git a/README.md b/README.md index 4e2d9d4..7e3a4ef 100644 --- a/README.md +++ b/README.md @@ -6,23 +6,28 @@ Exposes basic metrics for your repositories from the GitHub API, to a Prometheus This exporter is configured via environment variables. All variables are optional unless otherwise stated. Below is a list of supported configuration values: -| Variable | Description | Default | -|------------------------------|------------------------------------------------------------------------------------|--------------------------| -| `ORGS` | Comma-separated list of GitHub organizations to monitor (e.g. `org1,org2`). | | -| `REPOS` | Comma-separated list of repositories to monitor (e.g. `user/repo1,user/repo2`). | | -| `USERS` | Comma-separated list of GitHub users to monitor (e.g. `user1,user2`). | | -| `GITHUB_TOKEN` | GitHub personal access token for API authentication. | | -| `GITHUB_TOKEN_FILE` | Path to a file containing a GitHub personal access token. | | -| `GITHUB_APP` | Set to `true` to authenticate as a GitHub App. | `false` | -| `GITHUB_APP_ID` | The App ID of the GitHub App. Required if `GITHUB_APP` is `true`. | | -| `GITHUB_APP_INSTALLATION_ID` | The Installation ID of the GitHub App. Required if `GITHUB_APP` is `true`. | | -| `GITHUB_APP_KEY_PATH` | Path to the GitHub App private key file. Required if `GITHUB_APP` is `true`. | | -| `GITHUB_RATE_LIMIT_ENABLED` | Whether to fetch GitHub API rate limit metrics (`true` or `false`). | `true` | -| `GITHUB_RESULTS_PER_PAGE` | Number of results to request per page from the GitHub API (max 100). | `100` | -| `API_URL` | GitHub API URL. You should not need to change this unless using GitHub Enterprise. | `https://api.github.com` | -| `LISTEN_PORT` | The port the exporter will listen on. | `9171` | -| `METRICS_PATH` | The HTTP path to expose Prometheus metrics. | `/metrics` | -| `LOG_LEVEL` | Logging level (`debug`, `info`, `warn`, `error`). | `info` | +| Variable | Description | Default | +|-------------------------------|------------------------------------------------------------------------------------|--------------------------| +| `ORGS` | Comma-separated list of GitHub organizations to monitor (e.g. `org1,org2`). | | +| `REPOS` | Comma-separated list of repositories to monitor (e.g. `user/repo1,user/repo2`). | | +| `USERS` | Comma-separated list of GitHub users to monitor (e.g. `user1,user2`). | | +| `GITHUB_TOKEN` | GitHub personal access token for API authentication. | | +| `GITHUB_TOKEN_FILE` | Path to a file containing a GitHub personal access token. | | +| `GITHUB_APP` | Set to `true` to authenticate as a GitHub App. | `false` | +| `GITHUB_APP_ID` | The App ID of the GitHub App. Required if `GITHUB_APP` is `true`. | | +| `GITHUB_APP_INSTALLATION_ID` | The Installation ID of the GitHub App. Required if `GITHUB_APP` is `true`. | | +| `GITHUB_APP_KEY_PATH` | Path to the GitHub App private key file. Required if `GITHUB_APP` is `true`. | | +| `GITHUB_RATE_LIMIT_ENABLED` | Whether to fetch GitHub API rate limit metrics (`true` or `false`). | `true` | +| `GITHUB_RESULTS_PER_PAGE` | Number of results to request per page from the GitHub API (max 100). | `100` | +| `FETCH_REPO_RELEASES_ENABLED` | Whether to fetch repository release metrics (`true` or `false`). | `true` | +| `FETCH_ORGS_CONCURRENCY` | Number of concurrent requests to make when fetching organization data. | `1` | +| `FETCH_ORG_REPOS_CONCURRENCY` | Number of concurrent requests to make when fetching organization repository data. | `1` | +| `FETCH_USERS_CONCURRENCY` | Number of concurrent requests to make when fetching user data. | `1` | +| `FETCH_USERS_CONCURRENCY` | Number of concurrent requests to make when fetching repository data. | `1` | +| `API_URL` | GitHub API URL. You should not need to change this unless using GitHub Enterprise. | `https://api.github.com` | +| `LISTEN_PORT` | The port the exporter will listen on. | `9171` | +| `METRICS_PATH` | The HTTP path to expose Prometheus metrics. | `/metrics` | +| `LOG_LEVEL` | Logging level (`debug`, `info`, `warn`, `error`). | `info` | ### Credential Precedence diff --git a/VERSION b/VERSION index b02d37b..1defe53 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -v2.0.2 \ No newline at end of file +v2.1.0 diff --git a/config/config.go b/config/config.go index 08050d2..7fa5cb0 100644 --- a/config/config.go +++ b/config/config.go @@ -16,19 +16,24 @@ import ( // Config struct holds runtime configuration required for the application type Config struct { - MetricsPath string `envconfig:"METRICS_PATH" required:"false" default:"/metrics"` - ListenPort string `envconfig:"LISTEN_PORT" required:"false" default:"9171"` - LogLevel string `envconfig:"LOG_LEVEL" required:"false" default:"INFO"` - ApiUrl *url.URL `envconfig:"API_URL" required:"false" default:"https://api.github.com"` - Repositories []string `envconfig:"REPOS" required:"false"` - Organisations []string `envconfig:"ORGS" required:"false"` - Users []string `envconfig:"USERS" required:"false"` - GitHubResultsPerPage int `envconfig:"GITHUB_RESULTS_PER_PAGE" required:"false" default:"100"` - GithubToken string `envconfig:"GITHUB_TOKEN" required:"false"` - GithubTokenFile string `envconfig:"GITHUB_TOKEN_FILE" required:"false"` - GitHubApp bool `envconfig:"GITHUB_APP" required:"false" default:"false"` - GitHubRateLimitEnabled bool `envconfig:"GITHUB_RATE_LIMIT_ENABLED" required:"false" default:"true"` - *GitHubAppConfig `ignored:"true"` + MetricsPath string `envconfig:"METRICS_PATH" required:"false" default:"/metrics"` + ListenPort string `envconfig:"LISTEN_PORT" required:"false" default:"9171"` + LogLevel string `envconfig:"LOG_LEVEL" required:"false" default:"INFO"` + ApiUrl *url.URL `envconfig:"API_URL" required:"false" default:"https://api.github.com"` + Repositories []string `envconfig:"REPOS" required:"false"` + Organisations []string `envconfig:"ORGS" required:"false"` + Users []string `envconfig:"USERS" required:"false"` + GitHubResultsPerPage int `envconfig:"GITHUB_RESULTS_PER_PAGE" required:"false" default:"100"` + GithubToken string `envconfig:"GITHUB_TOKEN" required:"false"` + GithubTokenFile string `envconfig:"GITHUB_TOKEN_FILE" required:"false"` + GitHubApp bool `envconfig:"GITHUB_APP" required:"false" default:"false"` + GitHubRateLimitEnabled bool `envconfig:"GITHUB_RATE_LIMIT_ENABLED" required:"false" default:"true"` + FetchRepoReleasesEnabled bool `envconfig:"FETCH_REPO_RELEASES_ENABLED" required:"false" default:"true"` + FetchOrgsConcurrency int `envconfig:"FETCH_ORGS_CONCURRENCY" required:"false" default:"1"` + FetchOrgReposConcurrency int `envconfig:"FETCH_ORG_REPOS_CONCURRENCY" required:"false" default:"1"` + FetchReposConcurrency int `envconfig:"FETCH_REPOS_CONCURRENCY" required:"false" default:"1"` + FetchUsersConcurrency int `envconfig:"FETCH_USERS_CONCURRENCY" required:"false" default:"1"` + *GitHubAppConfig `ignored:"true"` } type GitHubAppConfig struct { diff --git a/config/config_test.go b/config/config_test.go index 3072798..d77f346 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -25,31 +25,41 @@ func TestConfig(t *testing.T) { Scheme: "https", Host: "api.github.com", }, - Repositories: []string{}, - Organisations: []string{}, - Users: []string{}, - GitHubResultsPerPage: 100, - GithubToken: "", - GithubTokenFile: "", - GitHubApp: false, - GitHubAppConfig: nil, - GitHubRateLimitEnabled: true, + Repositories: []string{}, + Organisations: []string{}, + Users: []string{}, + GitHubResultsPerPage: 100, + GithubToken: "", + GithubTokenFile: "", + GitHubApp: false, + GitHubAppConfig: nil, + GitHubRateLimitEnabled: true, + FetchRepoReleasesEnabled: true, + FetchOrgsConcurrency: 1, + FetchOrgReposConcurrency: 1, + FetchReposConcurrency: 1, + FetchUsersConcurrency: 1, }, expectedErr: nil, }, { name: "non-default config", envVars: map[string]string{ - "METRICS_PATH": "/otherendpoint", - "LISTEN_PORT": "1111", - "LOG_LEVEL": "DEBUG", - "API_URL": "https://example.com", - "REPOS": "repo1, repo2", - "ORGS": "org1,org2 ", - "USERS": " user1, user2 ", - "GITHUB_RESULTS_PER_PAGE": "50", - "GITHUB_TOKEN": "token", - "GITHUB_RATE_LIMIT_ENABLED": "false", + "METRICS_PATH": "/otherendpoint", + "LISTEN_PORT": "1111", + "LOG_LEVEL": "DEBUG", + "API_URL": "https://example.com", + "REPOS": "repo1, repo2", + "ORGS": "org1,org2 ", + "USERS": " user1, user2 ", + "GITHUB_RESULTS_PER_PAGE": "50", + "GITHUB_TOKEN": "token", + "GITHUB_RATE_LIMIT_ENABLED": "false", + "FETCH_REPO_RELEASES_ENABLED": "false", + "FETCH_ORGS_CONCURRENCY": "2", + "FETCH_ORG_REPOS_CONCURRENCY": "3", + "FETCH_REPOS_CONCURRENCY": "4", + "FETCH_USERS_CONCURRENCY": "5", }, expectedCfg: &Config{ MetricsPath: "/otherendpoint", @@ -71,12 +81,17 @@ func TestConfig(t *testing.T) { "user1", "user2", }, - GitHubResultsPerPage: 50, - GithubToken: "token", - GithubTokenFile: "", - GitHubApp: false, - GitHubAppConfig: nil, - GitHubRateLimitEnabled: false, + GitHubResultsPerPage: 50, + GithubToken: "token", + GithubTokenFile: "", + GitHubApp: false, + GitHubAppConfig: nil, + GitHubRateLimitEnabled: false, + FetchRepoReleasesEnabled: false, + FetchOrgsConcurrency: 2, + FetchOrgReposConcurrency: 3, + FetchReposConcurrency: 4, + FetchUsersConcurrency: 5, }, expectedErr: nil, }, diff --git a/exporter/prometheus.go b/exporter/prometheus.go index a124775..d0229c4 100644 --- a/exporter/prometheus.go +++ b/exporter/prometheus.go @@ -9,6 +9,7 @@ import ( "github.com/google/go-github/v76/github" "github.com/prometheus/client_golang/prometheus" log "github.com/sirupsen/logrus" + "golang.org/x/sync/errgroup" ) // Describe - loops through the API metrics and passes them to prometheus.Describe @@ -68,6 +69,7 @@ func (e *Exporter) getRateLimits(ctx context.Context) (*[]RateLimit, error) { return nil, nil } + log.Debug("fetching rate limits") rates, _, err := e.Client.RateLimit.Get(ctx) if err != nil { return nil, fmt.Errorf("fetching rate limits: %w", err) @@ -108,27 +110,37 @@ func (e *Exporter) getRateLimits(ctx context.Context) (*[]RateLimit, error) { // getRepoMetrics fetches metrics for the configured repositories func (e *Exporter) getRepoMetrics(ctx context.Context) ([]*Datum, error) { var data []*Datum + + gRepos, rCtx := errgroup.WithContext(ctx) + gRepos.SetLimit(e.FetchReposConcurrency) + for _, m := range e.Repositories { - // Split the repository string into owner and name - parts := strings.Split(m, "/") - if len(parts) != 2 { - log.Errorf("Invalid repository format: %s", m) - continue - } + gRepos.Go(func() error { + // Split the repository string into owner and name + parts := strings.Split(m, "/") + if len(parts) != 2 { + return fmt.Errorf("invalid repository format: %s", m) + } - repo, _, err := e.Client.Repositories.Get(ctx, parts[0], parts[1]) - if err != nil { - log.Errorf("Error fetching repository data: %v", err) - continue - } + log.Debugf("Fetching data for repository: %s", m) + repo, _, err := e.Client.Repositories.Get(rCtx, parts[0], parts[1]) + if err != nil { + return fmt.Errorf("fetching repository data: %v", err) + } - d, err := e.parseRepo(ctx, *repo) - if err != nil { - log.Errorf("Error parsing repository data: %v", err) - continue - } + d, err := e.parseRepo(ctx, *repo) + if err != nil { + return fmt.Errorf("parsing repository data: %v", err) + } - data = append(data, d) + data = append(data, d) + return nil + }) + } + + err := gRepos.Wait() + if err != nil { + return nil, fmt.Errorf("processing repositories: %v", err) } return data, nil @@ -137,86 +149,124 @@ func (e *Exporter) getRepoMetrics(ctx context.Context) ([]*Datum, error) { // getUserMetrics fetches metrics for the configured users func (e *Exporter) getUserMetrics(ctx context.Context) ([]*Datum, error) { var data []*Datum - for _, m := range e.Users { - repos, _, err := e.Client.Repositories.ListByUser(ctx, m, nil) - if err != nil { - log.Errorf("Error fetching user data: %v", err) - continue - } - for _, repo := range repos { - d, err := e.parseRepo(ctx, *repo) + gUsers, uCtx := errgroup.WithContext(ctx) + gUsers.SetLimit(e.FetchUsersConcurrency) + + for _, m := range e.Users { + gUsers.Go(func() error { + log.Debugf("Fetching data for user: %s", m) + repos, _, err := e.Client.Repositories.ListByUser(uCtx, m, nil) if err != nil { - log.Errorf("Error parsing user repository data: %v", err) - continue + return fmt.Errorf("fetching user data: %v", err) } - data = append(data, d) - } + for _, repo := range repos { + d, err := e.parseRepo(ctx, *repo) + if err != nil { + return fmt.Errorf("parsing user repository data: %v", err) + } + data = append(data, d) + } + return nil + }) } + + err := gUsers.Wait() + if err != nil { + return nil, fmt.Errorf("processing users: %v", err) + } + return data, nil } // getOrgMetrics fetches metrics for the configured organisations func (e *Exporter) getOrgMetrics(ctx context.Context) ([]*Datum, error) { var data []*Datum + + gOrgs, oCtx := errgroup.WithContext(ctx) + gOrgs.SetLimit(e.FetchOrgsConcurrency) + for _, m := range e.Organisations { - repos, _, err := e.Client.Repositories.ListByOrg(ctx, m, nil) - if err != nil { - log.Errorf("Error fetching organisation data: %v", err) - continue - } + gOrgs.Go(func() error { + log.Debugf("Fetching data for organisation: %s", m) + repos, _, err := e.Client.Repositories.ListByOrg(oCtx, m, nil) + if err != nil { + return fmt.Errorf("fetching organisation data: %v", err) + } - for _, repo := range repos { - d, err := e.parseRepo(ctx, *repo) + gRepos, rCtx := errgroup.WithContext(ctx) + gRepos.SetLimit(e.FetchOrgReposConcurrency) + + for _, repo := range repos { + gRepos.Go(func() error { + d, err := e.parseRepo(rCtx, *repo) + if err != nil { + return fmt.Errorf("parsing organisation repository data: %w", err) + } + + data = append(data, d) + return nil + }) + } + + err = gRepos.Wait() if err != nil { - log.Errorf("Error parsing organisation repository data: %v", err) - continue + return fmt.Errorf("processing organisation repositories: %v", err) } + return nil + }) + } - data = append(data, d) - } + err := gOrgs.Wait() + if err != nil { + return nil, fmt.Errorf("processing organisations: %v", err) } return data, nil } func (e *Exporter) parseRepo(ctx context.Context, repo github.Repository) (*Datum, error) { + var releases []Release + var pulls []Pull + repoOwner := repo.GetOwner().GetLogin() repoName := repo.GetName() - rel, _, err := e.Client.Repositories.ListReleases(ctx, repoOwner, repoName, nil) - if err != nil { - return nil, fmt.Errorf("listing releases: %w", err) - } + if e.FetchRepoReleasesEnabled { + log.Debugf("Fetching releases for repository: %s/%s", repoOwner, repoName) + rel, _, err := e.Client.Repositories.ListReleases(ctx, repoOwner, repoName, nil) + if err != nil { + return nil, fmt.Errorf("listing releases: %w", err) + } - var releases []Release - for _, release := range rel { - var assets []Asset - for _, asset := range release.Assets { - a := Asset{ - Name: asset.GetName(), - Size: asset.GetSize(), - Downloads: asset.GetDownloadCount(), - CreatedAt: asset.GetCreatedAt().Format(time.RFC3339), + for _, release := range rel { + var assets []Asset + for _, asset := range release.Assets { + a := Asset{ + Name: asset.GetName(), + Size: asset.GetSize(), + Downloads: asset.GetDownloadCount(), + CreatedAt: asset.GetCreatedAt().Format(time.RFC3339), + } + assets = append(assets, a) } - assets = append(assets, a) - } - r := Release{ - Name: release.GetName(), - Tag: release.GetTagName(), - Assets: assets, + r := Release{ + Name: release.GetName(), + Tag: release.GetTagName(), + Assets: assets, + } + releases = append(releases, r) } - releases = append(releases, r) } + log.Debugf("Fetching pull requests for repository: %s/%s", repoOwner, repoName) pullRequests, _, err := e.Client.PullRequests.List(ctx, repoOwner, repoName, nil) if err != nil { return nil, fmt.Errorf("fetching pull requests: %w", err) } - var pulls []Pull for _, pr := range pullRequests { p := Pull{ Url: pr.GetURL(), diff --git a/go.mod b/go.mod index ea67358..9adf8f4 100644 --- a/go.mod +++ b/go.mod @@ -13,6 +13,7 @@ require ( github.com/sirupsen/logrus v1.9.3 github.com/steinfletcher/apitest v1.3.8 github.com/stretchr/testify v1.11.1 + golang.org/x/sync v0.15.0 ) require ( diff --git a/go.sum b/go.sum index 3940cae..c401789 100644 --- a/go.sum +++ b/go.sum @@ -58,6 +58,8 @@ go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0= go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8= +golang.org/x/sync v0.15.0 h1:KWH3jNZsfyT6xfAfKiz6MRNmd46ByHDYaZ7KSkCtdW8= +golang.org/x/sync v0.15.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.37.0 h1:fdNQudmxPjkdUTPnLn5mdQv7Zwvbvpaxqs831goi9kQ= golang.org/x/sys v0.37.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= diff --git a/test/github_exporter_test.go b/test/github_exporter_test.go index ad55001..98a507f 100644 --- a/test/github_exporter_test.go +++ b/test/github_exporter_test.go @@ -80,7 +80,6 @@ func TestGithubExporterHttpErrorHandling(t *testing.T) { githubRepos(), githubReleases(), githubPullsError(), - githubRateLimit(), ). Get("/metrics"). Expect(t).