Alsways close request bodies and minor changes to new workflow behavior

changelog/unreleased/workflow-created.md

@@ -0,0 +1,10 @@
+Enhancement: New metrics and configs for workflow collector
+
+We have added a new metric for the duration of the time since the workflow run
+was created, defined in minutes. Beside that we have added two additional
+configurations to query the workflows for a specific status and you are able to
+define a different timeframe than 12 hours now. Finally we have also added the
+run ID to the labels.
+
+https://github.com/promhippie/github_exporter/pull/200
+https://github.com/promhippie/github_exporter/pull/214

docs/partials/envvars.md

@@ -70,8 +70,8 @@ GITHUB_EXPORTER_COLLECTOR_WORKFLOWS
 GITHUB_EXPORTER_WORKFLOWS_STATUS
 : Query workflows with specific status
 
-GITHUB_EXPORTER_WORKFLOWS_HISTORY_WINDOW
-: Duration for querying workflows since the time they were created, defaults to `12h0m0s`
+GITHUB_EXPORTER_WORKFLOWS_WINDOW
+: History window for querying workflows, defaults to `12h0m0s`
 
 GITHUB_EXPORTER_COLLECTOR_RUNNERS
 : Enable collector for runners, defaults to `false`

docs/partials/metrics.md

@@ -253,5 +253,8 @@ github_storage_billing_estimated_storage_for_month{<prometheus.ConstrainedLabel
 github_workflow_duration_ms{<prometheus.ConstrainedLabel Value>, <prometheus.ConstrainedLabel Value>, <prometheus.ConstrainedLabel Value>, <prometheus.ConstrainedLabel Value>, <prometheus.ConstrainedLabel Value>, <prometheus.ConstrainedLabel Value>, <prometheus.ConstrainedLabel Value>, <prometheus.ConstrainedLabel Value>, <prometheus.ConstrainedLabel Value>}
 : Duration of workflow runs
 
+github_workflow_duration_run_created_minutes{<prometheus.ConstrainedLabel Value>, <prometheus.ConstrainedLabel Value>, <prometheus.ConstrainedLabel Value>, <prometheus.ConstrainedLabel Value>, <prometheus.ConstrainedLabel Value>, <prometheus.ConstrainedLabel Value>, <prometheus.ConstrainedLabel Value>, <prometheus.ConstrainedLabel Value>, <prometheus.ConstrainedLabel Value>}
+: Duration since the workflow run creation time in minutes
+
 github_workflow_status{<prometheus.ConstrainedLabel Value>, <prometheus.ConstrainedLabel Value>, <prometheus.ConstrainedLabel Value>, <prometheus.ConstrainedLabel Value>, <prometheus.ConstrainedLabel Value>, <prometheus.ConstrainedLabel Value>, <prometheus.ConstrainedLabel Value>, <prometheus.ConstrainedLabel Value>, <prometheus.ConstrainedLabel Value>}
 : Status of workflow runs

pkg/command/command.go

@@ -217,14 +217,14 @@ func RootFlags(cfg *config.Config) []cli.Flag {
 			Value:       "",
 			Usage:       "Query workflows with specific status",
 			EnvVars:     []string{"GITHUB_EXPORTER_WORKFLOWS_STATUS"},
-			Destination: &cfg.Target.WorkflowsCfg.Status,
+			Destination: &cfg.Target.Workflows.Status,
 		},
 		&cli.DurationFlag{
-			Name:        "collector.workflows.history-window",
+			Name:        "collector.workflows.window",
 			Value:       12 * time.Hour,
-			Usage:       "Duration for querying workflows since the time they were created",
-			EnvVars:     []string{"GITHUB_EXPORTER_WORKFLOWS_HISTORY_WINDOW"},
-			Destination: &cfg.Target.WorkflowsCfg.HistoryWindow,
+			Usage:       "History window for querying workflows",
+			EnvVars:     []string{"GITHUB_EXPORTER_WORKFLOWS_WINDOW"},
+			Destination: &cfg.Target.Workflows.Window,
 		},
 		&cli.BoolFlag{
 			Name:        "collector.runners",

pkg/config/config.go

@@ -21,26 +21,26 @@ type Logs struct {
 	Pretty bool
 }
 
-// WorkflowsExporterConfig defines the workflow exporter specific configuration.
-type WorkflowsExporterConfig struct {
-	Status        string
-	HistoryWindow time.Duration
+// Workflows defines the workflow specific configuration.
+type Workflows struct {
+	Status string
+	Window time.Duration
 }
 
 // Target defines the target specific configuration.
 type Target struct {
-	Token        string
-	PrivateKey   string
-	AppID        int64
-	InstallID    int64
-	BaseURL      string
-	Insecure     bool
-	Enterprises  cli.StringSlice
-	Orgs         cli.StringSlice
-	Repos        cli.StringSlice
-	Timeout      time.Duration
-	PerPage      int
-	WorkflowsCfg WorkflowsExporterConfig
+	Token       string
+	PrivateKey  string
+	AppID       int64
+	InstallID   int64
+	BaseURL     string
+	Insecure    bool
+	Enterprises cli.StringSlice
+	Orgs        cli.StringSlice
+	Repos       cli.StringSlice
+	Timeout     time.Duration
+	PerPage     int
+	Workflows   Workflows
 }
 
 // Collector defines the collector specific configuration.

pkg/exporter/admin.go

@@ -378,8 +378,9 @@ func (c *AdminCollector) Collect(ch chan<- prometheus.Metric) {
 	defer cancel()
 
 	now := time.Now()
-	record, _, err := c.client.Admin.GetAdminStats(ctx)
+	record, resp, err := c.client.Admin.GetAdminStats(ctx)
 	c.duration.WithLabelValues("admin").Observe(time.Since(now).Seconds())
+	defer resp.Body.Close()
 
 	if err != nil {
 		level.Error(c.logger).Log(

pkg/exporter/billing.go

@@ -365,7 +365,7 @@ func (c *BillingCollector) getActionBilling() []*actionBilling {
 		}
 
 		record := &github.ActionBilling{}
-		_, err = c.client.Do(ctx, req, record)
+		resp, err := c.client.Do(ctx, req, record)
 
 		if err != nil {
 			level.Error(c.logger).Log(
@@ -379,6 +379,8 @@ func (c *BillingCollector) getActionBilling() []*actionBilling {
 			continue
 		}
 
+		defer resp.Body.Close()
+
 		result = append(result, &actionBilling{
 			Type:          "enterprise",
 			Name:          name,
@@ -387,7 +389,8 @@ func (c *BillingCollector) getActionBilling() []*actionBilling {
 	}
 
 	for _, name := range c.config.Orgs.Value() {
-		record, _, err := c.client.Billing.GetActionsBillingOrg(ctx, name)
+		record, resp, err := c.client.Billing.GetActionsBillingOrg(ctx, name)
+		defer resp.Body.Close()
 
 		if err != nil {
 			level.Error(c.logger).Log(
@@ -443,7 +446,7 @@ func (c *BillingCollector) getPackageBilling() []*packageBilling {
 		}
 
 		record := &github.PackageBilling{}
-		_, err = c.client.Do(ctx, req, record)
+		resp, err := c.client.Do(ctx, req, record)
 
 		if err != nil {
 			level.Error(c.logger).Log(
@@ -457,6 +460,8 @@ func (c *BillingCollector) getPackageBilling() []*packageBilling {
 			continue
 		}
 
+		defer resp.Body.Close()
+
 		result = append(result, &packageBilling{
 			Type:           "enterprise",
 			Name:           name,
@@ -468,7 +473,8 @@ func (c *BillingCollector) getPackageBilling() []*packageBilling {
 		ctx, cancel := context.WithTimeout(context.Background(), c.config.Timeout)
 		defer cancel()
 
-		record, _, err := c.client.Billing.GetPackagesBillingOrg(ctx, name)
+		record, resp, err := c.client.Billing.GetPackagesBillingOrg(ctx, name)
+		defer resp.Body.Close()
 
 		if err != nil {
 			level.Error(c.logger).Log(
@@ -524,7 +530,7 @@ func (c *BillingCollector) getStorageBilling() []*storageBilling {
 		}
 
 		record := &github.StorageBilling{}
-		_, err = c.client.Do(ctx, req, record)
+		resp, err := c.client.Do(ctx, req, record)
 
 		if err != nil {
 			level.Error(c.logger).Log(
@@ -538,6 +544,8 @@ func (c *BillingCollector) getStorageBilling() []*storageBilling {
 			continue
 		}
 
+		defer resp.Body.Close()
+
 		result = append(result, &storageBilling{
 			Type:           "enterprise",
 			Name:           name,
@@ -549,7 +557,8 @@ func (c *BillingCollector) getStorageBilling() []*storageBilling {
 		ctx, cancel := context.WithTimeout(context.Background(), c.config.Timeout)
 		defer cancel()
 
-		record, _, err := c.client.Billing.GetStorageBillingOrg(ctx, name)
+		record, resp, err := c.client.Billing.GetStorageBillingOrg(ctx, name)
+		defer resp.Body.Close()
 
 		if err != nil {
 			level.Error(c.logger).Log(

pkg/exporter/helper.go

@@ -46,6 +46,7 @@ func reposByOwnerAndName(ctx context.Context, client *github.Client, owner, repo
 			)
 
 			if err != nil {
+				resp.Body.Close()
 				return nil, err
 			}
 
@@ -55,9 +56,11 @@ func reposByOwnerAndName(ctx context.Context, client *github.Client, owner, repo
 			)
 
 			if resp.NextPage == 0 {
+				resp.Body.Close()
 				break
 			}
 
+			resp.Body.Close()
 			opts.Page = resp.NextPage
 		}
 

pkg/exporter/org.go

@@ -167,8 +167,9 @@ func (c *OrgCollector) Collect(ch chan<- prometheus.Metric) {
 		defer cancel()
 
 		now := time.Now()
-		record, _, err := c.client.Organizations.Get(ctx, name)
+		record, resp, err := c.client.Organizations.Get(ctx, name)
 		c.duration.WithLabelValues("org").Observe(time.Since(now).Seconds())
+		defer resp.Body.Close()
 
 		if err != nil {
 			level.Error(c.logger).Log(

pkg/exporter/runner.go

@@ -393,6 +393,7 @@ func (c *RunnerCollector) pagedRepoRunners(ctx context.Context, owner, name stri
 		)
 
 		if err != nil {
+			resp.Body.Close()
 			return nil, err
 		}
 
@@ -402,9 +403,11 @@ func (c *RunnerCollector) pagedRepoRunners(ctx context.Context, owner, name stri
 		)
 
 		if resp.NextPage == 0 {
+			resp.Body.Close()
 			break
 		}
 
+		resp.Body.Close()
 		opts.Page = resp.NextPage
 	}
 
@@ -459,6 +462,7 @@ func (c *RunnerCollector) pagedEnterpriseRunners(ctx context.Context, name strin
 		)
 
 		if err != nil {
+			resp.Body.Close()
 			return nil, err
 		}
 
@@ -468,9 +472,11 @@ func (c *RunnerCollector) pagedEnterpriseRunners(ctx context.Context, name strin
 		)
 
 		if resp.NextPage == 0 {
+			resp.Body.Close()
 			break
 		}
 
+		resp.Body.Close()
 		opts.Page = resp.NextPage
 	}
 
@@ -525,6 +531,7 @@ func (c *RunnerCollector) pagedOrgRunners(ctx context.Context, name string) ([]*
 		)
 
 		if err != nil {
+			resp.Body.Close()
 			return nil, err
 		}
 
@@ -534,9 +541,11 @@ func (c *RunnerCollector) pagedOrgRunners(ctx context.Context, name string) ([]*
 		)
 
 		if resp.NextPage == 0 {
+			resp.Body.Close()
 			break
 		}
 
+		resp.Body.Close()
 		opts.Page = resp.NextPage
 	}
 

pkg/exporter/workflow.go

@@ -23,9 +23,9 @@ type WorkflowCollector struct {
 	duration *prometheus.HistogramVec
 	config   config.Target
 
-	Status      *prometheus.Desc
-	Duration    *prometheus.Desc
-	RunCreation *prometheus.Desc
+	Status   *prometheus.Desc
+	Duration *prometheus.Desc
+	Creation *prometheus.Desc
 }
 
 // NewWorkflowCollector returns a new WorkflowCollector.
@@ -54,7 +54,7 @@ func NewWorkflowCollector(logger log.Logger, client *github.Client, failures *pr
 			labels,
 			nil,
 		),
-		RunCreation: prometheus.NewDesc(
+		Creation: prometheus.NewDesc(
 			"github_workflow_duration_run_created_minutes",
 			"Duration since the workflow run creation time in minutes",
 			labels,
@@ -68,14 +68,15 @@ func (c *WorkflowCollector) Metrics() []*prometheus.Desc {
 	return []*prometheus.Desc{
 		c.Status,
 		c.Duration,
+		c.Creation,
 	}
 }
 
 // Describe sends the super-set of all possible descriptors of metrics collected by this Collector.
 func (c *WorkflowCollector) Describe(ch chan<- *prometheus.Desc) {
 	ch <- c.Status
 	ch <- c.Duration
-	ch <- c.RunCreation
+	ch <- c.Creation
 }
 
 // Collect is called by the Prometheus registry when collecting metrics.
@@ -138,7 +139,7 @@ func (c *WorkflowCollector) Collect(ch chan<- prometheus.Metric) {
 		)
 
 		ch <- prometheus.MustNewConstMetric(
-			c.RunCreation,
+			c.Creation,
 			prometheus.GaugeValue,
 			time.Since(record.GetRunStartedAt().Time).Minutes(),
 			labels...,
@@ -224,12 +225,12 @@ func (c *WorkflowCollector) repoWorkflows() []*github.WorkflowRun {
 
 func (c *WorkflowCollector) pagedRepoWorkflows(ctx context.Context, owner, name string) ([]*github.WorkflowRun, error) {
 	startWindow := time.Now().Add(
-		-c.config.WorkflowsCfg.HistoryWindow,
+		-c.config.Workflows.Window,
 	).Format(time.RFC3339)
 
 	opts := &github.ListWorkflowRunsOptions{
 		Created: fmt.Sprintf(">=%s", startWindow),
-		Status:  c.config.WorkflowsCfg.Status,
+		Status:  c.config.Workflows.Status,
 		ListOptions: github.ListOptions{
 			PerPage: c.config.PerPage,
 		},
@@ -248,6 +249,7 @@ func (c *WorkflowCollector) pagedRepoWorkflows(ctx context.Context, owner, name
 		)
 
 		if err != nil {
+			resp.Body.Close()
 			return nil, err
 		}
 
@@ -257,9 +259,11 @@ func (c *WorkflowCollector) pagedRepoWorkflows(ctx context.Context, owner, name
 		)
 
 		if resp.NextPage == 0 {
+			resp.Body.Close()
 			break
 		}
 
+		resp.Body.Close()
 		opts.Page = resp.NextPage
 	}