feat: make metrics collection synchronous

Now the metrics collection is only performed when Prometheus scrapes the /metrics endpoint. Reference: https://prometheus.io/docs/instrumenting/writing_exporters/#scheduling
luissimas · May 28, 2024 · c2f5f7c · c2f5f7c
1 parent d815bfb
commit c2f5f7c
Show file tree

Hide file tree

Showing 6 changed files with 62 additions and 75 deletions.
diff --git a/README.md b/README.md
@@ -8,7 +8,6 @@
 - [X] Register total links
 - [X] Register links per note
 - [X] Expose prometheus metrics endpoint
-- [X] Run periodically
 - [X] Read config
 - [ ] Find all files recursivelly
 - [ ] Configurable ignore file patterns
@@ -18,4 +17,4 @@
 - [ ] Get zettelkasten from git url
 - [ ] Support private repositories (Maybe with Github's PAT?)
 
-https://prometheus.io/docs/instrumenting/writing_exporters/#metrics
+https://prometheus.io/docs/instrumenting/writing_exporters/
diff --git a/cmd/zettelkasten-exporter/main.go b/cmd/zettelkasten-exporter/main.go
@@ -5,10 +5,10 @@ import (
 	"log/slog"
 	"net/http"
 	"os"
-	"path/filepath"
 
 	"github.com/luissimas/zettelkasten-exporter/internal/collector"
 	"github.com/luissimas/zettelkasten-exporter/internal/config"
+	"github.com/luissimas/zettelkasten-exporter/internal/metrics"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 )
 
@@ -20,17 +20,29 @@ func main() {
 	}
 	slog.Info("Loaded config", slog.Any("config", cfg))
 
-	absolute_path, err := filepath.Abs(cfg.ZettelkastenDirectory)
-	collector := collector.NewCollector(absolute_path, cfg.ScrapeInterval)
+	collector, err := collector.NewCollector(cfg.ZettelkastenDirectory)
+	if err != nil {
+		slog.Error("Error creating collector", slog.Any("error", err))
+		os.Exit(1)
+	}
 
-	slog.Info("Starting metrics collector")
-	collector.StartCollecting()
-	slog.Info("Metrics collector started")
+	promHandler := promhttp.Handler()
+	http.Handle("/metrics", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		err := collector.CollectMetrics()
+		if err != nil {
+			slog.Error("Error collecting zettelkasten metrics", slog.Any("error", err))
+			metrics.ExporterUp.Set(0)
+		} else {
+			metrics.ExporterUp.Set(1)
+		}
+		promHandler.ServeHTTP(w, r)
+	}))
 
 	addr := fmt.Sprintf("%s:%d", cfg.IP, cfg.Port)
-	http.Handle("/metrics", promhttp.Handler())
 	slog.Info("Starting HTTP server", slog.String("address", addr))
 	err = http.ListenAndServe(addr, nil)
-	slog.Info("Error on HTTP server", slog.Any("error", err))
-	os.Exit(1)
+	if err != nil {
+		slog.Error("Error on HTTP server", slog.Any("error", err))
+		os.Exit(1)
+	}
 }
diff --git a/internal/collector/collector.go b/internal/collector/collector.go
@@ -16,24 +16,47 @@ type Metrics struct {
 }
 
 type CollectorConfig struct {
-	Interval time.Duration
-	Path     string
+	Path string
 }
 
 type Collector struct {
 	config CollectorConfig
 }
 
-func NewCollector(path string, interval time.Duration) Collector {
+func NewCollector(path string) (Collector, error) {
+	absolute_path, err := filepath.Abs(path)
+	if err != nil {
+		return Collector{}, err
+	}
+
 	return Collector{
 		config: CollectorConfig{
-			Path:     path,
-			Interval: interval,
+			Path: absolute_path,
 		},
+	}, nil
+}
+
+func (c *Collector) CollectMetrics() error {
+	started := time.Now()
+	slog.Info("Starting metrics collection")
+
+	collected, err := c.collectMetrics()
+	if err != nil {
+		return err
+	}
+
+	metrics.TotalNoteCount.Set(float64(collected.NoteCount))
+	for name, metric := range collected.Notes {
+		metrics.LinkCount.WithLabelValues(name).Set(float64(metric.LinkCount))
 	}
+
+	elapsed := time.Since(started)
+	metrics.CollectionDuration.Observe(float64(elapsed))
+	slog.Info("Completed metrics collection", slog.Duration("duration", elapsed))
+	return nil
 }
 
-func (c *Collector) CollectMetrics() (Metrics, error) {
+func (c *Collector) collectMetrics() (Metrics, error) {
 	// FIXME: filepath.Glob does not support double star expansion,
 	// so this pattern is not searching recursivelly. We'll need to
 	// walk the filesystem recursivelly.
@@ -61,27 +84,3 @@ func (c *Collector) CollectMetrics() (Metrics, error) {
 
 	return Metrics{NoteCount: noteCount, LinkCount: linkCount, Notes: notes}, nil
 }
-
-func (c *Collector) StartCollecting() {
-	go func() {
-		for {
-			started := time.Now()
-			slog.Info("Starting metrics collection")
-
-			collected, err := c.CollectMetrics()
-			if err != nil {
-				slog.Error("Error collecting note metrics", slog.Any("error", err))
-			}
-
-			metrics.TotalNoteCount.Set(float64(collected.NoteCount))
-			for name, metric := range collected.Notes {
-				metrics.LinkCount.WithLabelValues(name).Set(float64(metric.LinkCount))
-			}
-
-			elapsed := time.Since(started)
-			metrics.CollectionDuration.Observe(float64(elapsed))
-			slog.Info("Completed metrics collection", slog.Duration("duration", elapsed), slog.Time("next_collection", time.Now().Add(c.config.Interval)))
-			time.Sleep(c.config.Interval)
-		}
-	}()
-}
diff --git a/internal/config/config.go b/internal/config/config.go
@@ -3,7 +3,6 @@ package config
 import (
 	"log/slog"
 	"strings"
-	"time"
 
 	"github.com/gookit/validate"
 	"github.com/knadh/koanf"
@@ -12,22 +11,20 @@ import (
 )
 
 type Config struct {
-	IP                    string        `koanf:"ip" validate:"required|ip"`
-	Port                  int           `koanf:"port" validate:"required|uint"`
-	LogLevel              slog.Level    `koanf:"log_level"`
-	ScrapeInterval        time.Duration `koanf:"scrape_interval" validate:"required"`
-	ZettelkastenDirectory string        `koanf:"zettelkasten_directory" validate:"required"`
+	IP                    string     `koanf:"ip" validate:"required|ip"`
+	Port                  int        `koanf:"port" validate:"required|uint"`
+	LogLevel              slog.Level `koanf:"log_level"`
+	ZettelkastenDirectory string     `koanf:"zettelkasten_directory" validate:"required"`
 }
 
 func LoadConfig() (Config, error) {
 	k := koanf.New(".")
 
 	// Set default values
 	k.Load(structs.Provider(Config{
-		IP:             "0.0.0.0",
-		Port:           6969,
-		LogLevel:       slog.LevelInfo,
-		ScrapeInterval: time.Minute * 5,
+		IP:       "0.0.0.0",
+		Port:     6969,
+		LogLevel: slog.LevelInfo,
 	}, "koanf"), nil)
 
 	// Load env variables
@@ -39,14 +36,9 @@ func LoadConfig() (Config, error) {
 
 	// Validate config
 	v := validate.Struct(cfg)
-	v.AddValidator("duration", validateDuration)
 	if !v.Validate() {
 		return Config{}, v.Errors
 	}
 
 	return cfg, nil
 }
-
-func validateDuration(val time.Duration) bool {
-	return val > 0
-}
diff --git a/internal/config/config_test.go b/internal/config/config_test.go
@@ -3,7 +3,6 @@ package config
 import (
 	"log/slog"
 	"testing"
-	"time"
 )
 
 func TestLoadConfig_DefaultValues(t *testing.T) {
@@ -17,7 +16,6 @@ func TestLoadConfig_DefaultValues(t *testing.T) {
 		IP:                    "0.0.0.0",
 		Port:                  6969,
 		LogLevel:              slog.LevelInfo,
-		ScrapeInterval:        time.Minute * 5,
 		ZettelkastenDirectory: "/any/dir",
 	}
 	if c != expected {
@@ -38,7 +36,6 @@ func TestLoadConfig_PartialEnv(t *testing.T) {
 		IP:                    "0.0.0.0",
 		Port:                  4444,
 		LogLevel:              slog.LevelDebug,
-		ScrapeInterval:        time.Minute * 5,
 		ZettelkastenDirectory: "/any/dir",
 	}
 	if c != expected {
@@ -50,7 +47,6 @@ func TestLoadConfig_FullEnv(t *testing.T) {
 	t.Setenv("IP", "127.0.0.1")
 	t.Setenv("PORT", "4444")
 	t.Setenv("LOG_LEVEL", "DEBUG")
-	t.Setenv("SCRAPE_INTERVAL", "5m")
 	t.Setenv("ZETTELKASTEN_DIRECTORY", "/any/dir")
 	c, err := LoadConfig()
 	if err != nil {
@@ -61,7 +57,6 @@ func TestLoadConfig_FullEnv(t *testing.T) {
 		IP:                    "127.0.0.1",
 		Port:                  4444,
 		LogLevel:              slog.LevelDebug,
-		ScrapeInterval:        time.Minute * 5,
 		ZettelkastenDirectory: "/any/dir",
 	}
 	if c != expected {
@@ -91,7 +86,6 @@ func TestLoadConfig_Validate(t *testing.T) {
 				"IP":                     "any-string",
 				"PORT":                   "4444",
 				"LOG_LEVEL":              "INFO",
-				"SCRAPE_INTERVAL":        "5m",
 				"ZETTELKASTEN_DIRECTORY": "/any/dir",
 			},
 		},
@@ -102,18 +96,6 @@ func TestLoadConfig_Validate(t *testing.T) {
 				"IP":                     "0.0.0.0",
 				"PORT":                   "-1",
 				"LOG_LEVEL":              "INFO",
-				"SCRAPE_INTERVAL":        "5m",
-				"ZETTELKASTEN_DIRECTORY": "/any/dir",
-			},
-		},
-		{
-			name:        "invalid interval",
-			shouldError: true,
-			env: map[string]string{
-				"IP":                     "0.0.0.0",
-				"PORT":                   "4444",
-				"LOG_LEVEL":              "INFO",
-				"SCRAPE_INTERVAL":        "5",
 				"ZETTELKASTEN_DIRECTORY": "/any/dir",
 			},
 		},
@@ -124,7 +106,6 @@ func TestLoadConfig_Validate(t *testing.T) {
 				"IP":                     "0.0.0.0",
 				"PORT":                   "4444",
 				"LOG_LEVEL":              "INFO",
-				"SCRAPE_INTERVAL":        "5m",
 				"ZETTELKASTEN_DIRECTORY": "/any/dir",
 			},
 		},

diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go
@@ -6,6 +6,10 @@ import (
 )
 
 var (
+	ExporterUp = promauto.NewGauge(prometheus.GaugeOpts{
+		Name: "zettelkasten_up",
+		Help: "Whether the last zettelkasten scrape was successful",
+	})
 	TotalNoteCount = promauto.NewGauge(prometheus.GaugeOpts{
 		Name: "zettelkasten_total_note_count",
 		Help: "The total count of notes in the zettelkasten",