refactor: abstract metrics storage

README.md

@@ -17,7 +17,9 @@ Prometheus exporter that collects statistics from your second brain.
 - [X] Get zettelkasten from git url
 - [X] Register metrics on InfluxDB
 - [X] Make InfluxDB parameters configurable
+- [ ] Major refactor
 - [ ] Backfill data using git (only if bucket is empty)
+- [ ] Handle InfluxDB async write errors (https://github.com/influxdata/influxdb-client-go?tab=readme-ov-file#reading-async-errors)
 - [ ] Grafana dashboard
 - [ ] Docker compose example
 - [ ] Kubernetes example

cmd/zettelkasten-exporter/main.go

@@ -7,7 +7,7 @@ import (
 
 	"github.com/luissimas/zettelkasten-exporter/internal/collector"
 	"github.com/luissimas/zettelkasten-exporter/internal/config"
-	"github.com/luissimas/zettelkasten-exporter/internal/metrics"
+	"github.com/luissimas/zettelkasten-exporter/internal/storage"
 	"github.com/luissimas/zettelkasten-exporter/internal/zettel"
 )
 
@@ -19,23 +19,24 @@ func main() {
 		os.Exit(1)
 	}
 	slog.Debug("Loaded config", slog.Any("config", cfg))
-	metrics.ConnectDatabase(cfg)
+	storage := storage.NewInfluxDBStorage(cfg.InfluxDBURL, cfg.InfluxDBOrg, cfg.InfluxDBBucket, cfg.InfluxDBToken)
+
 	zettelkasten := zettel.NewZettel(cfg)
-	collector := collector.NewCollector(zettelkasten.GetRoot(), cfg.IgnoreFiles)
+	collector := collector.NewCollector(zettelkasten.GetRoot(), cfg.IgnoreFiles, storage)
 	err = zettelkasten.Ensure()
 	if err != nil {
 		slog.Error("Error ensuring that zettelkasten is ready", slog.Any("error", err))
 		os.Exit(1)
 	}
 	// TODO: check for empty bucket
-	slog.Info("Walking history")
-	start := time.Now()
-	err = zettelkasten.WalkHistory(collector.CollectMetrics)
-	if err != nil {
-		slog.Error("Error walking history", slog.Any("error", err))
-		os.Exit(1)
-	}
-	slog.Info("Collected historic metrics", slog.Duration("duration", time.Since(start)))
+	// slog.Info("Walking history")
+	// start := time.Now()
+	// err = zettelkasten.WalkHistory(collector.CollectMetrics)
+	// if err != nil {
+	// 	slog.Error("Error walking history", slog.Any("error", err))
+	// 	os.Exit(1)
+	// }
+	// slog.Info("Collected historic metrics", slog.Duration("duration", time.Since(start)))
 	for {
 		err = zettelkasten.Ensure()
 		if err != nil {

internal/collector/collector.go

@@ -9,29 +9,26 @@ import (
 	"time"
 
 	"github.com/luissimas/zettelkasten-exporter/internal/metrics"
+	"github.com/luissimas/zettelkasten-exporter/internal/storage"
 )
 
-type Metrics struct {
-	NoteCount int
-	LinkCount int
-	Notes     map[string]NoteMetrics
-}
-
 type CollectorConfig struct {
 	FileSystem     fs.FS
 	IgnorePatterns []string
 }
 
 type Collector struct {
-	config CollectorConfig
+	config  CollectorConfig
+	storage storage.Storage
 }
 
-func NewCollector(fileSystem fs.FS, ignorePatterns []string) Collector {
+func NewCollector(fileSystem fs.FS, ignorePatterns []string, storage storage.Storage) Collector {
 	return Collector{
 		config: CollectorConfig{
 			FileSystem:     fileSystem,
 			IgnorePatterns: ignorePatterns,
 		},
+		storage: storage,
 	}
 }
 
@@ -44,17 +41,17 @@ func (c *Collector) CollectMetrics(collectionTime time.Time) error {
 	}
 
 	for name, metric := range collected.Notes {
-		metrics.RegisterNoteMetric(name, metric.LinkCount, collectionTime)
+		c.storage.WriteMetric(name, metric, collectionTime)
 	}
 	slog.Info("Collected metrics", slog.Duration("duration", time.Since(start)))
 
 	return nil
 }
 
-func (c *Collector) collectMetrics() (Metrics, error) {
+func (c *Collector) collectMetrics() (metrics.Metrics, error) {
 	noteCount := 0
 	linkCount := 0
-	notes := make(map[string]NoteMetrics)
+	notes := make(map[string]metrics.NoteMetrics)
 
 	err := fs.WalkDir(c.config.FileSystem, ".", func(path string, dir fs.DirEntry, err error) error {
 		// Skip ignored files or directories
@@ -87,8 +84,8 @@ func (c *Collector) collectMetrics() (Metrics, error) {
 
 	if err != nil {
 		slog.Error("Error getting files", slog.Any("error", err))
-		return Metrics{}, err
+		return metrics.Metrics{}, err
 	}
 
-	return Metrics{NoteCount: noteCount, LinkCount: linkCount, Notes: notes}, nil
+	return metrics.Metrics{NoteCount: noteCount, LinkCount: linkCount, Notes: notes}, nil
 }

internal/collector/collector_test.go

@@ -4,6 +4,8 @@ import (
 	"testing"
 	"testing/fstest"
 
+	"github.com/luissimas/zettelkasten-exporter/internal/metrics"
+	"github.com/luissimas/zettelkasten-exporter/internal/storage"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -47,11 +49,11 @@ Link to [one](./one.md) and also a full link [[./dir1/dir2/three]] and a [[./dir
 		"ignoredir/test.md":     {Data: []byte("Test.md contents")},
 		"zettel/dir1/ignore.md": {Data: []byte("Ignore.md contents")},
 	}
-	c := NewCollector(fs, []string{"ignore.md", "ignoredir"})
-	expected := Metrics{
+	c := NewCollector(fs, []string{"ignore.md", "ignoredir"}, storage.NewFakeStorage())
+	expected := metrics.Metrics{
 		NoteCount: 4,
 		LinkCount: 8,
-		Notes: map[string]NoteMetrics{
+		Notes: map[string]metrics.NoteMetrics{
 			"zettel/one.md": {
 				Links:     map[string]int{"./dir1/two.md": 2},
 				LinkCount: 2,

internal/collector/note.go

@@ -4,6 +4,7 @@ import (
 	"log/slog"
 	"slices"
 
+	"github.com/luissimas/zettelkasten-exporter/internal/metrics"
 	"github.com/yuin/goldmark"
 	"github.com/yuin/goldmark/ast"
 	"github.com/yuin/goldmark/text"
@@ -16,18 +17,13 @@ var md = goldmark.New(
 	),
 )
 
-type NoteMetrics struct {
-	Links     map[string]int
-	LinkCount int
-}
-
-func CollectNoteMetrics(content []byte) NoteMetrics {
+func CollectNoteMetrics(content []byte) metrics.NoteMetrics {
 	links := collectLinks(content)
 	linkCount := 0
 	for _, v := range links {
 		linkCount += v
 	}
-	return NoteMetrics{Links: links, LinkCount: linkCount}
+	return metrics.NoteMetrics{Links: links, LinkCount: linkCount}
 }
 
 func collectLinks(content []byte) map[string]int {

internal/collector/note_test.go

@@ -3,59 +3,60 @@ package collector
 import (
 	"testing"
 
+	"github.com/luissimas/zettelkasten-exporter/internal/metrics"
 	"github.com/stretchr/testify/assert"
 )
 
 func TestCollectNoteMetrics(t *testing.T) {
 	data := []struct {
 		name     string
 		content  string
-		expected NoteMetrics
+		expected metrics.NoteMetrics
 	}{
 		{
 			name:    "empty file",
 			content: "",
-			expected: NoteMetrics{
+			expected: metrics.NoteMetrics{
 				Links:     map[string]int{},
 				LinkCount: 0,
 			},
 		},
 		{
 			name:    "wiki links",
 			content: "[[Link]]aksdjf[[something|another]]\n[[link]]",
-			expected: NoteMetrics{
+			expected: metrics.NoteMetrics{
 				Links:     map[string]int{"Link": 1, "something": 1, "link": 1},
 				LinkCount: 3,
 			},
 		},
 		{
 			name:    "markdown link",
 			content: "[Link](target.md)",
-			expected: NoteMetrics{
+			expected: metrics.NoteMetrics{
 				Links:     map[string]int{"target.md": 1},
 				LinkCount: 1,
 			},
 		},
 		{
 			name:    "mixed links",
 			content: "okok[Link](target.md)\n**ddk**[[linked]]`test`[[another|link]]\n\n[test](yet-another.md)",
-			expected: NoteMetrics{
+			expected: metrics.NoteMetrics{
 				Links:     map[string]int{"target.md": 1, "linked": 1, "another": 1, "yet-another.md": 1},
 				LinkCount: 4,
 			},
 		},
 		{
 			name:    "repeated links",
 			content: "[[target.md|link]]\n[link](target.md)\n[[link]]",
-			expected: NoteMetrics{
+			expected: metrics.NoteMetrics{
 				Links:     map[string]int{"target.md": 2, "link": 1},
 				LinkCount: 3,
 			},
 		},
 		{
 			name:    "ignore embeddedlinks",
 			content: "![[target.png]]\n!()[another.jpeg]\n[[link]]",
-			expected: NoteMetrics{
+			expected: metrics.NoteMetrics{
 				Links:     map[string]int{"link": 1},
 				LinkCount: 1,
 			},

internal/metrics/metrics.go

@@ -1,26 +1,12 @@
 package metrics
 
-import (
-	"time"
-
-	influxdb2 "github.com/influxdata/influxdb-client-go/v2"
-	"github.com/influxdata/influxdb-client-go/v2/api"
-	"github.com/luissimas/zettelkasten-exporter/internal/config"
-)
-
-var influxDB api.WriteAPI
-
-func ConnectDatabase(cfg config.Config) {
-	client := influxdb2.NewClient(cfg.InfluxDBURL, string(cfg.InfluxDBToken))
-	influxDB = client.WriteAPI(cfg.InfluxDBOrg, cfg.InfluxDBBucket)
+type Metrics struct {
+	NoteCount int
+	LinkCount int
+	Notes     map[string]NoteMetrics
 }
 
-func RegisterNoteMetric(name string, linkCount int, timestamp time.Time) {
-	point := influxdb2.NewPoint(
-		name,
-		map[string]string{"name": name},
-		map[string]interface{}{"link_count": linkCount},
-		timestamp,
-	)
-	influxDB.WritePoint(point)
+type NoteMetrics struct {
+	Links     map[string]int
+	LinkCount int
 }

internal/storage/fake.go

@@ -0,0 +1,23 @@
+package storage
+
+import (
+	"time"
+
+	"github.com/luissimas/zettelkasten-exporter/internal/metrics"
+)
+
+// FakeStorage represents a fake implementation of storage to be used in tests.
+type FakeStorage struct{}
+
+// FakeStorage creates a new `FakeStorage`.
+func NewFakeStorage() FakeStorage {
+	return FakeStorage{}
+}
+
+func (f FakeStorage) WriteMetric(noteName string, metric metrics.NoteMetrics, timestamp time.Time) {
+
+}
+
+func (f FakeStorage) IsEmpty() bool {
+	return false
+}

internal/storage/influxdb.go

@@ -0,0 +1,41 @@
+package storage
+
+import (
+	"time"
+
+	influxdb2 "github.com/influxdata/influxdb-client-go/v2"
+	"github.com/influxdata/influxdb-client-go/v2/api"
+	"github.com/luissimas/zettelkasten-exporter/internal/metrics"
+)
+
+// The measurement name to be used for all metrics within the InfluxDB bucket.
+const measurementName = "notes"
+
+// InfluxDBStorage represents the implementation of a metric storage using InfluxDB.
+type InfluxDBStorage struct {
+	writeAPI api.WriteAPI
+	queryAPI api.QueryAPI
+}
+
+// NewInfluxDBStorage creates a new `InfluxDBStorage`.
+func NewInfluxDBStorage(url, org, bucket, token string) InfluxDBStorage {
+	client := influxdb2.NewClient(url, string(token))
+	writeAPI := client.WriteAPI(org, bucket)
+	queryAPI := client.QueryAPI(org)
+	return InfluxDBStorage{writeAPI: writeAPI, queryAPI: queryAPI}
+}
+
+func (i InfluxDBStorage) WriteMetric(noteName string, metric metrics.NoteMetrics, timestamp time.Time) {
+	point := influxdb2.NewPoint(
+		measurementName,
+		map[string]string{"name": noteName},
+		map[string]interface{}{"link_count": metric.LinkCount},
+		timestamp,
+	)
+	i.writeAPI.WritePoint(point)
+}
+
+func (i InfluxDBStorage) IsEmpty() bool {
+	return false
+
+}

internal/storage/storage.go

@@ -0,0 +1,15 @@
+package storage
+
+import (
+	"time"
+
+	"github.com/luissimas/zettelkasten-exporter/internal/metrics"
+)
+
+// Storage represents a storage for metrics.
+type Storage interface {
+	// WriteMetric writes the note metric to the storage.
+	WriteMetric(noteName string, metric metrics.NoteMetrics, timestamp time.Time)
+	// IsEmpty tells if the storage is empty.
+	IsEmpty() bool
+}