feat: config option to collect historical metrics

luissimas · Jun 17, 2024 · 1d8180e · 1d8180e
1 parent 953894f
commit 1d8180e
Show file tree

Hide file tree

Showing 6 changed files with 74 additions and 71 deletions.
diff --git a/README.md b/README.md
@@ -18,7 +18,8 @@ Prometheus exporter that collects statistics from your second brain.
 - [X] Register metrics on InfluxDB
 - [X] Make InfluxDB parameters configurable
 - [X] Major refactor
-- [X] Backfill data using git (only if bucket is empty)
+- [X] Backfill data using git
+- [ ] Support private repositories (Maybe with Github's PAT?)
 - [ ] Handle InfluxDB async write errors (https://github.com/influxdata/influxdb-client-go?tab=readme-ov-file#reading-async-errors)
 - [ ] Grafana dashboard
 - [ ] Docker compose example
@@ -29,9 +30,10 @@ Prometheus exporter that collects statistics from your second brain.
 - [ ] Deploy on K8s
 - [ ] Asynchronous git fetching
 
-- [ ] Support private repositories (Maybe with Github's PAT?)
 - [ ] Exclude links to non existing files
 - [ ] Collect backlinks
+- [ ] Collect word count
+- [ ] Collect time to read
 
 https://prometheus.io/docs/instrumenting/writing_exporters/
 https://github.com/go-git/go-git/blob/master/_examples/pull/main.go

diff --git a/cmd/zettelkasten-exporter/main.go b/cmd/zettelkasten-exporter/main.go
@@ -30,8 +30,8 @@ func main() {
 	}
 
 	// Collect historical data
-	if storage.IsEmpty() {
-		slog.Info("Storage is empty, will collect historical metrics")
+	if cfg.CollectHistoricalMetrics {
+		slog.Info("Collecting historical metrics")
 		start := time.Now()
 		err = zet.Ensure()
 		if err != nil {
@@ -44,9 +44,7 @@ func main() {
 			slog.Error("Error walking history", slog.Any("error", err))
 			os.Exit(1)
 		}
-		slog.Info("Collected historic metrics", slog.Duration("duration", time.Since(start)))
-	} else {
-		slog.Info("Storage is not empty, will skip collecting historical metrics")
+		slog.Info("Collected historical metrics", slog.Duration("duration", time.Since(start)))
 	}
 
 	// Periodic collection loop

diff --git a/internal/config/config.go b/internal/config/config.go
@@ -14,27 +14,29 @@ import (
 )
 
 type Config struct {
-	ZettelkastenDirectory string        `koanf:"zettelkasten_directory" validate:"requiredWithout:ZettelkastenGitURL"`
-	ZettelkastenGitURL    string        `koanf:"zettelkasten_git_url" validate:"requiredWithout:ZettelkastenDirectory" validate:"url/isURL"`
-	ZettelkastenGitBranch string        `koanf:"zettelkasten_git_branch"`
-	LogLevel              slog.Level    `koanf:"log_level"`
-	IgnoreFiles           []string      `koanf:"ignore_files"`
-	CollectionInterval    time.Duration `koanf:"collection_interval"`
-	InfluxDBURL           string        `koanf:"influxdb_url" validate:"required|fullUrl"`
-	InfluxDBToken         string        `koanf:"influxdb_token" validate:"required"`
-	InfluxDBOrg           string        `koanf:"influxdb_org" validate:"required"`
-	InfluxDBBucket        string        `koanf:"influxdb_bucket" validate:"required"`
+	ZettelkastenDirectory    string        `koanf:"zettelkasten_directory" validate:"requiredWithout:ZettelkastenGitURL"`
+	ZettelkastenGitURL       string        `koanf:"zettelkasten_git_url" validate:"requiredWithout:ZettelkastenDirectory" validate:"url/isURL"`
+	ZettelkastenGitBranch    string        `koanf:"zettelkasten_git_branch"`
+	LogLevel                 slog.Level    `koanf:"log_level"`
+	IgnoreFiles              []string      `koanf:"ignore_files"`
+	CollectionInterval       time.Duration `koanf:"collection_interval"`
+	CollectHistoricalMetrics bool          `koanf:"collect_historical_metrics"`
+	InfluxDBURL              string        `koanf:"influxdb_url" validate:"required|fullUrl"`
+	InfluxDBToken            string        `koanf:"influxdb_token" validate:"required"`
+	InfluxDBOrg              string        `koanf:"influxdb_org" validate:"required"`
+	InfluxDBBucket           string        `koanf:"influxdb_bucket" validate:"required"`
 }
 
 func LoadConfig() (Config, error) {
 	k := koanf.New(".")
 
 	// Set default values
 	k.Load(structs.Provider(Config{
-		LogLevel:              slog.LevelInfo,
-		IgnoreFiles:           []string{".git", ".obsidian", ".trash", "README.md"},
-		ZettelkastenGitBranch: "main",
-		CollectionInterval:    time.Minute * 5,
+		LogLevel:                 slog.LevelInfo,
+		IgnoreFiles:              []string{".git", ".obsidian", ".trash", "README.md"},
+		ZettelkastenGitBranch:    "main",
+		CollectionInterval:       time.Minute * 5,
+		CollectHistoricalMetrics: true,
 	}, "koanf"), nil)
 
 	// Load env variables

diff --git a/internal/config/config_test.go b/internal/config/config_test.go
@@ -20,15 +20,16 @@ func TestLoadConfig_DefaultValues(t *testing.T) {
 	}
 
 	expected := Config{
-		InfluxDBURL:           "http://localhost:8086",
-		InfluxDBToken:         "any-token",
-		InfluxDBOrg:           "any-org",
-		InfluxDBBucket:        "any-bucket",
-		CollectionInterval:    time.Minute * 5,
-		LogLevel:              slog.LevelInfo,
-		ZettelkastenDirectory: "/any/dir",
-		ZettelkastenGitBranch: "main",
-		IgnoreFiles:           []string{".git", ".obsidian", ".trash", "README.md"},
+		InfluxDBURL:              "http://localhost:8086",
+		InfluxDBToken:            "any-token",
+		InfluxDBOrg:              "any-org",
+		InfluxDBBucket:           "any-bucket",
+		CollectionInterval:       time.Minute * 5,
+		CollectHistoricalMetrics: true,
+		LogLevel:                 slog.LevelInfo,
+		ZettelkastenDirectory:    "/any/dir",
+		ZettelkastenGitBranch:    "main",
+		IgnoreFiles:              []string{".git", ".obsidian", ".trash", "README.md"},
 	}
 	assert.Equal(t, expected, c)
 }
@@ -43,15 +44,16 @@ func TestLoadConfig_PartialEnv(t *testing.T) {
 	c, err := LoadConfig()
 	if assert.NoError(t, err) {
 		expected := Config{
-			InfluxDBURL:           "http://localhost:8086",
-			InfluxDBToken:         "any-token",
-			InfluxDBOrg:           "any-org",
-			InfluxDBBucket:        "any-bucket",
-			CollectionInterval:    time.Minute * 5,
-			LogLevel:              slog.LevelDebug,
-			ZettelkastenDirectory: "/any/dir",
-			ZettelkastenGitBranch: "main",
-			IgnoreFiles:           []string{".git", ".obsidian", ".trash", "README.md"},
+			InfluxDBURL:              "http://localhost:8086",
+			InfluxDBToken:            "any-token",
+			InfluxDBOrg:              "any-org",
+			InfluxDBBucket:           "any-bucket",
+			CollectionInterval:       time.Minute * 5,
+			CollectHistoricalMetrics: true,
+			LogLevel:                 slog.LevelDebug,
+			ZettelkastenDirectory:    "/any/dir",
+			ZettelkastenGitBranch:    "main",
+			IgnoreFiles:              []string{".git", ".obsidian", ".trash", "README.md"},
 		}
 		assert.Equal(t, expected, c)
 	}
@@ -63,21 +65,23 @@ func TestLoadConfig_FullEnvDirectory(t *testing.T) {
 	t.Setenv("INFLUXDB_ORG", "any-org")
 	t.Setenv("INFLUXDB_BUCKET", "any-bucket")
 	t.Setenv("COLLECTION_INTERVAL", "2h")
+	t.Setenv("COLLECT_HISTORICAL_METRICS", "false")
 	t.Setenv("LOG_LEVEL", "WARN")
 	t.Setenv("ZETTELKASTEN_DIRECTORY", "/any/dir")
 	t.Setenv("IGNORE_FILES", ".obsidian,test,/something/another,dir/file.md")
 	c, err := LoadConfig()
 	if assert.NoError(t, err) {
 		expected := Config{
-			InfluxDBURL:           "http://localhost:8086",
-			InfluxDBToken:         "any-token",
-			InfluxDBOrg:           "any-org",
-			InfluxDBBucket:        "any-bucket",
-			CollectionInterval:    time.Hour * 2,
-			LogLevel:              slog.LevelWarn,
-			ZettelkastenDirectory: "/any/dir",
-			ZettelkastenGitBranch: "main",
-			IgnoreFiles:           []string{".obsidian", "test", "/something/another", "dir/file.md"},
+			InfluxDBURL:              "http://localhost:8086",
+			InfluxDBToken:            "any-token",
+			InfluxDBOrg:              "any-org",
+			InfluxDBBucket:           "any-bucket",
+			CollectionInterval:       time.Hour * 2,
+			CollectHistoricalMetrics: false,
+			LogLevel:                 slog.LevelWarn,
+			ZettelkastenDirectory:    "/any/dir",
+			ZettelkastenGitBranch:    "main",
+			IgnoreFiles:              []string{".obsidian", "test", "/something/another", "dir/file.md"},
 		}
 		assert.Equal(t, expected, c)
 	}
@@ -89,21 +93,23 @@ func TestLoadConfig_FullEnvGit(t *testing.T) {
 	t.Setenv("INFLUXDB_ORG", "any-org")
 	t.Setenv("INFLUXDB_BUCKET", "any-bucket")
 	t.Setenv("COLLECTION_INTERVAL", "15m")
+	t.Setenv("COLLECT_HISTORICAL_METRICS", "false")
 	t.Setenv("LOG_LEVEL", "ERROR")
 	t.Setenv("ZETTELKASTEN_GIT_URL", "https://github.com/user/zettel")
 	t.Setenv("IGNORE_FILES", ".obsidian,test,/something/another,dir/file.md")
 	c, err := LoadConfig()
 	if assert.NoError(t, err) {
 		expected := Config{
-			InfluxDBURL:           "http://localhost:8086",
-			InfluxDBToken:         "any-token",
-			InfluxDBOrg:           "any-org",
-			InfluxDBBucket:        "any-bucket",
-			CollectionInterval:    time.Minute * 15,
-			LogLevel:              slog.LevelError,
-			ZettelkastenGitURL:    "https://github.com/user/zettel",
-			ZettelkastenGitBranch: "main",
-			IgnoreFiles:           []string{".obsidian", "test", "/something/another", "dir/file.md"},
+			InfluxDBURL:              "http://localhost:8086",
+			InfluxDBToken:            "any-token",
+			InfluxDBOrg:              "any-org",
+			InfluxDBBucket:           "any-bucket",
+			CollectionInterval:       time.Minute * 15,
+			CollectHistoricalMetrics: false,
+			LogLevel:                 slog.LevelError,
+			ZettelkastenGitURL:       "https://github.com/user/zettel",
+			ZettelkastenGitBranch:    "main",
+			IgnoreFiles:              []string{".obsidian", "test", "/something/another", "dir/file.md"},
 		}
 		assert.Equal(t, expected, c)
 	}
@@ -135,13 +141,15 @@ func TestLoadConfig_Validate(t *testing.T) {
 			name:        "valid config",
 			shouldError: false,
 			env: map[string]string{
-				"LOG_LEVEL":               "INFO",
-				"ZETTELKASTEN_GIT_URL":    "any-url",
-				"ZETTELKASTEN_GIT_BRANCH": "any-branch",
-				"INFLUXDB_URL":            "http://localhost:8086",
-				"INFLUXDB_TOKEN":          "any-token",
-				"INFLUXDB_ORG":            "any-org",
-				"INFLUXDB_BUCKET":         "any-bucket",
+				"LOG_LEVEL":                  "INFO",
+				"ZETTELKASTEN_GIT_URL":       "any-url",
+				"ZETTELKASTEN_GIT_BRANCH":    "any-branch",
+				"COLLECTION_INTERVAL":        "15m",
+				"COLLECT_HISTORICAL_METRICS": "false",
+				"INFLUXDB_URL":               "http://localhost:8086",
+				"INFLUXDB_TOKEN":             "any-token",
+				"INFLUXDB_ORG":               "any-org",
+				"INFLUXDB_BUCKET":            "any-bucket",
 			},
 		},
 	}

diff --git a/internal/storage/influxdb.go b/internal/storage/influxdb.go
@@ -34,8 +34,3 @@ func (i InfluxDBStorage) WriteMetric(noteName string, metric metrics.NoteMetrics
 	)
 	i.writeAPI.WritePoint(point)
 }
-
-func (i InfluxDBStorage) IsEmpty() bool {
-	// TODO: fetch data from InfluxDB
-	return false
-}
diff --git a/internal/storage/storage.go b/internal/storage/storage.go
@@ -10,6 +10,4 @@ import (
 type Storage interface {
 	// WriteMetric writes the note metric to the storage.
 	WriteMetric(noteName string, metric metrics.NoteMetrics, timestamp time.Time)
-	// IsEmpty tells if the storage is empty.
-	IsEmpty() bool
 }