Merge pull request #1 from luissimas/feature/backfill-data

Feature/backfill data

README.md

@@ -1,5 +1,7 @@
 # Zettelkasten exporter
 
+Prometheus exporter that collects statistics from your second brain. 
+
 ## TODO
 
 - [X] Get zettelkasten from directory
@@ -12,13 +14,30 @@
 - [X] Find all files recursivelly
 - [X] Parse markdown links
 - [X] Configurable ignore file patterns
-- [ ] Get zettelkasten from git url
+- [X] Get zettelkasten from git url
+- [X] Register metrics on InfluxDB
+- [X] Make InfluxDB parameters configurable
+- [X] Major refactor
+- [X] Backfill data using git
 - [ ] Support private repositories (Maybe with Github's PAT?)
-- [ ] Exclude links to non existing files
-- [ ] Collect backlinks
-- [ ] Github actions CI
+- [ ] Handle InfluxDB async write errors (https://github.com/influxdata/influxdb-client-go?tab=readme-ov-file#reading-async-errors)
+- [ ] Grafana dashboard
+- [ ] Docker compose example
+- [ ] Kubernetes example
 - [ ] Document usage in README
+- [ ] Github actions CI
 - [ ] Build image and push to OCI registry
 - [ ] Deploy on K8s
+- [ ] Asynchronous git fetching
+
+- [ ] Exclude links to non existing files
+- [ ] Collect backlinks
+- [ ] Collect word count
+- [ ] Collect time to read
 
 https://prometheus.io/docs/instrumenting/writing_exporters/
+https://github.com/go-git/go-git/blob/master/_examples/pull/main.go
+https://medium.com/tlvince/prometheus-backfilling-a92573eb712c
+https://github.com/influxdata/helm-charts/tree/master/charts/influxdb2
+https://grafana.com/docs/grafana/latest/getting-started/get-started-grafana-influxdb/
+https://docs.influxdata.com/flux/v0/get-started/

cmd/zettelkasten-exporter/main.go

@@ -1,64 +1,65 @@
 package main
 
 import (
-	"fmt"
 	"log/slog"
-	"net/http"
 	"os"
 	"time"
 
 	"github.com/luissimas/zettelkasten-exporter/internal/collector"
 	"github.com/luissimas/zettelkasten-exporter/internal/config"
-	"github.com/luissimas/zettelkasten-exporter/internal/metrics"
-	"github.com/luissimas/zettelkasten-exporter/internal/zettel"
-	"github.com/prometheus/client_golang/prometheus/promhttp"
+	"github.com/luissimas/zettelkasten-exporter/internal/storage"
+	"github.com/luissimas/zettelkasten-exporter/internal/zettelkasten"
 )
 
 func main() {
+	// Setup
 	cfg, err := config.LoadConfig()
+	slog.SetLogLoggerLevel(cfg.LogLevel)
 	if err != nil {
 		slog.Error("Error loading config", slog.Any("error", err))
 		os.Exit(1)
 	}
 	slog.Debug("Loaded config", slog.Any("config", cfg))
-	zettelkasten := zettel.NewZettel(cfg)
-	err = zettelkasten.Ensure()
-	if err != nil {
-		slog.Error("Error ensuring that zettelkasten is ready", slog.Any("error", err))
-		os.Exit(1)
+	storage := storage.NewInfluxDBStorage(cfg.InfluxDBURL, cfg.InfluxDBOrg, cfg.InfluxDBBucket, cfg.InfluxDBToken)
+	collector := collector.NewCollector(cfg.IgnoreFiles, storage)
+	var zet zettelkasten.Zettelkasten
+	if cfg.ZettelkastenGitURL != "" {
+		zet = zettelkasten.NewGitZettelkasten(cfg.ZettelkastenGitURL, cfg.ZettelkastenGitBranch)
+	} else {
+		zet = zettelkasten.NewLocalZettelkasten(cfg.ZettelkastenDirectory)
 	}
 
-	collector := collector.NewCollector(zettelkasten.GetRoot(), cfg.IgnoreFiles)
-	promHandler := promhttp.Handler()
-	http.Handle("/metrics", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		slog.Info("Starting metrics collection")
-		started := time.Now()
-		err := zettelkasten.Ensure()
+	// Collect historical data
+	if cfg.CollectHistoricalMetrics {
+		slog.Info("Collecting historical metrics")
+		start := time.Now()
+		err = zet.Ensure()
 		if err != nil {
-			slog.Error("Error ensuring zettelkasten", slog.Any("error", err))
-			metrics.CollectionSuccessful.Set(0)
-		} else {
-			err = collector.CollectMetrics()
-			if err != nil {
-				slog.Error("Error collecting zettelkasten metrics", slog.Any("error", err))
-				metrics.CollectionSuccessful.Set(0)
-			} else {
-				metrics.CollectionSuccessful.Set(1)
-			}
+			slog.Error("Error ensuring that zettelkasten is ready", slog.Any("error", err))
+			os.Exit(1)
 		}
+		slog.Info("Walking zettelkasten history")
+		err = zet.WalkHistory(collector.CollectMetrics)
+		if err != nil {
+			slog.Error("Error walking history", slog.Any("error", err))
+			os.Exit(1)
+		}
+		slog.Info("Collected historical metrics", slog.Duration("duration", time.Since(start)))
+	}
 
-		elapsed := time.Since(started)
-		metrics.CollectionDuration.Observe(float64(elapsed))
-		slog.Info("Completed metrics collection", slog.Duration("duration", elapsed))
-
-		promHandler.ServeHTTP(w, r)
-	}))
-
-	addr := fmt.Sprintf("%s:%d", cfg.IP, cfg.Port)
-	slog.Info("Starting HTTP server", slog.String("address", addr))
-	err = http.ListenAndServe(addr, nil)
-	if err != nil {
-		slog.Error("Error on HTTP server", slog.Any("error", err))
-		os.Exit(1)
+	// Periodic collection loop
+	for {
+		err = zet.Ensure()
+		if err != nil {
+			slog.Error("Error ensuring that zettelkasten is ready", slog.Any("error", err))
+			os.Exit(1)
+		}
+		root := zet.GetRoot()
+		err = collector.CollectMetrics(root, time.Now())
+		if err != nil {
+			slog.Error("Error collecting metrics", slog.Any("error", err))
+			os.Exit(1)
+		}
+		time.Sleep(cfg.CollectionInterval)
 	}
 }