feat: collect historical data from past git commits

cmd/zettelkasten-exporter/main.go

@@ -22,13 +22,27 @@ func main() {
 	metrics.ConnectDatabase(cfg)
 	zettelkasten := zettel.NewZettel(cfg)
 	collector := collector.NewCollector(zettelkasten.GetRoot(), cfg.IgnoreFiles)
+	err = zettelkasten.Ensure()
+	if err != nil {
+		slog.Error("Error ensuring that zettelkasten is ready", slog.Any("error", err))
+		os.Exit(1)
+	}
+	// TODO: check for empty bucket
+	slog.Info("Walking history")
+	start := time.Now()
+	err = zettelkasten.WalkHistory(collector.CollectMetrics)
+	if err != nil {
+		slog.Error("Error walking history", slog.Any("error", err))
+		os.Exit(1)
+	}
+	slog.Info("Collected historic metrics", slog.Duration("duration", time.Since(start)))
 	for {
 		err = zettelkasten.Ensure()
 		if err != nil {
 			slog.Error("Error ensuring that zettelkasten is ready", slog.Any("error", err))
 			os.Exit(1)
 		}
-		err = collector.CollectMetrics()
+		err = collector.CollectMetrics(time.Now())
 		if err != nil {
 			slog.Error("Error collecting metrics", slog.Any("error", err))
 			os.Exit(1)

internal/collector/collector.go

@@ -35,15 +35,14 @@ func NewCollector(fileSystem fs.FS, ignorePatterns []string) Collector {
 	}
 }
 
-func (c *Collector) CollectMetrics() error {
+func (c *Collector) CollectMetrics(collectionTime time.Time) error {
 	slog.Info("Collecting metrics")
 	start := time.Now()
 	collected, err := c.collectMetrics()
 	if err != nil {
 		return err
 	}
 
-	collectionTime := time.Now()
 	for name, metric := range collected.Notes {
 		metrics.RegisterNoteMetric(name, metric.LinkCount, collectionTime)
 	}

internal/collector/note.go

@@ -50,6 +50,7 @@ func collectLinks(content []byte) map[string]int {
 				return ast.WalkContinue, nil
 			}
 
+			// TODO: check if target is not a http link
 			v, ok := links[target]
 			if !ok {
 				links[target] = 0

internal/zettel/git.go

@@ -12,6 +12,7 @@ import (
 
 	"github.com/go-git/go-git/v5"
 	"github.com/go-git/go-git/v5/plumbing"
+	"github.com/go-git/go-git/v5/plumbing/object"
 )
 
 type GitZettel struct {
@@ -43,6 +44,24 @@ func (g *GitZettel) Ensure() error {
 		return err
 	}
 	slog.Debug("Git repository open", slog.String("url", g.Config.ZettelkastenGitURL), slog.String("branch", g.Config.ZettelkastenGitBranch))
+	w, err := repo.Worktree()
+	if err != nil {
+		slog.Error("Unexpected error when getting git repository worktree", slog.Any("error", err))
+		return err
+	}
+	branch, err := repo.Branch(g.Config.ZettelkastenGitBranch)
+	if err != nil {
+		slog.Error("Unexpected error when getting git repository branch", slog.Any("error", err))
+		return err
+	}
+
+	rev, err := repo.ResolveRevision(plumbing.Revision(branch.Name))
+	if err != nil {
+		slog.Error("Unexpected error when getting git repository remote revision", slog.Any("error", err))
+		return err
+	}
+
+	w.Reset(&git.ResetOptions{Commit: *rev})
 
 	slog.Info("Pulling from repository", slog.String("url", g.Config.ZettelkastenGitURL), slog.String("branch", g.Config.ZettelkastenGitBranch))
 	start := time.Now()
@@ -56,6 +75,31 @@ func (g *GitZettel) Ensure() error {
 	return nil
 }
 
+func (g *GitZettel) WalkHistory(walkFunc func(time.Time) error) error {
+	repo, err := git.PlainOpen(g.RepositoryPath)
+	if err != nil {
+		slog.Error("Unexpected error when opening git repository", slog.Any("error", err), slog.String("path", g.RepositoryPath))
+		return err
+	}
+	w, err := repo.Worktree()
+	if err != nil {
+		slog.Error("Unexpected error when getting git repository worktree", slog.Any("error", err))
+		return err
+	}
+	log, err := repo.Log(&git.LogOptions{Order: git.LogOrderCommitterTime})
+	log.ForEach(func(c *object.Commit) error {
+		slog.Debug("Walking commit", slog.String("sha", c.Hash.String()), slog.String("message", c.Message), slog.Time("date", c.Committer.When))
+		w.Reset(&git.ResetOptions{Commit: c.Hash, Mode: git.HardReset})
+		err := walkFunc(c.Committer.When)
+		if err != nil {
+			slog.Error("Error when walking commit", slog.String("hash", c.Hash.String()), slog.Any("error", err))
+			return err
+		}
+		return nil
+	})
+	return nil
+}
+
 func cloneRepository(url, branch, target string) (*git.Repository, error) {
 	slog.Info("Cloning git repository", slog.String("url", url), slog.String("branch", branch), slog.String("target", target))
 	repo, err := git.PlainClone(target, false, &git.CloneOptions{
@@ -81,8 +125,12 @@ func forcePullRepository(repo *git.Repository) error {
 
 	// NOTE: instead of just pulling, we fetch and then hard reset to
 	// account for the case of force pushes to the remote branch
-	err = repo.Fetch(&git.FetchOptions{})
-	if err != nil {
+	err = repo.Fetch(&git.FetchOptions{Depth: 2147483647})
+
+	if errors.Is(err, git.NoErrAlreadyUpToDate) {
+		slog.Info("Already up to date with remote repository, no changes pulled")
+		return nil
+	} else if err != nil {
 		slog.Error("Unexpected error when fetching from git repository", slog.Any("error", err))
 		return err
 	}
@@ -109,11 +157,8 @@ func forcePullRepository(repo *git.Repository) error {
 		Commit: *rev,
 		Mode:   git.HardReset,
 	})
-	if errors.Is(err, git.NoErrAlreadyUpToDate) {
-		slog.Info("Already up to date with remote repository, no changes pulled")
-		return nil
-	} else if err != nil {
-		slog.Error("Unexpected error when pulling from git repository", slog.Any("error", err))
+	if err != nil {
+		slog.Error("Unexpected error when reseting from git repository", slog.Any("error", err))
 		return err
 	}
 	return nil

internal/zettel/zettel.go

@@ -2,22 +2,21 @@ package zettel
 
 import (
 	"io/fs"
+	"time"
 
 	"github.com/luissimas/zettelkasten-exporter/internal/config"
 )
 
 // Zettel represents a zettelkasten
 type Zettel interface {
-	// Ensure makes sure that the zettelkasten is updated and operational
+	// Ensure makes sure that the zettelkasten is updated and operational.
 	Ensure() error
-	// GetRoot retrieves the root of the Zettelkasten directory structure
+	// GetRoot retrieves the root of the Zettelkasten directory structure.
 	GetRoot() fs.FS
+	// WalkHistory walks the history of the Zettelkasten, calling `walkFunc` on each point.
+	WalkHistory(walkFunc func(time.Time) error) error
 }
 
 func NewZettel(cfg config.Config) Zettel {
-	if cfg.ZettelkastenGitURL != "" {
-		return NewGitZettel(cfg)
-	} else {
-		return NewLocalZettel(cfg)
-	}
+	return NewGitZettel(cfg)
 }