Skip to content

Commit

Permalink
feat: fetch zettelkasten from git url
Browse files Browse the repository at this point in the history
  • Loading branch information
luissimas committed Jun 5, 2024
1 parent 1164af5 commit 8800bea
Show file tree
Hide file tree
Showing 9 changed files with 332 additions and 39 deletions.
27 changes: 15 additions & 12 deletions cmd/zettelkasten-exporter/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,11 +5,12 @@ import (
"log/slog"
"net/http"
"os"
"path/filepath"
"time"

"github.com/luissimas/zettelkasten-exporter/internal/collector"
"github.com/luissimas/zettelkasten-exporter/internal/config"
"github.com/luissimas/zettelkasten-exporter/internal/metrics"
"github.com/luissimas/zettelkasten-exporter/internal/zettel"
"github.com/prometheus/client_golang/prometheus/promhttp"
)

Expand All @@ -20,30 +21,32 @@ func main() {
os.Exit(1)
}
slog.Info("Loaded config", slog.Any("config", cfg))

absolute_path, err := filepath.Abs(cfg.ZettelkastenDirectory)
if err != nil {
slog.Error("Error getting absolute path", slog.Any("error", err), slog.String("path", cfg.ZettelkastenDirectory))
os.Exit(1)
}
_, err = os.Stat(absolute_path)
zettelkasten := zettel.NewZettel(cfg)
err = zettelkasten.Ensure()
if err != nil {
slog.Error("Cannot stat zettelkasten directory", slog.Any("error", err), slog.String("path", absolute_path))
slog.Error("Error ensuring that zettelkasten is ready", slog.Any("error", err))
os.Exit(1)
}

fs := os.DirFS(absolute_path)
collector := collector.NewCollector(fs, cfg.IgnoreFiles)

collector := collector.NewCollector(zettelkasten.GetRoot(), cfg.IgnoreFiles)
promHandler := promhttp.Handler()
http.Handle("/metrics", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
started := time.Now()
slog.Info("Starting metrics collection")

zettelkasten.Ensure()
err := collector.CollectMetrics()
if err != nil {
slog.Error("Error collecting zettelkasten metrics", slog.Any("error", err))
metrics.ExporterUp.Set(0)
} else {
metrics.ExporterUp.Set(1)
}

elapsed := time.Since(started)
metrics.CollectionDuration.Observe(float64(elapsed))
slog.Info("Completed metrics collection", slog.Duration("duration", elapsed))

promHandler.ServeHTTP(w, r)
}))

Expand Down
24 changes: 22 additions & 2 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ module github.com/luissimas/zettelkasten-exporter
go 1.22.2

require (
github.com/go-git/go-git/v5 v5.12.0
github.com/gookit/validate v1.5.2
github.com/knadh/koanf v1.5.0
github.com/prometheus/client_golang v1.19.1
Expand All @@ -12,23 +13,42 @@ require (
)

require (
dario.cat/mergo v1.0.0 // indirect
github.com/Microsoft/go-winio v0.6.1 // indirect
github.com/ProtonMail/go-crypto v1.0.0 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/cespare/xxhash/v2 v2.2.0 // indirect
github.com/cloudflare/circl v1.3.7 // indirect
github.com/cyphar/filepath-securejoin v0.2.4 // indirect
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
github.com/emirpasic/gods v1.18.1 // indirect
github.com/fatih/structs v1.1.0 // indirect
github.com/fsnotify/fsnotify v1.7.0 // indirect
github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect
github.com/go-git/go-billy/v5 v5.5.0 // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/gookit/filter v1.2.1 // indirect
github.com/gookit/goutil v0.6.15 // indirect
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
github.com/kevinburke/ssh_config v1.2.0 // indirect
github.com/mitchellh/copystructure v1.2.0 // indirect
github.com/mitchellh/mapstructure v1.5.0 // indirect
github.com/mitchellh/reflectwalk v1.0.2 // indirect
github.com/pjbgf/sha1cd v0.3.0 // indirect
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
github.com/prometheus/client_model v0.5.0 // indirect
github.com/prometheus/common v0.48.0 // indirect
github.com/prometheus/procfs v0.12.0 // indirect
golang.org/x/sys v0.17.0 // indirect
golang.org/x/term v0.17.0 // indirect
github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect
github.com/skeema/knownhosts v1.2.2 // indirect
github.com/xanzy/ssh-agent v0.3.3 // indirect
golang.org/x/crypto v0.21.0 // indirect
golang.org/x/mod v0.12.0 // indirect
golang.org/x/net v0.22.0 // indirect
golang.org/x/sys v0.18.0 // indirect
golang.org/x/text v0.14.0 // indirect
golang.org/x/tools v0.13.0 // indirect
google.golang.org/protobuf v1.33.0 // indirect
gopkg.in/warnings.v0 v0.1.2 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)
107 changes: 100 additions & 7 deletions go.sum

Large diffs are not rendered by default.

7 changes: 0 additions & 7 deletions internal/collector/collector.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@ import (
"log/slog"
"path/filepath"
"slices"
"time"

"github.com/luissimas/zettelkasten-exporter/internal/metrics"
)
Expand Down Expand Up @@ -36,9 +35,6 @@ func NewCollector(fileSystem fs.FS, ignorePatterns []string) Collector {
}

func (c *Collector) CollectMetrics() error {
started := time.Now()
slog.Info("Starting metrics collection")

collected, err := c.collectMetrics()
if err != nil {
return err
Expand All @@ -49,9 +45,6 @@ func (c *Collector) CollectMetrics() error {
metrics.LinkCount.WithLabelValues(name).Set(float64(metric.LinkCount))
}

elapsed := time.Since(started)
metrics.CollectionDuration.Observe(float64(elapsed))
slog.Info("Completed metrics collection", slog.Duration("duration", elapsed))
return nil
}

Expand Down
17 changes: 12 additions & 5 deletions internal/config/config.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package config

import (
"errors"
"log/slog"
"strings"

Expand All @@ -13,7 +14,9 @@ import (
type Config struct {
IP string `koanf:"ip" validate:"required|ip"`
Port int `koanf:"port" validate:"required|uint"`
ZettelkastenDirectory string `koanf:"zettelkasten_directory" validate:"required"`
ZettelkastenDirectory string `koanf:"zettelkasten_directory" validate:"requiredWithout:ZettelkastenGitURL"`
ZettelkastenGitURL string `koanf:"zettelkasten_git_url" validate:"requiredWithout:ZettelkastenDirectory" validate:"url/isURL"`
ZettelkastenGitBranch string `koanf:"zettelkasten_git_branch"`
LogLevel slog.Level `koanf:"log_level"`
IgnoreFiles []string `koanf:"ignore_files"`
}
Expand All @@ -23,10 +26,11 @@ func LoadConfig() (Config, error) {

// Set default values
k.Load(structs.Provider(Config{
IP: "0.0.0.0",
Port: 6969,
LogLevel: slog.LevelInfo,
IgnoreFiles: []string{".git", ".obsidian", ".trash"},
IP: "0.0.0.0",
Port: 6969,
LogLevel: slog.LevelInfo,
IgnoreFiles: []string{".git", ".obsidian", ".trash"},
ZettelkastenGitBranch: "main",
}, "koanf"), nil)

// Load env variables
Expand All @@ -41,6 +45,9 @@ func LoadConfig() (Config, error) {
if !v.Validate() {
return Config{}, v.Errors
}
if cfg.ZettelkastenGitURL != "" && cfg.ZettelkastenDirectory != "" {
return Config{}, errors.New("ZettelkastenGitURL and ZettelkastenDirectory cannot be provided together")
}

return cfg, nil
}
47 changes: 41 additions & 6 deletions internal/config/config_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ func TestLoadConfig_DefaultValues(t *testing.T) {
Port: 6969,
LogLevel: slog.LevelInfo,
ZettelkastenDirectory: "/any/dir",
ZettelkastenGitBranch: "main",
IgnoreFiles: []string{".git", ".obsidian", ".trash"},
}
assert.Equal(t, expected, c)
Expand All @@ -35,13 +36,14 @@ func TestLoadConfig_PartialEnv(t *testing.T) {
Port: 4444,
LogLevel: slog.LevelDebug,
ZettelkastenDirectory: "/any/dir",
ZettelkastenGitBranch: "main",
IgnoreFiles: []string{".git", ".obsidian", ".trash"},
}
assert.Equal(t, expected, c)
}
}

func TestLoadConfig_FullEnv(t *testing.T) {
func TestLoadConfig_FullEnvDirectory(t *testing.T) {
t.Setenv("IP", "127.0.0.1")
t.Setenv("PORT", "4444")
t.Setenv("LOG_LEVEL", "DEBUG")
Expand All @@ -54,6 +56,27 @@ func TestLoadConfig_FullEnv(t *testing.T) {
Port: 4444,
LogLevel: slog.LevelDebug,
ZettelkastenDirectory: "/any/dir",
ZettelkastenGitBranch: "main",
IgnoreFiles: []string{".obsidian", "test", "/something/another", "dir/file.md"},
}
assert.Equal(t, expected, c)
}
}

func TestLoadConfig_FullEnvGit(t *testing.T) {
t.Setenv("IP", "127.0.0.1")
t.Setenv("PORT", "4444")
t.Setenv("LOG_LEVEL", "DEBUG")
t.Setenv("ZETTELKASTEN_GIT_URL", "https://github.com/user/zettel")
t.Setenv("IGNORE_FILES", ".obsidian,test,/something/another,dir/file.md")
c, err := LoadConfig()
if assert.NoError(t, err) {
expected := Config{
IP: "127.0.0.1",
Port: 4444,
LogLevel: slog.LevelDebug,
ZettelkastenGitURL: "https://github.com/user/zettel",
ZettelkastenGitBranch: "main",
IgnoreFiles: []string{".obsidian", "test", "/something/another", "dir/file.md"},
}
assert.Equal(t, expected, c)
Expand All @@ -67,14 +90,25 @@ func TestLoadConfig_Validate(t *testing.T) {
env map[string]string
}{
{
name: "missing directory",
name: "missing source",
shouldError: true,
env: map[string]string{
"IP": "0.0.0.0",
"PORT": "4444",
"LOG_LEVEL": "INFO",
},
},
{
name: "both sources",
shouldError: true,
env: map[string]string{
"IP": "0.0.0.0",
"PORT": "4444",
"LOG_LEVEL": "INFO",
"ZETTELKASTEN_DIRECTORY": "/any/dir",
"ZETTELKASTEN_GIT_URL": "any-string",
},
},
{
name: "invalid ip",
shouldError: true,
Expand All @@ -99,10 +133,11 @@ func TestLoadConfig_Validate(t *testing.T) {
name: "valid config",
shouldError: false,
env: map[string]string{
"IP": "0.0.0.0",
"PORT": "4444",
"LOG_LEVEL": "INFO",
"ZETTELKASTEN_DIRECTORY": "/any/dir",
"IP": "0.0.0.0",
"PORT": "4444",
"LOG_LEVEL": "INFO",
"ZETTELKASTEN_GIT_URL": "any-url",
"ZETTELKASTEN_GIT_BRANCH": "any-branch",
},
},
}
Expand Down
75 changes: 75 additions & 0 deletions internal/zettel/git.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,75 @@
package zettel

import (
"errors"
"io/fs"
"log/slog"
"os"

"github.com/luissimas/zettelkasten-exporter/internal/config"

"github.com/go-git/go-git/v5"
"github.com/go-git/go-git/v5/plumbing"
)

type GitZettel struct {
Config config.Config
RepositoryPath string
}

func NewGitZettel(cfg config.Config) *GitZettel {
return &GitZettel{RepositoryPath: "/tmp/zettelkasten-exporter", Config: cfg}
}

// GetRoot retrieves the root of the zettelkasten git repository
func (g *GitZettel) GetRoot() fs.FS {
return os.DirFS(g.RepositoryPath)
}

// Ensure makes sure that the git repository is valid and updated with the
// latest changes from the remote.
func (g *GitZettel) Ensure() error {
repo, err := git.PlainOpen(g.RepositoryPath)
if errors.Is(err, git.ErrRepositoryNotExists) {
repo, err = cloneGitRepository(g.Config.ZettelkastenGitURL, g.Config.ZettelkastenGitBranch, g.RepositoryPath)
if err != nil {
slog.Error("Unexpected error when cloning git repository", slog.Any("error", err), slog.String("path", g.RepositoryPath))
return err
}
} else if err != nil {
slog.Error("Unexpected error when opening git repository", slog.Any("error", err), slog.String("path", g.RepositoryPath))
return err
}
slog.Info("Git repository open", slog.Any("repo", repo))

w, err := repo.Worktree()
if err != nil {
slog.Error("Unexpected error when getting git repository worktree", slog.Any("error", err), slog.Any("repo", repo))
return err
}

slog.Info("Pulling from repository", slog.Any("repo", repo))
err = w.Pull(&git.PullOptions{RemoteName: "origin"})
if err != nil {
slog.Error("Unexpected error when pulling from git repository", slog.Any("error", err), slog.Any("repo", repo))
return err
}

return nil
}

func cloneGitRepository(url, branch, target string) (*git.Repository, error) {
slog.Info("Cloning git repository", slog.String("url", url), slog.String("branch", branch), slog.String("target", target))
repo, err := git.PlainClone(target, false, &git.CloneOptions{
URL: url,
Depth: 1,
SingleBranch: true,
ReferenceName: plumbing.NewBranchReferenceName(branch),
})
if err != nil {
slog.Error("Could not clone git repository", slog.String("url", url), slog.String("branch", branch), slog.String("target", target))
return nil, err
}
slog.Info("Git repository cloned")
return repo, err
}
44 changes: 44 additions & 0 deletions internal/zettel/local.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
package zettel

import (
"io/fs"
"log/slog"
"os"
"path/filepath"

"github.com/luissimas/zettelkasten-exporter/internal/config"
)

type LocalZettel struct {
Config config.Config
LocalDirectory string
}

// NewLocalZettel creates a new LocalZettel
func NewLocalZettel(cfg config.Config) *LocalZettel {
return &LocalZettel{Config: cfg, LocalDirectory: cfg.ZettelkastenDirectory}
}

// GetRoot retrieves the root of the local zettelkasten directory
func (l *LocalZettel) GetRoot() fs.FS {
return os.DirFS(l.LocalDirectory)
}

// Ensure ensures that the local zettelkasten directory exists and is accessible.
func (l *LocalZettel) Ensure() error {
if !filepath.IsAbs(l.LocalDirectory) {
absolute_path, err := filepath.Abs(l.LocalDirectory)
if err != nil {
slog.Error("Error getting absolute path", slog.Any("error", err), slog.String("path", l.LocalDirectory))
return err
}
l.LocalDirectory = absolute_path
}
_, err := os.Stat(l.LocalDirectory)
if err != nil {
slog.Error("Cannot stat zettelkasten directory", slog.Any("error", err), slog.String("path", l.LocalDirectory))
return err
}

return nil
}
Loading

0 comments on commit 8800bea

Please sign in to comment.