Skip to content

Commit

Permalink
feat: add influxdb config
Browse files Browse the repository at this point in the history
  • Loading branch information
@luissimas
luissimas committed Jun 13, 2024
1 parent caedad7 commit 5eddc2b
Show file tree
Hide file tree
Showing 5 changed files with 103 additions and 66 deletions.
3 changes: 2 additions & 1 deletion README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,8 @@ Prometheus exporter that collects statistics from your second brain.
- [X] Configurable ignore file patterns
- [X] Get zettelkasten from git url
- [X] Register metrics on InfluxDB
- [ ] Backfill data using git
- [X] Make InfluxDB parameters configurable
- [ ] Backfill data using git (only if bucket is empty)
- [ ] Grafana dashboard
- [ ] Docker compose example
- [ ] Kubernetes example
Expand Down
5 changes: 2 additions & 3 deletions cmd/zettelkasten-exporter/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ func main() {
os.Exit(1)
}
slog.Debug("Loaded config", slog.Any("config", cfg))
metrics.ConnectDatabase()
metrics.ConnectDatabase(cfg)
zettelkasten := zettel.NewZettel(cfg)
err = zettelkasten.Ensure()
if err != nil {
Expand All @@ -28,13 +28,12 @@ func main() {
}

collector := collector.NewCollector(zettelkasten.GetRoot(), cfg.IgnoreFiles)

for {
err := collector.CollectMetrics()
if err != nil {
slog.Error("Error collecting metrics", slog.Any("error", err))
os.Exit(1)
}
time.Sleep(10 * time.Second)
time.Sleep(cfg.CollectionInterval)
}
}
58 changes: 48 additions & 10 deletions internal/config/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,10 @@ package config

import (
"errors"
"fmt"
"log/slog"
"strings"
"time"

"github.com/gookit/validate"
"github.com/knadh/koanf"
Expand All @@ -12,29 +14,42 @@ import (
)

type Config struct {
IP string `koanf:"ip" validate:"required|ip"`
Port int `koanf:"port" validate:"required|uint"`
ZettelkastenDirectory string `koanf:"zettelkasten_directory" validate:"requiredWithout:ZettelkastenGitURL"`
ZettelkastenGitURL string `koanf:"zettelkasten_git_url" validate:"requiredWithout:ZettelkastenDirectory" validate:"url/isURL"`
ZettelkastenGitBranch string `koanf:"zettelkasten_git_branch"`
LogLevel slog.Level `koanf:"log_level"`
IgnoreFiles []string `koanf:"ignore_files"`
ZettelkastenDirectory string `koanf:"zettelkasten_directory" validate:"requiredWithout:ZettelkastenGitURL"`
ZettelkastenGitURL string `koanf:"zettelkasten_git_url" validate:"requiredWithout:ZettelkastenDirectory" validate:"url/isURL"`
ZettelkastenGitBranch string `koanf:"zettelkasten_git_branch"`
LogLevel slog.Level `koanf:"log_level"`
IgnoreFiles []string `koanf:"ignore_files"`
CollectionInterval time.Duration `koanf:"collection_interval"`
InfluxDBURL string `koanf:"influxdb_url" validate:"required|fullUrl"`
InfluxDBToken string `koanf:"influxdb_token" validate:"required"`
InfluxDBOrg string `koanf:"influxdb_org" validate:"required"`
InfluxDBBucket string `koanf:"influxdb_bucket" validate:"required"`
}

func LoadConfig() (Config, error) {
k := koanf.New(".")

// Set default values
k.Load(structs.Provider(Config{
IP: "0.0.0.0",
Port: 10018,
LogLevel: slog.LevelInfo,
IgnoreFiles: []string{".git", ".obsidian", ".trash", "README.md"},
ZettelkastenGitBranch: "main",
CollectionInterval: time.Minute * 5,
}, "koanf"), nil)

// Load env variables
k.Load(env.Provider("", ".", strings.ToLower), nil)
k.Load(env.ProviderWithValue("", ".", func(key, value string) (string, interface{}) {
key = strings.ToLower(key)
if key == "collection_interval" {
parsedValue, err := parseCollectionInterval(value)
if err != nil {
slog.Warn("Error parsing collection_interval", slog.Any("error", err))
return key, ""
}
return key, parsedValue
}
return key, value
}), nil)

// Unmarshal into config struct
var cfg Config
Expand All @@ -51,3 +66,26 @@ func LoadConfig() (Config, error) {

return cfg, nil
}

func (c Config) LogValue() slog.Value {
return slog.GroupValue(
slog.String("ZettelkastenDirectory", c.ZettelkastenDirectory),
slog.String("ZettelkastenGitURL", c.ZettelkastenGitURL),
slog.String("ZettelkastenGitBranch", c.ZettelkastenGitBranch),
slog.String("LogLevel", c.LogLevel.String()),
slog.Any("IgnoreFiles", c.IgnoreFiles),
slog.Duration("CollectionInterval", c.CollectionInterval),
slog.String("InfluxDBURL", c.InfluxDBURL),
slog.String("InfluxDBToken", "[REDACTED]"),
slog.String("InfluxDBOrg", c.InfluxDBOrg),
slog.String("InfluxDBBucket", c.InfluxDBBucket),
)
}

func parseCollectionInterval(value string) (time.Duration, error) {
parsed, err := time.ParseDuration(value)
if err != nil {
return 0, fmt.Errorf("invalid config argument: %w", err)
}
return parsed, nil
}
90 changes: 47 additions & 43 deletions internal/config/config_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,20 +3,28 @@ package config
import (
"log/slog"
"testing"
"time"

"github.com/stretchr/testify/assert"
)

func TestLoadConfig_DefaultValues(t *testing.T) {
t.Setenv("INFLUXDB_URL", "http://localhost:8086")
t.Setenv("INFLUXDB_TOKEN", "any-token")
t.Setenv("INFLUXDB_ORG", "any-org")
t.Setenv("INFLUXDB_BUCKET", "any-bucket")
t.Setenv("ZETTELKASTEN_DIRECTORY", "/any/dir")
c, err := LoadConfig()
if err != nil {
t.Fatalf("Expected no error, got: %v", err)
}

expected := Config{
IP: "0.0.0.0",
Port: 10018,
InfluxDBURL: "http://localhost:8086",
InfluxDBToken: "any-token",
InfluxDBOrg: "any-org",
InfluxDBBucket: "any-bucket",
CollectionInterval: time.Minute * 5,
LogLevel: slog.LevelInfo,
ZettelkastenDirectory: "/any/dir",
ZettelkastenGitBranch: "main",
Expand All @@ -26,14 +34,20 @@ func TestLoadConfig_DefaultValues(t *testing.T) {
}

func TestLoadConfig_PartialEnv(t *testing.T) {
t.Setenv("PORT", "4444")
t.Setenv("INFLUXDB_URL", "http://localhost:8086")
t.Setenv("INFLUXDB_TOKEN", "any-token")
t.Setenv("INFLUXDB_ORG", "any-org")
t.Setenv("INFLUXDB_BUCKET", "any-bucket")
t.Setenv("LOG_LEVEL", "DEBUG")
t.Setenv("ZETTELKASTEN_DIRECTORY", "/any/dir")
c, err := LoadConfig()
if assert.NoError(t, err) {
expected := Config{
IP: "0.0.0.0",
Port: 4444,
InfluxDBURL: "http://localhost:8086",
InfluxDBToken: "any-token",
InfluxDBOrg: "any-org",
InfluxDBBucket: "any-bucket",
CollectionInterval: time.Minute * 5,
LogLevel: slog.LevelDebug,
ZettelkastenDirectory: "/any/dir",
ZettelkastenGitBranch: "main",
Expand All @@ -44,17 +58,23 @@ func TestLoadConfig_PartialEnv(t *testing.T) {
}

func TestLoadConfig_FullEnvDirectory(t *testing.T) {
t.Setenv("IP", "127.0.0.1")
t.Setenv("PORT", "4444")
t.Setenv("LOG_LEVEL", "DEBUG")
t.Setenv("INFLUXDB_URL", "http://localhost:8086")
t.Setenv("INFLUXDB_TOKEN", "any-token")
t.Setenv("INFLUXDB_ORG", "any-org")
t.Setenv("INFLUXDB_BUCKET", "any-bucket")
t.Setenv("COLLECTION_INTERVAL", "2h")
t.Setenv("LOG_LEVEL", "WARN")
t.Setenv("ZETTELKASTEN_DIRECTORY", "/any/dir")
t.Setenv("IGNORE_FILES", ".obsidian,test,/something/another,dir/file.md")
c, err := LoadConfig()
if assert.NoError(t, err) {
expected := Config{
IP: "127.0.0.1",
Port: 4444,
LogLevel: slog.LevelDebug,
InfluxDBURL: "http://localhost:8086",
InfluxDBToken: "any-token",
InfluxDBOrg: "any-org",
InfluxDBBucket: "any-bucket",
CollectionInterval: time.Hour * 2,
LogLevel: slog.LevelWarn,
ZettelkastenDirectory: "/any/dir",
ZettelkastenGitBranch: "main",
IgnoreFiles: []string{".obsidian", "test", "/something/another", "dir/file.md"},
Expand All @@ -64,17 +84,23 @@ func TestLoadConfig_FullEnvDirectory(t *testing.T) {
}

func TestLoadConfig_FullEnvGit(t *testing.T) {
t.Setenv("IP", "127.0.0.1")
t.Setenv("PORT", "4444")
t.Setenv("LOG_LEVEL", "DEBUG")
t.Setenv("INFLUXDB_URL", "http://localhost:8086")
t.Setenv("INFLUXDB_TOKEN", "any-token")
t.Setenv("INFLUXDB_ORG", "any-org")
t.Setenv("INFLUXDB_BUCKET", "any-bucket")
t.Setenv("COLLECTION_INTERVAL", "15m")
t.Setenv("LOG_LEVEL", "ERROR")
t.Setenv("ZETTELKASTEN_GIT_URL", "https://github.com/user/zettel")
t.Setenv("IGNORE_FILES", ".obsidian,test,/something/another,dir/file.md")
c, err := LoadConfig()
if assert.NoError(t, err) {
expected := Config{
IP: "127.0.0.1",
Port: 4444,
LogLevel: slog.LevelDebug,
InfluxDBURL: "http://localhost:8086",
InfluxDBToken: "any-token",
InfluxDBOrg: "any-org",
InfluxDBBucket: "any-bucket",
CollectionInterval: time.Minute * 15,
LogLevel: slog.LevelError,
ZettelkastenGitURL: "https://github.com/user/zettel",
ZettelkastenGitBranch: "main",
IgnoreFiles: []string{".obsidian", "test", "/something/another", "dir/file.md"},
Expand All @@ -93,51 +119,29 @@ func TestLoadConfig_Validate(t *testing.T) {
name: "missing source",
shouldError: true,
env: map[string]string{
"IP": "0.0.0.0",
"PORT": "4444",
"LOG_LEVEL": "INFO",
},
},
{
name: "both sources",
shouldError: true,
env: map[string]string{
"IP": "0.0.0.0",
"PORT": "4444",
"LOG_LEVEL": "INFO",
"ZETTELKASTEN_DIRECTORY": "/any/dir",
"ZETTELKASTEN_GIT_URL": "any-string",
},
},
{
name: "invalid ip",
shouldError: true,
env: map[string]string{
"IP": "any-string",
"PORT": "4444",
"LOG_LEVEL": "INFO",
"ZETTELKASTEN_DIRECTORY": "/any/dir",
},
},
{
name: "invalid port",
shouldError: true,
env: map[string]string{
"IP": "0.0.0.0",
"PORT": "-1",
"LOG_LEVEL": "INFO",
"ZETTELKASTEN_DIRECTORY": "/any/dir",
},
},
{
name: "valid config",
shouldError: false,
env: map[string]string{
"IP": "0.0.0.0",
"PORT": "4444",
"LOG_LEVEL": "INFO",
"ZETTELKASTEN_GIT_URL": "any-url",
"ZETTELKASTEN_GIT_BRANCH": "any-branch",
"INFLUXDB_URL": "http://localhost:8086",
"INFLUXDB_TOKEN": "any-token",
"INFLUXDB_ORG": "any-org",
"INFLUXDB_BUCKET": "any-bucket",
},
},
}
Expand Down
13 changes: 4 additions & 9 deletions internal/metrics/metrics.go
View file
Original file line number Diff line number Diff line change
@@ -1,23 +1,18 @@
package metrics

import (
"os"
"time"

influxdb2 "github.com/influxdata/influxdb-client-go/v2"
"github.com/influxdata/influxdb-client-go/v2/api"
"github.com/luissimas/zettelkasten-exporter/internal/config"
)

var influxDB api.WriteAPI

func ConnectDatabase() {
// TODO: get values from config
token := os.Getenv("INFLUXDB_TOKEN")
url := "http://localhost:8086"
client := influxdb2.NewClient(url, token)
org := "default"
bucket := "zettelkasten"
influxDB = client.WriteAPI(org, bucket)
func ConnectDatabase(cfg config.Config) {
client := influxdb2.NewClient(cfg.InfluxDBURL, string(cfg.InfluxDBToken))
influxDB = client.WriteAPI(cfg.InfluxDBOrg, cfg.InfluxDBBucket)
}

func RegisterNoteMetric(name string, linkCount int, timestamp time.Time) {
Expand Down

0 comments on commit 5eddc2b

Please sign in to comment.