Skip to content

Commit

Permalink
encoder: add back the commented option (#893)
Browse files Browse the repository at this point in the history
  • Loading branch information
@pelletier
pelletier committed Aug 28, 2023
1 parent 4a5c27c commit 9dd7f1a
Show file tree
Hide file tree
Showing 3 changed files with 245 additions and 13 deletions.
13 changes: 3 additions & 10 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -497,27 +497,20 @@ is not necessary anymore.

V1 used to provide multiple struct tags: `comment`, `commented`, `multiline`,
`toml`, and `omitempty`. To behave more like the standard library, v2 has merged
`toml`, `multiline`, and `omitempty`. For example:
`toml`, `multiline`, `commented`, and `omitempty`. For example:

```go
type doc struct {
// v1
F string `toml:"field" multiline:"true" omitempty:"true"`
F string `toml:"field" multiline:"true" omitempty:"true" commented:"true"`
// v2
F string `toml:"field,multiline,omitempty"`
F string `toml:"field,multiline,omitempty,commented"`
}
```

Has a result, the `Encoder.SetTag*` methods have been removed, as there is just
one tag now.


#### `commented` tag has been removed

There is no replacement for the `commented` tag. This feature would be better
suited in a proper document model for go-toml v2, which has been [cut from
scope][nodoc] at the moment.

#### `Encoder.ArraysWithOneElementPerLine` has been renamed

The new name is `Encoder.SetArraysMultiline`. The behavior should be the same.
Expand Down
31 changes: 29 additions & 2 deletions marshaler.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -148,6 +148,9 @@ func (enc *Encoder) SetIndentTables(indent bool) *Encoder {
//
// The "omitempty" option prevents empty values or groups from being emitted.
//
// The "commented" option prefixes the value and all its children with a comment
// symbol.
//
// In addition to the "toml" tag struct tag, a "comment" tag can be used to emit
// a TOML comment before the value being annotated. Comments are ignored inside
// inline tables. For array tables, the comment is only present before the first
Expand Down Expand Up @@ -180,6 +183,7 @@ func (enc *Encoder) Encode(v interface{}) error {
type valueOptions struct {
multiline bool
omitempty bool
commented bool
comment string
}

Expand All @@ -205,6 +209,9 @@ type encoderCtx struct {
// Indentation level
indent int

// Prefix the current value with a comment.
commented bool

// Options coming from struct tags
options valueOptions
}
Expand Down Expand Up @@ -357,6 +364,7 @@ func (enc *Encoder) encodeKv(b []byte, ctx encoderCtx, options valueOptions, v r

if !ctx.inline {
b = enc.encodeComment(ctx.indent, options.comment, b)
b = enc.commented(ctx.commented, b)
b = enc.indent(ctx.indent, b)
}

Expand All @@ -378,6 +386,13 @@ func (enc *Encoder) encodeKv(b []byte, ctx encoderCtx, options valueOptions, v r
return b, nil
}

func (enc *Encoder) commented(commented bool, b []byte) []byte {
if commented {
return append(b, "# "...)
}
return b
}

func isEmptyValue(v reflect.Value) bool {
switch v.Kind() {
case reflect.Struct:
Expand Down Expand Up @@ -704,6 +719,7 @@ func walkStruct(ctx encoderCtx, t *table, v reflect.Value) {
options := valueOptions{
multiline: opts.multiline,
omitempty: opts.omitempty,
commented: opts.commented,
comment: fieldType.Tag.Get("comment"),
}

Expand Down Expand Up @@ -763,6 +779,7 @@ type tagOptions struct {
multiline bool
inline bool
omitempty bool
commented bool
}

func parseTag(tag string) (string, tagOptions) {
Expand Down Expand Up @@ -790,6 +807,8 @@ func parseTag(tag string) (string, tagOptions) {
opts.inline = true
case "omitempty":
opts.omitempty = true
case "commented":
opts.commented = true
}
}

Expand All @@ -806,6 +825,7 @@ func (enc *Encoder) encodeTable(b []byte, ctx encoderCtx, t table) ([]byte, erro
}

if !ctx.skipTableHeader {
b = enc.commented(ctx.commented, b)
b, err = enc.encodeTableHeader(ctx, b)
if err != nil {
return nil, err
Expand All @@ -825,8 +845,10 @@ func (enc *Encoder) encodeTable(b []byte, ctx encoderCtx, t table) ([]byte, erro
hasNonEmptyKV = true

ctx.setKey(kv.Key)
ctx2 := ctx
ctx2.commented = kv.Options.commented || ctx2.commented

b, err = enc.encodeKv(b, ctx, kv.Options, kv.Value)
b, err = enc.encodeKv(b, ctx2, kv.Options, kv.Value)
if err != nil {
return nil, err
}
Expand All @@ -851,8 +873,10 @@ func (enc *Encoder) encodeTable(b []byte, ctx encoderCtx, t table) ([]byte, erro
ctx.setKey(table.Key)

ctx.options = table.Options
ctx2 := ctx
ctx2.commented = ctx2.commented || ctx.options.commented

b, err = enc.encode(b, ctx, table.Value)
b, err = enc.encode(b, ctx2, table.Value)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -970,6 +994,9 @@ func (enc *Encoder) encodeSliceAsArrayTable(b []byte, ctx encoderCtx, v reflect.
ctx.shiftKey()

scratch := make([]byte, 0, 64)

scratch = enc.commented(ctx.commented, scratch)

scratch = append(scratch, "[["...)

for i, k := range ctx.parentKey {
Expand Down
214 changes: 213 additions & 1 deletion marshaler_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1337,7 +1337,7 @@ func TestMarshalUint64Overflow(t *testing.T) {

func TestIndentWithInlineTable(t *testing.T) {
x := map[string][]map[string]string{
"one": []map[string]string{
"one": {
{"0": "0"},
{"1": "1"},
},
Expand All @@ -1356,6 +1356,94 @@ func TestIndentWithInlineTable(t *testing.T) {
assert.Equal(t, expected, buf.String())
}

type C3 struct {
Value int `toml:",commented"`
Values []int `toml:",commented"`
}

type C2 struct {
Int int64
String string
ArrayInts []int
Structs []C3
}

type C1 struct {
Int int64 `toml:",commented"`
String string `toml:",commented"`
ArrayInts []int `toml:",commented"`
Structs []C3 `toml:",commented"`
}

type Commented struct {
Int int64 `toml:",commented"`
String string `toml:",commented"`

C1 C1
C2 C2 `toml:",commented"` // same as C1, but commented at top level
}

func TestMarshalCommented(t *testing.T) {
c := Commented{
Int: 42,
String: "root",

C1: C1{
Int: 11,
String: "C1",
ArrayInts: []int{1, 2, 3},
Structs: []C3{
{Value: 100},
{Values: []int{4, 5, 6}},
},
},
C2: C2{
Int: 22,
String: "C2",
ArrayInts: []int{1, 2, 3},
Structs: []C3{
{Value: 100},
{Values: []int{4, 5, 6}},
},
},
}

out, err := toml.Marshal(c)
require.NoError(t, err)

expected := `# Int = 42
# String = 'root'

[C1]
# Int = 11
# String = 'C1'
# ArrayInts = [1, 2, 3]

# [[C1.Structs]]
# Value = 100
# Values = []

# [[C1.Structs]]
# Value = 0
# Values = [4, 5, 6]

# [C2]
# Int = 22
# String = 'C2'
# ArrayInts = [1, 2, 3]

# [[C2.Structs]]
# Value = 100
# Values = []

# [[C2.Structs]]
# Value = 0
# Values = [4, 5, 6]
`

require.Equal(t, expected, string(out))
}

func ExampleMarshal() {
type MyConfig struct {
Version int
Expand All @@ -1380,3 +1468,127 @@ func ExampleMarshal() {
// Name = 'go-toml'
// Tags = ['go', 'toml']
}

// Example that uses the 'commented' field tag option to generate an example
// configuration file that has commented out sections (example from
// go-graphite/graphite-clickhouse).
func ExampleMarshal_commented() {

type Common struct {
Listen string `toml:"listen" comment:"general listener"`
PprofListen string `toml:"pprof-listen" comment:"listener to serve /debug/pprof requests. '-pprof' argument overrides it"`
MaxMetricsPerTarget int `toml:"max-metrics-per-target" comment:"limit numbers of queried metrics per target in /render requests, 0 or negative = unlimited"`
MemoryReturnInterval time.Duration `toml:"memory-return-interval" comment:"daemon will return the freed memory to the OS when it>0"`
}

type Costs struct {
Cost *int `toml:"cost" comment:"default cost (for wildcarded equalence or matched with regex, or if no value cost set)"`
ValuesCost map[string]int `toml:"values-cost" comment:"cost with some value (for equalence without wildcards) (additional tuning, usually not needed)"`
}

type ClickHouse struct {
URL string `toml:"url" comment:"default url, see https://clickhouse.tech/docs/en/interfaces/http. Can be overwritten with query-params"`

RenderMaxQueries int `toml:"render-max-queries" comment:"Max queries to render queiries"`
RenderConcurrentQueries int `toml:"render-concurrent-queries" comment:"Concurrent queries to render queiries"`
TaggedCosts map[string]*Costs `toml:"tagged-costs,commented"`
TreeTable string `toml:"tree-table,commented"`
ReverseTreeTable string `toml:"reverse-tree-table,commented"`
DateTreeTable string `toml:"date-tree-table,commented"`
DateTreeTableVersion int `toml:"date-tree-table-version,commented"`
TreeTimeout time.Duration `toml:"tree-timeout,commented"`
TagTable string `toml:"tag-table,commented"`
ExtraPrefix string `toml:"extra-prefix" comment:"add extra prefix (directory in graphite) for all metrics, w/o trailing dot"`
ConnectTimeout time.Duration `toml:"connect-timeout" comment:"TCP connection timeout"`
DataTableLegacy string `toml:"data-table,commented"`
RollupConfLegacy string `toml:"rollup-conf,commented"`
MaxDataPoints int `toml:"max-data-points" comment:"max points per metric when internal-aggregation=true"`
InternalAggregation bool `toml:"internal-aggregation" comment:"ClickHouse-side aggregation, see doc/aggregation.md"`
}

type Tags struct {
Rules string `toml:"rules"`
Date string `toml:"date"`
ExtraWhere string `toml:"extra-where"`
InputFile string `toml:"input-file"`
OutputFile string `toml:"output-file"`
}

type Config struct {
Common Common `toml:"common"`
ClickHouse ClickHouse `toml:"clickhouse"`
Tags Tags `toml:"tags,commented"`
}

cfg := &Config{
Common: Common{
Listen: ":9090",
PprofListen: "",
MaxMetricsPerTarget: 15000, // This is arbitrary value to protect CH from overload
MemoryReturnInterval: 0,
},
ClickHouse: ClickHouse{
URL: "http://localhost:8123?cancel_http_readonly_queries_on_client_close=1",
ExtraPrefix: "",
ConnectTimeout: time.Second,
DataTableLegacy: "",
RollupConfLegacy: "auto",
MaxDataPoints: 1048576,
InternalAggregation: true,
},
Tags: Tags{},
}

out, err := toml.Marshal(cfg)
if err != nil {
panic(err)
}
err = toml.Unmarshal(out, &cfg)
if err != nil {
panic(err)
}

fmt.Println(string(out))

// Output:
// [common]
// # general listener
// listen = ':9090'
// # listener to serve /debug/pprof requests. '-pprof' argument overrides it
// pprof-listen = ''
// # limit numbers of queried metrics per target in /render requests, 0 or negative = unlimited
// max-metrics-per-target = 15000
// # daemon will return the freed memory to the OS when it>0
// memory-return-interval = 0
//
// [clickhouse]
// # default url, see https://clickhouse.tech/docs/en/interfaces/http. Can be overwritten with query-params
// url = 'http://localhost:8123?cancel_http_readonly_queries_on_client_close=1'
// # Max queries to render queiries
// render-max-queries = 0
// # Concurrent queries to render queiries
// render-concurrent-queries = 0
// # tree-table = ''
// # reverse-tree-table = ''
// # date-tree-table = ''
// # date-tree-table-version = 0
// # tree-timeout = 0
// # tag-table = ''
// # add extra prefix (directory in graphite) for all metrics, w/o trailing dot
// extra-prefix = ''
// # TCP connection timeout
// connect-timeout = 1000000000
// # data-table = ''
// # rollup-conf = 'auto'
// # max points per metric when internal-aggregation=true
// max-data-points = 1048576
// # ClickHouse-side aggregation, see doc/aggregation.md
// internal-aggregation = true
//
// # [tags]
// # rules = ''
// # date = ''
// # extra-where = ''
// # input-file = ''
// # output-file = ''
}

0 comments on commit 9dd7f1a

Please sign in to comment.