From e57dcf53d8144ff0b2019a664340c3b6424def5a Mon Sep 17 00:00:00 2001 From: Antoine Martin Date: Tue, 24 Jan 2023 10:55:47 +0000 Subject: [PATCH 01/10] =?UTF-8?q?=E2=9C=A8=20Add=20extended=20ReplacementT?= =?UTF-8?q?ransformer?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This transformer works the same as the builtin one that it replaces but adds extended path specifications based on the _lenses_ of knot8. The typical use case for this is when you want to replace a value in the `values` field of an application: ```yaml values: | uninode: true apps: enabled: true common: targetRevision: main repoURL: https://github.com/anotherproject/anothergit ``` This commit allows you to write: ```yaml fieldPaths: ... - spec.source.helm.values.!!yaml.common.repoURL ``` Note the double exclamation point (!!) before `yaml`. This tells _change the `common.repoURL` value of the `spec.source.helm.values` property that is encoded in YAML. Apart from `yaml`, this commit add two other encodings: - `base64`, to decode/encode base64 based values. - `regex` to allow regexp based changes. Paths for this encoding have two elements. The first one is the regexp to match and the second one is the matching group index to replace. Example: ^\s+HostName\s+(\S+)\s*$.1 It means: Look for a line with `HostName` followed by a word and replace this word (matching group 1) with the value passed. [knot8]: https://github.com/mkmik/knot8 --- go.mod | 6 +- go.sum | 11 +- pkg/extras/extender.go | 421 ++++++++++++++++++ pkg/extras/extender_test.go | 278 ++++++++++++ pkg/extras/extendertype_string.go | 26 ++ pkg/extras/replacement.go | 401 +++++++++++++++++ pkg/plugins/factories.go | 2 +- .../expected/argocd.yaml} | 0 .../functions/patch-transformer.yaml | 2 +- .../original}/argocd.yaml | 0 tests/replacement/expected/argocd.yaml | 41 ++ .../functions}/01_configmap-generator.yaml | 2 +- .../02_replacement-transformer.yaml | 4 +- .../original/argocd.yaml} | 7 + tests/test_krmfnbuiltin.sh | 21 +- 15 files changed, 1205 insertions(+), 17 deletions(-) create mode 100644 pkg/extras/extender.go create mode 100644 pkg/extras/extender_test.go create mode 100644 pkg/extras/extendertype_string.go create mode 100644 pkg/extras/replacement.go rename tests/{compare/argocd.expected.yaml => patch/expected/argocd.yaml} (100%) rename tests/{ => patch}/functions/patch-transformer.yaml (95%) rename tests/{applications => patch/original}/argocd.yaml (100%) create mode 100644 tests/replacement/expected/argocd.yaml rename tests/{functions2 => replacement/functions}/01_configmap-generator.yaml (92%) rename tests/{functions2 => replacement/functions}/02_replacement-transformer.yaml (84%) rename tests/{compare/argocd.original.yaml => replacement/original/argocd.yaml} (80%) diff --git a/go.mod b/go.mod index 333d365..1b0fb6c 100644 --- a/go.mod +++ b/go.mod @@ -4,10 +4,14 @@ go 1.18 require ( github.com/go-git/go-git/v5 v5.5.2 + github.com/lithammer/dedent v1.1.0 + github.com/stretchr/testify v1.8.1 + golang.org/x/text v0.6.0 golang.org/x/tools v0.5.0 sigs.k8s.io/kustomize/api v0.12.1 sigs.k8s.io/kustomize/kyaml v0.13.10 sigs.k8s.io/yaml v1.2.0 + ) require ( @@ -39,6 +43,7 @@ require ( github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 // indirect github.com/pjbgf/sha1cd v0.2.3 // indirect github.com/pkg/errors v0.9.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect github.com/sergi/go-diff v1.1.0 // indirect github.com/skeema/knownhosts v1.1.0 // indirect github.com/spf13/cobra v1.4.0 // indirect @@ -49,7 +54,6 @@ require ( golang.org/x/mod v0.7.0 // indirect golang.org/x/net v0.5.0 // indirect golang.org/x/sys v0.4.0 // indirect - golang.org/x/text v0.6.0 // indirect google.golang.org/protobuf v1.28.0 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect diff --git a/go.sum b/go.sum index 3c91dce..836e3d7 100644 --- a/go.sum +++ b/go.sum @@ -110,6 +110,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/lithammer/dedent v1.1.0 h1:VNzHMVCBNG1j0fh3OrsFRkVUwStdDArbgBWoPAffktY= +github.com/lithammer/dedent v1.1.0/go.mod h1:jrXYCQtgg0nJiN+StA2KgR7w6CiQNv9Fd/Z9BP0jIOc= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= @@ -147,13 +149,18 @@ github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.4.0 h1:M2gUjqZET1qApGOWNSnZ49BAIMX4F/1plDv3+l31EJ4= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM= github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw= github.com/xlab/treeprint v1.1.0 h1:G/1DjNkPpfZCFt9CSh6b5/nY4VimlbHF3Rh4obvtzDk= diff --git a/pkg/extras/extender.go b/pkg/extras/extender.go new file mode 100644 index 0000000..abb9851 --- /dev/null +++ b/pkg/extras/extender.go @@ -0,0 +1,421 @@ +package extras + +import ( + "bytes" + "encoding/base64" + "fmt" + "regexp" + "strconv" + "strings" + + "golang.org/x/text/cases" + "golang.org/x/text/language" + "sigs.k8s.io/kustomize/kyaml/errors" + "sigs.k8s.io/kustomize/kyaml/kio" + "sigs.k8s.io/kustomize/kyaml/yaml" +) + +type Extender interface { + SetPayload(payload []byte) error + GetPayload() ([]byte, error) + Get(path []string) ([]byte, error) + Set(path []string, value any) error +} + +type ExtendedSegment struct { + Encoding string + Path []string +} + +func (e *ExtendedSegment) String() string { + if len(e.Path) > 0 { + return fmt.Sprintf("!!%s", e.Encoding) + } else { + return fmt.Sprintf("!!%s.%s", e.Encoding, strings.Join(e.Path, ".")) + } +} + +type any interface{} + +//go:generate go run golang.org/x/tools/cmd/stringer -type=ExtenderType +type ExtenderType int + +const ( + Unknown ExtenderType = iota + YamlExtender + Base64Extender + RegexExtender +) + +var stringToExtenderTypeMap map[string]ExtenderType + +func init() { //nolint:gochecknoinits + stringToExtenderTypeMap = makeStringToExtenderTypeMap() +} + +func GetByteValue(value any) []byte { + switch v := value.(type) { + case *yaml.Node: + return []byte(v.Value) + case []byte: + return v + case string: + return []byte(v) + } + return []byte{} +} + +func makeStringToExtenderTypeMap() (result map[string]ExtenderType) { + result = make(map[string]ExtenderType, 3) + for k := range ExtenderFactories { + result[k.String()] = k + } + return +} + +func GetExtenderType(n string) ExtenderType { + result, ok := stringToExtenderTypeMap[n] + if ok { + return result + } + return Unknown +} + +//////////////// +// YAML Extender +//////////////// + +type yamlExtender struct { + node *yaml.RNode +} + +func (e *yamlExtender) SetPayload(payload []byte) error { + + nodes, err := (&kio.ByteReader{ + Reader: bytes.NewBuffer(payload), + OmitReaderAnnotations: false, + PreserveSeqIndent: true, + WrapBareSeqNode: true, + }).Read() + + if err != nil { + return errors.WrapPrefixf(err, "while reading yaml payload") + } + e.node = nodes[0] + return nil +} + +func (e *yamlExtender) GetPayload() ([]byte, error) { + var b bytes.Buffer + err := (&kio.ByteWriter{Writer: &b}).Write([]*yaml.RNode{e.node}) + return b.Bytes(), err +} + +func (e *yamlExtender) LookupNode() *yaml.RNode { + seqNode, err := e.node.Pipe(yaml.Lookup(yaml.BareSeqNodeWrappingKey)) + if err == nil && !seqNode.IsNilOrEmpty() { + return seqNode + } + return e.node +} + +func (e *yamlExtender) Lookup(path []string) ([]*yaml.RNode, error) { + node, err := e.LookupNode().Pipe(&yaml.PathMatcher{Path: path}) + if err != nil { + return nil, errors.WrapPrefixf(err, "while getting path %s", strings.Join(path, ".")) + } + + return node.Elements() +} + +func (e *yamlExtender) Get(path []string) ([]byte, error) { + targetFields, err := e.Lookup(path) + if err != nil { + return nil, fmt.Errorf("error fetching elements in replacement target: %w", err) + } + + if len(targetFields) == 1 && targetFields[0].YNode().Kind == yaml.ScalarNode { + return []byte(targetFields[0].YNode().Value), nil + } + + var b bytes.Buffer + err = (&kio.ByteWriter{Writer: &b}).Write(targetFields) + if err != nil { + return nil, errors.WrapPrefixf(err, "while serializing path %s", strings.Join(path, ".")) + } + return b.Bytes(), err +} + +func (e *yamlExtender) Set(path []string, value any) error { + + targetFields, err := e.Lookup(path) + if err != nil { + return fmt.Errorf("error fetching elements in replacement target: %w", err) + } + + for _, t := range targetFields { + if t.YNode().Kind == yaml.ScalarNode { + t.YNode().Value = string(GetByteValue(value)) + } else { + if v, ok := value.(*yaml.Node); ok { + t.SetYNode(v) + } else { + return fmt.Errorf("setting non yaml object in place of object of type %s at path %s", t.YNode().Tag, strings.Join(path, ".")) + } + } + } + return nil +} + +func NewYamlExtender() Extender { + return &yamlExtender{} +} + +///////// +// Base64 +///////// + +type base64Extender struct { + decoded []byte +} + +func (e *base64Extender) SetPayload(payload []byte) error { + decoded, err := base64.StdEncoding.DecodeString(string(payload)) + if err != nil { + return errors.WrapPrefixf(err, "while decoding base64") + } + e.decoded = decoded + return nil +} + +func (e *base64Extender) GetPayload() ([]byte, error) { + return e.decoded, nil +} + +func (e *base64Extender) Get(path []string) ([]byte, error) { + if len(path) > 0 { + return nil, fmt.Errorf("path is invalid for base64: %s", strings.Join(path, ".")) + } + return e.decoded, nil +} + +func (e *base64Extender) Set(path []string, value any) error { + if len(path) > 0 { + return fmt.Errorf("path is invalid for base64: %s", strings.Join(path, ".")) + } + e.decoded = []byte(base64.StdEncoding.EncodeToString(GetByteValue(value))) + return nil +} + +func NewBase64Extender() Extender { + return &base64Extender{} +} + +///////// +// Regex +//////// + +type regexExtender struct { + text []byte +} + +func (e *regexExtender) SetPayload(payload []byte) error { + e.text = payload + return nil +} + +func (e *regexExtender) GetPayload() ([]byte, error) { + return []byte(e.text), nil +} + +func (e *regexExtender) Get(path []string) ([]byte, error) { + if len(path) < 1 { + return nil, fmt.Errorf("path for regex should at least be one") + } + re, err := regexp.Compile(path[0]) + if err != nil { + return nil, fmt.Errorf("bad regex %s", path[0]) + } + return re.Find(e.text), nil +} + +func (e *regexExtender) Set(path []string, value any) error { + if len(path) != 2 { + return fmt.Errorf("path for regex should at least be one") + } + re, err := regexp.Compile("(?m)" + path[0]) + if err != nil { + return fmt.Errorf("bad regex %s", path[0]) + } + + group, err := strconv.Atoi(path[1]) + if err != nil { + return fmt.Errorf("bad capturing group") + } + + var b bytes.Buffer + start := 0 + matched := false + + for _, v := range re.FindAllSubmatchIndex(e.text, -1) { + matched = true + startIndex := group * 2 + + b.Write(e.text[start:v[startIndex]]) + b.Write(GetByteValue(value)) + start = v[startIndex+1] + } + + if matched { + if start < len(e.text) { + b.Write(e.text[start:len(e.text)]) + } + e.text = b.Bytes() + } + + return nil +} + +func NewRegexExtender() Extender { + return ®exExtender{} +} + +//////////// +// Factories +//////////// + +var ExtenderFactories = map[ExtenderType]func() Extender{ + YamlExtender: NewYamlExtender, + Base64Extender: NewBase64Extender, + RegexExtender: NewRegexExtender, +} + +func (path *ExtendedSegment) Extender(payload []byte) (Extender, error) { + bpt := GetExtenderType(cases.Title(language.English, cases.NoLower).String(path.Encoding) + "Extender") + if f, ok := ExtenderFactories[bpt]; ok { + result := f() + if err := result.SetPayload(payload); err != nil { + return nil, err + } + + return result, nil + } + return nil, errors.Errorf("unable to load extender %s", path.Encoding) +} + +/////////////// +// ExtendedPath +/////////////// + +func splitExtendedPath(path []string, extensions *[]*ExtendedSegment) (basePath []string, err error) { + + if len(path) == 0 { + return + } + + for i, p := range path { + if strings.HasPrefix(p, "!!") { + extension := ExtendedSegment{Encoding: p[2:]} + if extension.Encoding == "" { + err = fmt.Errorf("extension cannot be empty") + return + } + *extensions = append(*extensions, &extension) + var remainder []string + remainder, err = splitExtendedPath(path[i+1:], extensions) + if err != nil { + err = errors.WrapPrefixf(err, "while getting subpath of extension %s", extension.Encoding) + return + } + extension.Path = remainder + return + } else { + basePath = append(basePath, p) + } + } + return +} + +type ExtendedPath struct { + resourcePath []string + extendedSegments *[]*ExtendedSegment +} + +func NewExtendedPath(path []string) (*ExtendedPath, error) { + extensions := []*ExtendedSegment{} + prefix, err := splitExtendedPath(path, &extensions) + if err != nil { + return nil, errors.WrapPrefixf(err, "while getting extended path") + } + + return &ExtendedPath{resourcePath: prefix, extendedSegments: &extensions}, nil +} + +func (ep *ExtendedPath) HasExtensions() bool { + return len(*ep.extendedSegments) > 0 +} + +func (ep *ExtendedPath) String() string { + out := strings.Join(ep.resourcePath, ".") + if len(*ep.extendedSegments) > 0 { + segmentStrings := []string{} + for _, s := range *ep.extendedSegments { + segmentStrings = append(segmentStrings, s.String()) + } + out = fmt.Sprintf("%s.%s", out, strings.Join(segmentStrings, ".")) + } + return out +} + +func (ep *ExtendedPath) ApplyIndex(index int, input []byte, value *yaml.Node) ([]byte, error) { + if index >= len(*ep.extendedSegments) || index < 0 { + return nil, fmt.Errorf("invalid extended path index: %d", index) + } + + segment := (*ep.extendedSegments)[index] + extender, err := segment.Extender(input) + if err != nil { + return nil, errors.WrapPrefixf(err, "creating extender at index: %d", index) + } + + if index == len(*ep.extendedSegments)-1 { + err := extender.Set(segment.Path, value) + if err != nil { + return nil, errors.WrapPrefixf(err, "setting value on path %s", segment.String()) + } + } else { + nextInput, err := extender.Get(segment.Path) + if err != nil { + return nil, errors.WrapPrefixf(err, "getting value on path %s", segment.String()) + } + newValue, err := ep.ApplyIndex(index+1, nextInput, value) + if err != nil { + return nil, err + } + + err = extender.Set(segment.Path, newValue) + if err != nil { + return nil, errors.WrapPrefixf(err, "setting value on path %s", segment.String()) + } + } + return extender.GetPayload() +} + +func (ep *ExtendedPath) Apply(target *yaml.RNode, value *yaml.RNode) error { + if target.YNode().Kind != yaml.ScalarNode { + return fmt.Errorf("extended path only works on scalar nodes") + } + + outValue := value.YNode().Value + if len(*ep.extendedSegments) > 0 { + input := []byte(target.YNode().Value) + output, err := ep.ApplyIndex(0, input, value.YNode()) + if err != nil { + return errors.WrapPrefixf(err, "applying value on extended segment %s", ep.String()) + } + + outValue = string(output) + } + target.YNode().Value = outValue + return nil +} diff --git a/pkg/extras/extender_test.go b/pkg/extras/extender_test.go new file mode 100644 index 0000000..f479a8f --- /dev/null +++ b/pkg/extras/extender_test.go @@ -0,0 +1,278 @@ +package extras + +import ( + "bytes" + "testing" + + "github.com/lithammer/dedent" + "github.com/stretchr/testify/suite" + "sigs.k8s.io/kustomize/kyaml/kio" + kyaml_utils "sigs.k8s.io/kustomize/kyaml/utils" + "sigs.k8s.io/kustomize/kyaml/yaml" +) + +type ExtenderTestSuite struct { + suite.Suite +} + +func (s *ExtenderTestSuite) SetupTest() { +} + +func (s *ExtenderTestSuite) TeardownTest() { +} + +func (s *ExtenderTestSuite) TestSplitPath() { + require := s.Require() + p := "toto.tata.!!yaml.toto.tata" + path := kyaml_utils.SmarterPathSplitter(p, ".") + + extensions := []*ExtendedSegment{} + remainder, err := splitExtendedPath(path, &extensions) + + require.NoError(err) + require.Len(remainder, 2, "Remainder path should be 2") + require.Len(extensions, 1, "Should only have one extension") + require.Equal("yaml", extensions[0].Encoding, "Extension should be yaml") + require.Len(extensions[0].Path, 2, "Extension path len should be 2") +} + +func (s *ExtenderTestSuite) TestRegexExtender() { + text := dedent.Dedent(` + PubkeyAcceptedKeyTypes +ssh-rsa + Host sishserver + HostName holepunch.in + Port 2222 + BatchMode yes + IdentityFile ~/.ssh_keys/id_rsa + IdentitiesOnly yes + LogLevel ERROR + ServerAliveInterval 10 + ServerAliveCountMax 2 + RemoteCommand sni-proxy=true + RemoteForward citest.holepunch.in:443 traefik.traefik.svc:443 + `) + expected := dedent.Dedent(` + PubkeyAcceptedKeyTypes +ssh-rsa + Host sishserver + HostName kaweezle.com + Port 2222 + BatchMode yes + IdentityFile ~/.ssh_keys/id_rsa + IdentitiesOnly yes + LogLevel ERROR + ServerAliveInterval 10 + ServerAliveCountMax 2 + RemoteCommand sni-proxy=true + RemoteForward citest.holepunch.in:443 traefik.traefik.svc:443 + `) + require := s.Require() + path := &ExtendedSegment{ + Encoding: "regex", + Path: []string{`^\s+HostName\s+(\S+)\s*$`, `1`}, + } + + extender, err := path.Extender([]byte(text)) + require.NoError(err) + require.NotNil(extender) + + require.NoError(extender.Set(path.Path, []byte("kaweezle.com"))) + + out, err := extender.GetPayload() + require.NoError(err) + require.Equal(expected, string(out), "Text should be modified") +} + +func (s *ExtenderTestSuite) TestBase64Extender() { + encoded := "UHVia2V5QWNjZXB0ZWRLZXlUeXBlcyArc3NoLXJzYQpIb3N0IHNpc2hzZXJ2ZXIKICBIb3N0TmFtZSBob2xlcHVuY2guaW4KICBQb3J0IDIyMjIKICBCYXRjaE1vZGUgeWVzCiAgSWRlbnRpdHlGaWxlIH4vLnNzaF9rZXlzL2lkX3JzYQogIElkZW50aXRpZXNPbmx5IHllcwogIExvZ0xldmVsIEVSUk9SCiAgU2VydmVyQWxpdmVJbnRlcnZhbCAxMAogIFNlcnZlckFsaXZlQ291bnRNYXggMgogIFJlbW90ZUNvbW1hbmQgc25pLXByb3h5PXRydWUKICBSZW1vdGVGb3J3YXJkIGNpdGVzdC5ob2xlcHVuY2guaW46NDQzIHRyYWVmaWsudHJhZWZpay5zdmM6NDQzCg==" + decodedExpected := dedent.Dedent(` + PubkeyAcceptedKeyTypes +ssh-rsa + Host sishserver + HostName holepunch.in + Port 2222 + BatchMode yes + IdentityFile ~/.ssh_keys/id_rsa + IdentitiesOnly yes + LogLevel ERROR + ServerAliveInterval 10 + ServerAliveCountMax 2 + RemoteCommand sni-proxy=true + RemoteForward citest.holepunch.in:443 traefik.traefik.svc:443 + `)[1:] + + modifiedEncoded := "UHVia2V5QWNjZXB0ZWRLZXlUeXBlcyArc3NoLXJzYQpIb3N0IHNpc2hzZXJ2ZXIKICBIb3N0TmFtZSBrYXdlZXpsZS5jb20KICBQb3J0IDIyMjIKICBCYXRjaE1vZGUgeWVzCiAgSWRlbnRpdHlGaWxlIH4vLnNzaF9rZXlzL2lkX3JzYQogIElkZW50aXRpZXNPbmx5IHllcwogIExvZ0xldmVsIEVSUk9SCiAgU2VydmVyQWxpdmVJbnRlcnZhbCAxMAogIFNlcnZlckFsaXZlQ291bnRNYXggMgogIFJlbW90ZUNvbW1hbmQgc25pLXByb3h5PXRydWUKICBSZW1vdGVGb3J3YXJkIGNpdGVzdC5ob2xlcHVuY2guaW46NDQzIHRyYWVmaWsudHJhZWZpay5zdmM6NDQzCg==" + + require := s.Require() + + p := `!!base64.!!regex.\s+HostName\s+(\S+).1` + path := kyaml_utils.SmarterPathSplitter(p, ".") + + extensions := []*ExtendedSegment{} + prefix, err := splitExtendedPath(path, &extensions) + require.NoError(err) + require.Len(prefix, 0, "There should be no prefix") + require.Len(extensions, 2, "There should be 2 extensions") + require.Equal("base64", extensions[0].Encoding, "The first extension should be base64") + + b64Ext := extensions[0] + b64Extender, err := b64Ext.Extender([]byte(encoded)) + require.NoError(err) + require.IsType(&base64Extender{}, b64Extender, "Should be a base64 extender") + + decoded, err := b64Extender.Get(b64Ext.Path) + require.NoError(err) + require.Equal(decodedExpected, string(decoded), "bad base64 decoding") + + regexExt := extensions[1] + reExtender, err := regexExt.Extender(decoded) + require.NoError(err) + require.IsType(®exExtender{}, reExtender, "Should be a regex extender") + + require.NoError(reExtender.Set(regexExt.Path, []byte("kaweezle.com"))) + modified, err := reExtender.GetPayload() + require.NoError(err) + require.NoError(b64Extender.Set(b64Ext.Path, modified)) + final, err := b64Extender.GetPayload() + require.NoError(err) + require.Equal(modifiedEncoded, string(final), "final base64 is bad") +} + +func (s *ExtenderTestSuite) TestYamlExtender() { + require := s.Require() + source := dedent.Dedent(` + uninode: true + common: + targetRevision: main + apps: + enabled: true + `)[1:] + expected := dedent.Dedent(` + uninode: true + common: + targetRevision: deploy/citest + apps: + enabled: true + `)[1:] + + p := `!!yaml.common.targetRevision` + path := kyaml_utils.SmarterPathSplitter(p, ".") + + extensions := []*ExtendedSegment{} + prefix, err := splitExtendedPath(path, &extensions) + require.NoError(err) + require.Len(prefix, 0, "There should be no prefix") + require.Len(extensions, 1, "There should be 2 extensions") + require.Equal("yaml", extensions[0].Encoding, "The first extension should be base64") + + yamlXP := extensions[0] + yamlExt, err := yamlXP.Extender([]byte(source)) + require.NoError(err) + value, err := yamlExt.Get(yamlXP.Path) + require.NoError(err) + require.Equal("main", string(value), "error fetching value") + require.NoError(yamlExt.Set(yamlXP.Path, []byte("deploy/citest"))) + + modified, err := yamlExt.GetPayload() + require.NoError(err) + require.Equal(expected, string(modified), "final yaml") + + value, err = yamlExt.Get(yamlXP.Path) + require.NoError(err) + require.Equal("deploy/citest", string(value), "error fetching changed value") +} + +func (s *ExtenderTestSuite) TestYamlExtenderWithSequence() { + require := s.Require() + source := dedent.Dedent(` + - name: common.targetRevision + value: main + - name: common.repoURL + value: https://github.com/antoinemartin/autocloud.git + `)[1:] + expected := dedent.Dedent(` + - name: common.targetRevision + value: deploy/citest + - name: common.repoURL + value: https://github.com/antoinemartin/autocloud.git + `)[1:] + + p := `!!yaml.[name=common.targetRevision].value` + path := kyaml_utils.SmarterPathSplitter(p, ".") + + extensions := []*ExtendedSegment{} + prefix, err := splitExtendedPath(path, &extensions) + require.NoError(err) + require.Len(prefix, 0, "There should be no prefix") + require.Len(extensions, 1, "There should be 2 extensions") + require.Equal("yaml", extensions[0].Encoding, "The first extension should be base64") + + yamlXP := extensions[0] + yamlExt, err := yamlXP.Extender([]byte(source)) + require.NoError(err) + require.NoError(yamlExt.Set(yamlXP.Path, []byte("deploy/citest"))) + + modified, err := yamlExt.GetPayload() + require.NoError(err) + require.Equal(expected, string(modified), "final yaml") +} + +func (s *ExtenderTestSuite) TestYamlExtenderWithYaml() { + require := s.Require() + sources, err := (&kio.ByteReader{Reader: bytes.NewBufferString(` +common: | + uninode: true + common: + targetRevision: main + apps: + enabled: true +`)}).Read() + require.NoError(err) + require.Len(sources, 1) + source := sources[0] + + expected := dedent.Dedent(` + common: | + uninode: true + common: + targetRevision: deploy/citest + repoURL: https://github.com/antoinemartin/autocloud.git + apps: + enabled: true + `)[1:] + + replacements, err := (&kio.ByteReader{Reader: bytes.NewBufferString(` +common: + targetRevision: deploy/citest + repoURL: https://github.com/antoinemartin/autocloud.git +`)}).Read() + require.NoError(err) + require.Len(replacements, 1) + replacement := replacements[0] + + p := `common.!!yaml.common` + path := kyaml_utils.SmarterPathSplitter(p, ".") + e, err := NewExtendedPath(path) + require.NoError(err) + require.Len(e.resourcePath, 1, "no resource path") + + sourcePath := []string{"common"} + + target, err := source.Pipe(&yaml.PathGetter{Path: e.resourcePath}) + require.NoError(err) + + value, err := replacement.Pipe(&yaml.PathGetter{Path: sourcePath}) + require.NoError(err) + err = e.Apply(target, value) + require.NoError(err) + + var b bytes.Buffer + err = (&kio.ByteWriter{Writer: &b}).Write(sources) + require.NoError(err) + + sString, err := source.String() + require.NoError(err) + require.Equal(expected, b.String(), sString, "replacement failed") +} + +func TestExtender(t *testing.T) { + suite.Run(t, new(ExtenderTestSuite)) +} diff --git a/pkg/extras/extendertype_string.go b/pkg/extras/extendertype_string.go new file mode 100644 index 0000000..a613560 --- /dev/null +++ b/pkg/extras/extendertype_string.go @@ -0,0 +1,26 @@ +// Code generated by "stringer -type=ExtenderType"; DO NOT EDIT. + +package extras + +import "strconv" + +func _() { + // An "invalid array index" compiler error signifies that the constant values have changed. + // Re-run the stringer command to generate them again. + var x [1]struct{} + _ = x[Unknown-0] + _ = x[YamlExtender-1] + _ = x[Base64Extender-2] + _ = x[RegexExtender-3] +} + +const _ExtenderType_name = "UnknownYamlExtenderBase64ExtenderRegexExtender" + +var _ExtenderType_index = [...]uint8{0, 7, 19, 33, 46} + +func (i ExtenderType) String() string { + if i < 0 || i >= ExtenderType(len(_ExtenderType_index)-1) { + return "ExtenderType(" + strconv.FormatInt(int64(i), 10) + ")" + } + return _ExtenderType_name[_ExtenderType_index[i]:_ExtenderType_index[i+1]] +} diff --git a/pkg/extras/replacement.go b/pkg/extras/replacement.go new file mode 100644 index 0000000..578a87c --- /dev/null +++ b/pkg/extras/replacement.go @@ -0,0 +1,401 @@ +// Copyright 2021 The Kubernetes Authors. +// SPDX-License-Identifier: Apache-2.0 + +package extras + +import ( + "errors" + "fmt" + "reflect" + "strings" + + "sigs.k8s.io/kustomize/api/konfig" + "sigs.k8s.io/kustomize/api/resmap" + "sigs.k8s.io/kustomize/api/resource" + "sigs.k8s.io/kustomize/api/types" + "sigs.k8s.io/kustomize/kyaml/resid" + kyaml_utils "sigs.k8s.io/kustomize/kyaml/utils" + "sigs.k8s.io/kustomize/kyaml/yaml" +) + +type Filter struct { + Replacements []types.Replacement `json:"replacements,omitempty" yaml:"replacements,omitempty"` +} + +// Filter replaces values of targets with values from sources +func (f Filter) Filter(nodes []*yaml.RNode) ([]*yaml.RNode, error) { + for i, r := range f.Replacements { + if r.Source == nil || r.Targets == nil { + return nil, fmt.Errorf("replacements must specify a source and at least one target") + } + value, err := getReplacement(nodes, &f.Replacements[i]) + if err != nil { + return nil, err + } + nodes, err = applyReplacement(nodes, value, r.Targets) + if err != nil { + return nil, err + } + } + return nodes, nil +} + +func getReplacement(nodes []*yaml.RNode, r *types.Replacement) (*yaml.RNode, error) { + source, err := selectSourceNode(nodes, r.Source) + if err != nil { + return nil, err + } + + if r.Source.FieldPath == "" { + r.Source.FieldPath = types.DefaultReplacementFieldPath + } + fieldPath := kyaml_utils.SmarterPathSplitter(r.Source.FieldPath, ".") + + rn, err := source.Pipe(yaml.Lookup(fieldPath...)) + if err != nil { + return nil, fmt.Errorf("error looking up replacement source: %w", err) + } + if rn.IsNilOrEmpty() { + return nil, fmt.Errorf("fieldPath `%s` is missing for replacement source %s", r.Source.FieldPath, r.Source.ResId) + } + + return getRefinedValue(r.Source.Options, rn) +} + +// selectSourceNode finds the node that matches the selector, returning +// an error if multiple or none are found +func selectSourceNode(nodes []*yaml.RNode, selector *types.SourceSelector) (*yaml.RNode, error) { + var matches []*yaml.RNode + for _, n := range nodes { + ids, err := MakeResIds(n) + if err != nil { + return nil, fmt.Errorf("error getting node IDs: %w", err) + } + for _, id := range ids { + if id.IsSelectedBy(selector.ResId) { + if len(matches) > 0 { + return nil, fmt.Errorf( + "multiple matches for selector %s", selector) + } + matches = append(matches, n) + break + } + } + } + if len(matches) == 0 { + return nil, fmt.Errorf("nothing selected by %s", selector) + } + return matches[0], nil +} + +func getRefinedValue(options *types.FieldOptions, rn *yaml.RNode) (*yaml.RNode, error) { + if options == nil || options.Delimiter == "" { + return rn, nil + } + if rn.YNode().Kind != yaml.ScalarNode { + return nil, fmt.Errorf("delimiter option can only be used with scalar nodes") + } + value := strings.Split(yaml.GetValue(rn), options.Delimiter) + if options.Index >= len(value) || options.Index < 0 { + return nil, fmt.Errorf("options.index %d is out of bounds for value %s", options.Index, yaml.GetValue(rn)) + } + n := rn.Copy() + n.YNode().Value = value[options.Index] + return n, nil +} + +func applyReplacement(nodes []*yaml.RNode, value *yaml.RNode, targetSelectors []*types.TargetSelector) ([]*yaml.RNode, error) { + for _, selector := range targetSelectors { + if selector.Select == nil { + return nil, errors.New("target must specify resources to select") + } + if len(selector.FieldPaths) == 0 { + selector.FieldPaths = []string{types.DefaultReplacementFieldPath} + } + for _, possibleTarget := range nodes { + ids, err := MakeResIds(possibleTarget) + if err != nil { + return nil, err + } + + // filter targets by label and annotation selectors + selectByAnnoAndLabel, err := selectByAnnoAndLabel(possibleTarget, selector) + if err != nil { + return nil, err + } + if !selectByAnnoAndLabel { + continue + } + + // filter targets by matching resource IDs + for i, id := range ids { + if id.IsSelectedBy(selector.Select.ResId) && !rejectId(selector.Reject, &ids[i]) { + err := copyValueToTarget(possibleTarget, value, selector) + if err != nil { + return nil, err + } + break + } + } + } + } + return nodes, nil +} + +func selectByAnnoAndLabel(n *yaml.RNode, t *types.TargetSelector) (bool, error) { + if matchesSelect, err := matchesAnnoAndLabelSelector(n, t.Select); !matchesSelect || err != nil { + return false, err + } + for _, reject := range t.Reject { + if reject.AnnotationSelector == "" && reject.LabelSelector == "" { + continue + } + if m, err := matchesAnnoAndLabelSelector(n, reject); m || err != nil { + return false, err + } + } + return true, nil +} + +func matchesAnnoAndLabelSelector(n *yaml.RNode, selector *types.Selector) (bool, error) { + r := resource.Resource{RNode: *n} + annoMatch, err := r.MatchesAnnotationSelector(selector.AnnotationSelector) + if err != nil { + return false, err + } + labelMatch, err := r.MatchesLabelSelector(selector.LabelSelector) + if err != nil { + return false, err + } + return annoMatch && labelMatch, nil +} + +func rejectId(rejects []*types.Selector, id *resid.ResId) bool { + for _, r := range rejects { + if !r.ResId.IsEmpty() && id.IsSelectedBy(r.ResId) { + return true + } + } + return false +} + +func copyValueToTarget(target *yaml.RNode, value *yaml.RNode, selector *types.TargetSelector) error { + for _, fp := range selector.FieldPaths { + fieldPath := kyaml_utils.SmarterPathSplitter(fp, ".") + extendedPath, err := NewExtendedPath(fieldPath) + if err != nil { + return err + } + create, err := shouldCreateField(selector.Options, extendedPath.resourcePath) + if err != nil { + return err + } + + var targetFields []*yaml.RNode + if create { + createdField, createErr := target.Pipe(yaml.LookupCreate(value.YNode().Kind, extendedPath.resourcePath...)) + if createErr != nil { + return fmt.Errorf("error creating replacement node: %w", createErr) + } + targetFields = append(targetFields, createdField) + } else { + // may return multiple fields, always wrapped in a sequence node + foundFieldSequence, lookupErr := target.Pipe(&yaml.PathMatcher{Path: extendedPath.resourcePath}) + if lookupErr != nil { + return fmt.Errorf("error finding field in replacement target: %w", lookupErr) + } + targetFields, err = foundFieldSequence.Elements() + if err != nil { + return fmt.Errorf("error fetching elements in replacement target: %w", err) + } + } + + for _, t := range targetFields { + if err := setFieldValue(selector.Options, t, value, extendedPath); err != nil { + return err + } + } + + } + return nil +} + +func setFieldValue(options *types.FieldOptions, targetField *yaml.RNode, value *yaml.RNode, extendedPath *ExtendedPath) error { + value = value.Copy() + if options != nil && options.Delimiter != "" { + if extendedPath.HasExtensions() { + return fmt.Errorf("delimiter option cannot be used with extensions") + } + if targetField.YNode().Kind != yaml.ScalarNode { + return fmt.Errorf("delimiter option can only be used with scalar nodes") + } + tv := strings.Split(targetField.YNode().Value, options.Delimiter) + v := yaml.GetValue(value) + // TODO: Add a way to remove an element + switch { + case options.Index < 0: // prefix + tv = append([]string{v}, tv...) + case options.Index >= len(tv): // suffix + tv = append(tv, v) + default: // replace an element + tv[options.Index] = v + } + value.YNode().Value = strings.Join(tv, options.Delimiter) + } + + if targetField.YNode().Kind == yaml.ScalarNode { + return extendedPath.Apply(targetField, value) + } else { + if extendedPath.HasExtensions() { + return fmt.Errorf("path extensions should start at a scalar node") + } + + targetField.SetYNode(value.YNode()) + } + + return nil +} + +func shouldCreateField(options *types.FieldOptions, fieldPath []string) (bool, error) { + if options == nil || !options.Create { + return false, nil + } + // create option is not supported in a wildcard matching + for _, f := range fieldPath { + if f == "*" { + return false, fmt.Errorf("cannot support create option in a multi-value target") + } + } + return true, nil +} + +// Copied + +const ( + BuildAnnotationPreviousKinds = konfig.ConfigAnnoDomain + "/previousKinds" + BuildAnnotationPreviousNames = konfig.ConfigAnnoDomain + "/previousNames" + BuildAnnotationPrefixes = konfig.ConfigAnnoDomain + "/prefixes" + BuildAnnotationSuffixes = konfig.ConfigAnnoDomain + "/suffixes" + BuildAnnotationPreviousNamespaces = konfig.ConfigAnnoDomain + "/previousNamespaces" +) + +// MakeResIds returns all of an RNode's current and previous Ids +func MakeResIds(n *yaml.RNode) ([]resid.ResId, error) { + var result []resid.ResId + apiVersion := n.Field(yaml.APIVersionField) + var group, version string + if apiVersion != nil { + group, version = resid.ParseGroupVersion(yaml.GetValue(apiVersion.Value)) + } + result = append(result, resid.NewResIdWithNamespace( + resid.Gvk{Group: group, Version: version, Kind: n.GetKind()}, n.GetName(), n.GetNamespace()), + ) + prevIds, err := PrevIds(n) + if err != nil { + return nil, err + } + result = append(result, prevIds...) + return result, nil +} + +// PrevIds returns all of an RNode's previous Ids +func PrevIds(n *yaml.RNode) ([]resid.ResId, error) { + var ids []resid.ResId + // TODO: merge previous names and namespaces into one list of + // pairs on one annotation so there is no chance of error + annotations := n.GetAnnotations() + if _, ok := annotations[BuildAnnotationPreviousNames]; !ok { + return nil, nil + } + names := strings.Split(annotations[BuildAnnotationPreviousNames], ",") + ns := strings.Split(annotations[BuildAnnotationPreviousNamespaces], ",") + kinds := strings.Split(annotations[BuildAnnotationPreviousKinds], ",") + // This should never happen + if len(names) != len(ns) || len(names) != len(kinds) { + return nil, fmt.Errorf( + "number of previous names, " + + "number of previous namespaces, " + + "number of previous kinds not equal") + } + for i := range names { + meta, err := n.GetMeta() + if err != nil { + return nil, err + } + group, version := resid.ParseGroupVersion(meta.APIVersion) + gvk := resid.Gvk{ + Group: group, + Version: version, + Kind: kinds[i], + } + ids = append(ids, resid.NewResIdWithNamespace( + gvk, names[i], ns[i])) + } + return ids, nil +} + +// plugin + +// Replace values in targets with values from a source +type ExtendedReplacementTransformerPlugin struct { + ReplacementList []types.ReplacementField `json:"replacements,omitempty" yaml:"replacements,omitempty"` + Replacements []types.Replacement `json:"omitempty" yaml:"omitempty"` +} + +func (p *ExtendedReplacementTransformerPlugin) Config( + h *resmap.PluginHelpers, c []byte) (err error) { + p.ReplacementList = []types.ReplacementField{} + if err := yaml.Unmarshal(c, p); err != nil { + return err + } + + for _, r := range p.ReplacementList { + if r.Path != "" && (r.Source != nil || len(r.Targets) != 0) { + return fmt.Errorf("cannot specify both path and inline replacement") + } + if r.Path != "" { + // load the replacement from the path + content, err := h.Loader().Load(r.Path) + if err != nil { + return err + } + // find if the path contains a a list of replacements or a single replacement + var replacement interface{} + err = yaml.Unmarshal(content, &replacement) + if err != nil { + return err + } + items := reflect.ValueOf(replacement) + switch items.Kind() { + case reflect.Slice: + repl := []types.Replacement{} + if err := yaml.Unmarshal(content, &repl); err != nil { + return err + } + p.Replacements = append(p.Replacements, repl...) + case reflect.Map: + repl := types.Replacement{} + if err := yaml.Unmarshal(content, &repl); err != nil { + return err + } + p.Replacements = append(p.Replacements, repl) + default: + return fmt.Errorf("unsupported replacement type encountered within replacement path: %v", items.Kind()) + } + } else { + // replacement information is already loaded + p.Replacements = append(p.Replacements, r.Replacement) + } + } + return nil +} + +func (p *ExtendedReplacementTransformerPlugin) Transform(m resmap.ResMap) (err error) { + return m.ApplyFilter(Filter{ + Replacements: p.Replacements, + }) +} + +func NewExtendedReplacementTransformerPlugin() resmap.TransformerPlugin { + return &ExtendedReplacementTransformerPlugin{} +} diff --git a/pkg/plugins/factories.go b/pkg/plugins/factories.go index 58041c2..87595e2 100644 --- a/pkg/plugins/factories.go +++ b/pkg/plugins/factories.go @@ -106,7 +106,7 @@ var TransformerFactories = map[BuiltinPluginType]func() resmap.TransformerPlugin PrefixSuffixTransformer: NewMultiTransformer, PrefixTransformer: builtins.NewPrefixTransformerPlugin, SuffixTransformer: builtins.NewSuffixTransformerPlugin, - ReplacementTransformer: builtins.NewReplacementTransformerPlugin, + ReplacementTransformer: extras.NewExtendedReplacementTransformerPlugin, ReplicaCountTransformer: builtins.NewReplicaCountTransformerPlugin, ValueAddTransformer: builtins.NewValueAddTransformerPlugin, // Do not wired SortOrderTransformer as a builtin plugin. diff --git a/tests/compare/argocd.expected.yaml b/tests/patch/expected/argocd.yaml similarity index 100% rename from tests/compare/argocd.expected.yaml rename to tests/patch/expected/argocd.yaml diff --git a/tests/functions/patch-transformer.yaml b/tests/patch/functions/patch-transformer.yaml similarity index 95% rename from tests/functions/patch-transformer.yaml rename to tests/patch/functions/patch-transformer.yaml index 9ad6376..9c656f0 100644 --- a/tests/functions/patch-transformer.yaml +++ b/tests/patch/functions/patch-transformer.yaml @@ -5,7 +5,7 @@ metadata: annotations: config.kubernetes.io/function: | exec: - path: ../krmfnbuiltin + path: ../../krmfnbuiltin patch: |- - op: replace path: /spec/source/repoURL diff --git a/tests/applications/argocd.yaml b/tests/patch/original/argocd.yaml similarity index 100% rename from tests/applications/argocd.yaml rename to tests/patch/original/argocd.yaml diff --git a/tests/replacement/expected/argocd.yaml b/tests/replacement/expected/argocd.yaml new file mode 100644 index 0000000..ae05158 --- /dev/null +++ b/tests/replacement/expected/argocd.yaml @@ -0,0 +1,41 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: argo-cd + namespace: argocd + annotations: + autocloud/local: "true" +spec: + destination: + namespace: argocd + server: https://kubernetes.default.svc + ignoreDifferences: + - group: argoproj.io + jsonPointers: + - /status + kind: Application + project: default + source: + path: packages/argocd + repoURL: https://github.com/antoinemartin/autocloud.git + targetRevision: deploy/citest + helm: + parameters: + - name: common.targetRevision + value: deploy/citest + - name: common.repoURL + value: https://github.com/antoinemartin/autocloud.git + values: | + uninode: true + apps: + enabled: true + common: + targetRevision: deploy/citest + repoURL: https://github.com/antoinemartin/autocloud.git + syncPolicy: + automated: + allowEmpty: true + prune: true + selfHeal: true + syncOptions: + - CreateNamespace=true diff --git a/tests/functions2/01_configmap-generator.yaml b/tests/replacement/functions/01_configmap-generator.yaml similarity index 92% rename from tests/functions2/01_configmap-generator.yaml rename to tests/replacement/functions/01_configmap-generator.yaml index 926742b..0ac9370 100644 --- a/tests/functions2/01_configmap-generator.yaml +++ b/tests/replacement/functions/01_configmap-generator.yaml @@ -8,7 +8,7 @@ metadata: annotations: config.kubernetes.io/function: | exec: - path: ../krmfnbuiltin + path: ../../krmfnbuiltin # When using GitConfigMapGenerator, these are automatically injected literals: - repoURL=https://github.com/antoinemartin/autocloud.git diff --git a/tests/functions2/02_replacement-transformer.yaml b/tests/replacement/functions/02_replacement-transformer.yaml similarity index 84% rename from tests/functions2/02_replacement-transformer.yaml rename to tests/replacement/functions/02_replacement-transformer.yaml index 4d69b70..1f56b17 100644 --- a/tests/functions2/02_replacement-transformer.yaml +++ b/tests/replacement/functions/02_replacement-transformer.yaml @@ -7,7 +7,7 @@ metadata: config.kubernetes.io/prune-local: "true" config.kubernetes.io/function: | exec: - path: ../krmfnbuiltin + path: ../../krmfnbuiltin replacements: - source: kind: ConfigMap @@ -19,6 +19,7 @@ replacements: fieldPaths: - spec.source.repoURL - spec.source.helm.parameters.[name=common.repoURL].value + - spec.source.helm.values.!!yaml.common.repoURL - source: kind: ConfigMap fieldPath: data.targetRevision @@ -29,3 +30,4 @@ replacements: fieldPaths: - spec.source.targetRevision - spec.source.helm.parameters.[name=common.targetRevision].value + - spec.source.helm.values.!!yaml.common.targetRevision diff --git a/tests/compare/argocd.original.yaml b/tests/replacement/original/argocd.yaml similarity index 80% rename from tests/compare/argocd.original.yaml rename to tests/replacement/original/argocd.yaml index e88b9e7..3c07fc6 100644 --- a/tests/compare/argocd.original.yaml +++ b/tests/replacement/original/argocd.yaml @@ -25,6 +25,13 @@ spec: value: main - name: common.repoURL value: https://github.com/anotherproject/anothergit + values: | + uninode: true + apps: + enabled: true + common: + targetRevision: main + repoURL: https://github.com/anotherproject/anothergit syncPolicy: automated: allowEmpty: true diff --git a/tests/test_krmfnbuiltin.sh b/tests/test_krmfnbuiltin.sh index 969731d..ff78ecb 100755 --- a/tests/test_krmfnbuiltin.sh +++ b/tests/test_krmfnbuiltin.sh @@ -1,21 +1,22 @@ #!/bin/bash # DEPENDENCEIS -# sops # kustomize -# age # yq #set -uexo pipefail set -e pipefail -trap "cp compare/argocd.original.yaml applications/argocd.yaml" EXIT +trap "rm -rf {patch,replacement}/applications" EXIT -echo "Running kustomize with patch transformer..." -kustomize fn run --enable-exec --fn-path functions applications -diff <(yq eval -P compare/argocd.expected.yaml) <(yq eval -P applications/argocd.yaml) -cp compare/argocd.original.yaml applications/argocd.yaml -echo "Running kustomize with replacement transformer..." -kustomize fn run --enable-exec --fn-path functions2 applications -diff <(yq eval -P compare/argocd.expected.yaml) <(yq eval -P applications/argocd.yaml) + +for d in patch replacement; do + echo "Running Test in $d..." + cd $d + rm -rf appllications + cp -r original applications + kustomize fn run --enable-exec --fn-path functions applications + diff <(yq eval -P expected/argocd.yaml) <(yq eval -P applications/argocd.yaml) + cd .. +done echo "Done ok 🎉" From 10cb26b91bfc42746a409e29ddb3676959e2ceb9 Mon Sep 17 00:00:00 2001 From: Antoine Martin Date: Tue, 24 Jan 2023 16:24:53 +0000 Subject: [PATCH 02/10] =?UTF-8?q?=E2=9C=A8=20Added=20json=20extended=20enc?= =?UTF-8?q?oding?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/extras/extender.go | 122 +++++++++++++++++++++++++----- pkg/extras/extender_test.go | 100 ++++++++++++++++++++++++ pkg/extras/extendertype_string.go | 5 +- 3 files changed, 205 insertions(+), 22 deletions(-) diff --git a/pkg/extras/extender.go b/pkg/extras/extender.go index abb9851..509f2db 100644 --- a/pkg/extras/extender.go +++ b/pkg/extras/extender.go @@ -3,6 +3,7 @@ package extras import ( "bytes" "encoding/base64" + "encoding/json" "fmt" "regexp" "strconv" @@ -12,6 +13,7 @@ import ( "golang.org/x/text/language" "sigs.k8s.io/kustomize/kyaml/errors" "sigs.k8s.io/kustomize/kyaml/kio" + "sigs.k8s.io/kustomize/kyaml/kio/kioutil" "sigs.k8s.io/kustomize/kyaml/yaml" ) @@ -45,6 +47,7 @@ const ( YamlExtender Base64Extender RegexExtender + JsonExtender ) var stringToExtenderTypeMap map[string]ExtenderType @@ -89,8 +92,7 @@ type yamlExtender struct { node *yaml.RNode } -func (e *yamlExtender) SetPayload(payload []byte) error { - +func readPayload(payload []byte) (*yaml.RNode, error) { nodes, err := (&kio.ByteReader{ Reader: bytes.NewBuffer(payload), OmitReaderAnnotations: false, @@ -99,28 +101,36 @@ func (e *yamlExtender) SetPayload(payload []byte) error { }).Read() if err != nil { - return errors.WrapPrefixf(err, "while reading yaml payload") + return nil, errors.WrapPrefixf(err, "while reading payload") } - e.node = nodes[0] - return nil + return nodes[0], nil } -func (e *yamlExtender) GetPayload() ([]byte, error) { +func (e *yamlExtender) SetPayload(payload []byte) (err error) { + e.node, err = readPayload(payload) + return +} + +func getNodeBytes(nodes []*yaml.RNode) ([]byte, error) { var b bytes.Buffer - err := (&kio.ByteWriter{Writer: &b}).Write([]*yaml.RNode{e.node}) + err := (&kio.ByteWriter{Writer: &b}).Write(nodes) return b.Bytes(), err } -func (e *yamlExtender) LookupNode() *yaml.RNode { - seqNode, err := e.node.Pipe(yaml.Lookup(yaml.BareSeqNodeWrappingKey)) +func (e *yamlExtender) GetPayload() ([]byte, error) { + return getNodeBytes([]*yaml.RNode{e.node}) +} + +func LookupNode(node *yaml.RNode) *yaml.RNode { + seqNode, err := node.Pipe(yaml.Lookup(yaml.BareSeqNodeWrappingKey)) if err == nil && !seqNode.IsNilOrEmpty() { return seqNode } - return e.node + return node } -func (e *yamlExtender) Lookup(path []string) ([]*yaml.RNode, error) { - node, err := e.LookupNode().Pipe(&yaml.PathMatcher{Path: path}) +func Lookup(node *yaml.RNode, path []string) ([]*yaml.RNode, error) { + node, err := LookupNode(node).Pipe(&yaml.PathMatcher{Path: path}) if err != nil { return nil, errors.WrapPrefixf(err, "while getting path %s", strings.Join(path, ".")) } @@ -129,7 +139,7 @@ func (e *yamlExtender) Lookup(path []string) ([]*yaml.RNode, error) { } func (e *yamlExtender) Get(path []string) ([]byte, error) { - targetFields, err := e.Lookup(path) + targetFields, err := Lookup(e.node, path) if err != nil { return nil, fmt.Errorf("error fetching elements in replacement target: %w", err) } @@ -138,17 +148,12 @@ func (e *yamlExtender) Get(path []string) ([]byte, error) { return []byte(targetFields[0].YNode().Value), nil } - var b bytes.Buffer - err = (&kio.ByteWriter{Writer: &b}).Write(targetFields) - if err != nil { - return nil, errors.WrapPrefixf(err, "while serializing path %s", strings.Join(path, ".")) - } - return b.Bytes(), err + return getNodeBytes(targetFields) } func (e *yamlExtender) Set(path []string, value any) error { - targetFields, err := e.Lookup(path) + targetFields, err := Lookup(e.node, path) if err != nil { return fmt.Errorf("error fetching elements in replacement target: %w", err) } @@ -280,6 +285,82 @@ func NewRegexExtender() Extender { return ®exExtender{} } +/////// +// JSON +/////// + +type jsonExtender struct { + node *yaml.RNode +} + +func (e *jsonExtender) SetPayload(payload []byte) (err error) { + e.node, err = readPayload(payload) + return +} + +func getJSONPayload(node *yaml.RNode) ([]byte, error) { + var b bytes.Buffer + if node.YNode().Kind == yaml.MappingNode { + node = node.Copy() + node.Pipe(yaml.ClearAnnotation(kioutil.IndexAnnotation)) + node.Pipe(yaml.ClearAnnotation(kioutil.LegacyIndexAnnotation)) + node.Pipe(yaml.ClearAnnotation(kioutil.SeqIndentAnnotation)) + yaml.ClearEmptyAnnotations(node) + } + encoder := json.NewEncoder(&b) + encoder.SetIndent("", " ") + err := errors.Wrap(encoder.Encode(node)) + + return b.Bytes(), err +} + +func (e *jsonExtender) GetPayload() ([]byte, error) { + return getJSONPayload(LookupNode(e.node)) +} + +func (e *jsonExtender) Get(path []string) ([]byte, error) { + targetFields, err := Lookup(e.node, path) + if err != nil { + return nil, errors.WrapPrefixf(err, "error fetching elements in replacement target") + } + + if len(targetFields) > 1 { + return nil, fmt.Errorf("path %s returned %d items. Expected one", strings.Join(path, "."), len(targetFields)) + } + + target := targetFields[0] + if target.YNode().Kind == yaml.ScalarNode { + return []byte(target.YNode().Value), nil + } + + return getJSONPayload(target) +} + +func (e *jsonExtender) Set(path []string, value any) error { + + targetFields, err := Lookup(e.node, path) + if err != nil { + return fmt.Errorf("error fetching elements in replacement target: %w", err) + } + + for _, t := range targetFields { + if t.YNode().Kind == yaml.ScalarNode { + t.YNode().Value = string(GetByteValue(value)) + } else { + if v, ok := value.(*yaml.Node); ok { + t.SetYNode(v) + } else { + return fmt.Errorf("setting non json object in place of object of type %s at path %s", t.YNode().Tag, strings.Join(path, ".")) + } + } + } + return nil +} + +func NewJsonExtender() Extender { + return &jsonExtender{} +} + //////////// // Factories //////////// @@ -288,6 +369,7 @@ var ExtenderFactories = map[ExtenderType]func() Extender{ YamlExtender: NewYamlExtender, Base64Extender: NewBase64Extender, RegexExtender: NewRegexExtender, + JsonExtender: NewJsonExtender, } func (path *ExtendedSegment) Extender(payload []byte) (Extender, error) { diff --git a/pkg/extras/extender_test.go b/pkg/extras/extender_test.go index f479a8f..b19973e 100644 --- a/pkg/extras/extender_test.go +++ b/pkg/extras/extender_test.go @@ -273,6 +273,106 @@ common: require.Equal(expected, b.String(), sString, "replacement failed") } +func (s *ExtenderTestSuite) TestJsonExtender() { + require := s.Require() + source := `{ + "common": { + "targetRevision": "main" + }, + "uninode": true, + "apps": { + "enabled": true + } +}` + expected := `{ + "apps": { + "enabled": true + }, + "common": { + "targetRevision": "deploy/citest" + }, + "uninode": true +} +` + + p := `!!json.common.targetRevision` + path := kyaml_utils.SmarterPathSplitter(p, ".") + + extensions := []*ExtendedSegment{} + prefix, err := splitExtendedPath(path, &extensions) + require.NoError(err) + require.Len(prefix, 0, "There should be no prefix") + require.Len(extensions, 1, "There should be 2 extensions") + require.Equal("json", extensions[0].Encoding, "The first extension should be json") + + jsonXP := extensions[0] + jsonExt, err := jsonXP.Extender([]byte(source)) + require.NoError(err) + value, err := jsonExt.Get(jsonXP.Path) + require.NoError(err) + require.Equal("main", string(value), "error fetching value") + require.NoError(jsonExt.Set(jsonXP.Path, []byte("deploy/citest"))) + + modified, err := jsonExt.GetPayload() + require.NoError(err) + require.Equal(expected, string(modified), "final json") + + value, err = jsonExt.Get(jsonXP.Path) + require.NoError(err) + require.Equal("deploy/citest", string(value), "error fetching changed value") +} + +func (s *ExtenderTestSuite) TestJsonArrayExtender() { + require := s.Require() + source := `[ + { + "name": "targetRevision", + "value": "main" + }, + { + "name": "repoURL", + "value": "https://github.com/kaweezle/example.git" + } +]` + expected := `[ + { + "name": "targetRevision", + "value": "deploy/citest" + }, + { + "name": "repoURL", + "value": "https://github.com/kaweezle/example.git" + } +] +` + + p := `!!json.[name=targetRevision].value` + path := kyaml_utils.SmarterPathSplitter(p, ".") + + extensions := []*ExtendedSegment{} + prefix, err := splitExtendedPath(path, &extensions) + require.NoError(err) + require.Len(prefix, 0, "There should be no prefix") + require.Len(extensions, 1, "There should be 2 extensions") + require.Equal("json", extensions[0].Encoding, "The first extension should be json") + + jsonXP := extensions[0] + jsonExt, err := jsonXP.Extender([]byte(source)) + require.NoError(err) + value, err := jsonExt.Get(jsonXP.Path) + require.NoError(err) + require.Equal("main", string(value), "error fetching value") + require.NoError(jsonExt.Set(jsonXP.Path, []byte("deploy/citest"))) + + modified, err := jsonExt.GetPayload() + require.NoError(err) + require.Equal(expected, string(modified), "final json") + + value, err = jsonExt.Get(jsonXP.Path) + require.NoError(err) + require.Equal("deploy/citest", string(value), "error fetching changed value") +} + func TestExtender(t *testing.T) { suite.Run(t, new(ExtenderTestSuite)) } diff --git a/pkg/extras/extendertype_string.go b/pkg/extras/extendertype_string.go index a613560..4188249 100644 --- a/pkg/extras/extendertype_string.go +++ b/pkg/extras/extendertype_string.go @@ -12,11 +12,12 @@ func _() { _ = x[YamlExtender-1] _ = x[Base64Extender-2] _ = x[RegexExtender-3] + _ = x[JsonExtender-4] } -const _ExtenderType_name = "UnknownYamlExtenderBase64ExtenderRegexExtender" +const _ExtenderType_name = "UnknownYamlExtenderBase64ExtenderRegexExtenderJsonExtender" -var _ExtenderType_index = [...]uint8{0, 7, 19, 33, 46} +var _ExtenderType_index = [...]uint8{0, 7, 19, 33, 46, 58} func (i ExtenderType) String() string { if i < 0 || i >= ExtenderType(len(_ExtenderType_index)-1) { From 5cf91099444673fda65528be9c0a0e6d85fdcb10 Mon Sep 17 00:00:00 2001 From: Antoine Martin Date: Tue, 24 Jan 2023 18:14:20 +0000 Subject: [PATCH 03/10] =?UTF-8?q?=E2=9C=A8=20Add=20TOML=20extender?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- go.mod | 1 + go.sum | 2 + pkg/extras/extender.go | 82 +++++++++++++++++++++++++++++++ pkg/extras/extender_test.go | 45 +++++++++++++++++ pkg/extras/extendertype_string.go | 5 +- 5 files changed, 133 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 1b0fb6c..388b4b0 100644 --- a/go.mod +++ b/go.mod @@ -41,6 +41,7 @@ require ( github.com/mailru/easyjson v0.7.7 // indirect github.com/mitchellh/mapstructure v1.4.3 // indirect github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 // indirect + github.com/pelletier/go-toml/v2 v2.0.6 github.com/pjbgf/sha1cd v0.2.3 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect diff --git a/go.sum b/go.sum index 836e3d7..eb4f259 100644 --- a/go.sum +++ b/go.sum @@ -129,6 +129,8 @@ github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8m github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= +github.com/pelletier/go-toml/v2 v2.0.6 h1:nrzqCb7j9cDFj2coyLNLaZuJTLjWjlaz6nvTvIwycIU= +github.com/pelletier/go-toml/v2 v2.0.6/go.mod h1:eumQOmlWiOPt5WriQQqoM5y18pDHwha2N+QD+EUNTek= github.com/pjbgf/sha1cd v0.2.3 h1:uKQP/7QOzNtKYH7UTohZLcjF5/55EnTw0jO/Ru4jZwI= github.com/pjbgf/sha1cd v0.2.3/go.mod h1:HOK9QrgzdHpbc2Kzip0Q1yi3M2MFGPADtR6HjG65m5M= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= diff --git a/pkg/extras/extender.go b/pkg/extras/extender.go index 509f2db..2853c3c 100644 --- a/pkg/extras/extender.go +++ b/pkg/extras/extender.go @@ -9,6 +9,7 @@ import ( "strconv" "strings" + "github.com/pelletier/go-toml/v2" "golang.org/x/text/cases" "golang.org/x/text/language" "sigs.k8s.io/kustomize/kyaml/errors" @@ -48,6 +49,7 @@ const ( Base64Extender RegexExtender JsonExtender + TomlExtender ) var stringToExtenderTypeMap map[string]ExtenderType @@ -361,6 +363,85 @@ func NewJsonExtender() Extender { return &jsonExtender{} } +/////// +// TOML +/////// + +type tomlExtender struct { + node *yaml.RNode +} + +func (e *tomlExtender) SetPayload(payload []byte) error { + + m := map[string]interface{}{} + err := toml.Unmarshal(payload, &m) + if err != nil { + return errors.WrapPrefixf(err, "while un-marshalling toml") + } + + e.node, err = yaml.FromMap(m) + if err != nil { + return errors.WrapPrefixf(err, "while converting into yaml") + } + + return nil +} + +func getTOMLPayload(node *yaml.RNode) ([]byte, error) { + m, err := node.Map() + if err != nil { + return nil, errors.WrapPrefixf(err, "while encoding to map") + } + return toml.Marshal(m) +} + +func (e *tomlExtender) GetPayload() ([]byte, error) { + return getTOMLPayload(e.node) +} + +func (e *tomlExtender) Get(path []string) ([]byte, error) { + targetFields, err := Lookup(e.node, path) + if err != nil { + return nil, errors.WrapPrefixf(err, "error fetching elements in replacement target") + } + + if len(targetFields) > 1 { + return nil, fmt.Errorf("path %s returned %d items. Expected one", strings.Join(path, "."), len(targetFields)) + } + + target := targetFields[0] + if target.YNode().Kind == yaml.ScalarNode { + return []byte(target.YNode().Value), nil + } + + return getTOMLPayload(target) +} + +func (e *tomlExtender) Set(path []string, value any) error { + + targetFields, err := Lookup(e.node, path) + if err != nil { + return fmt.Errorf("error fetching elements in replacement target: %w", err) + } + + for _, t := range targetFields { + if t.YNode().Kind == yaml.ScalarNode { + t.YNode().Value = string(GetByteValue(value)) + } else { + if v, ok := value.(*yaml.Node); ok { + t.SetYNode(v) + } else { + return fmt.Errorf("setting non toml object in place of object of type %s at path %s", t.YNode().Tag, strings.Join(path, ".")) + } + } + } + return nil +} + +func NewTomlExtender() Extender { + return &tomlExtender{} +} + //////////// // Factories //////////// @@ -370,6 +451,7 @@ var ExtenderFactories = map[ExtenderType]func() Extender{ Base64Extender: NewBase64Extender, RegexExtender: NewRegexExtender, JsonExtender: NewJsonExtender, + TomlExtender: NewTomlExtender, } func (path *ExtendedSegment) Extender(payload []byte) (Extender, error) { diff --git a/pkg/extras/extender_test.go b/pkg/extras/extender_test.go index b19973e..a9661c6 100644 --- a/pkg/extras/extender_test.go +++ b/pkg/extras/extender_test.go @@ -373,6 +373,51 @@ func (s *ExtenderTestSuite) TestJsonArrayExtender() { require.Equal("deploy/citest", string(value), "error fetching changed value") } +func (s *ExtenderTestSuite) TestTomlExtender() { + require := s.Require() + source := ` +uninode = true +[common] +targetRevision = 'main' +[apps] +enabled = true +` + expected := `uninode = true + +[apps] +enabled = true + +[common] +targetRevision = 'deploy/citest' +` + + p := `!!toml.common.targetRevision` + path := kyaml_utils.SmarterPathSplitter(p, ".") + + extensions := []*ExtendedSegment{} + prefix, err := splitExtendedPath(path, &extensions) + require.NoError(err) + require.Len(prefix, 0, "There should be no prefix") + require.Len(extensions, 1, "There should be 2 extensions") + require.Equal("toml", extensions[0].Encoding, "The first extension should be toml") + + tomlXP := extensions[0] + tomlExt, err := tomlXP.Extender([]byte(source)) + require.NoError(err) + value, err := tomlExt.Get(tomlXP.Path) + require.NoError(err) + require.Equal("main", string(value), "error fetching value") + require.NoError(tomlExt.Set(tomlXP.Path, []byte("deploy/citest"))) + + modified, err := tomlExt.GetPayload() + require.NoError(err) + require.Equal(expected, string(modified), "final toml") + + value, err = tomlExt.Get(tomlXP.Path) + require.NoError(err) + require.Equal("deploy/citest", string(value), "error fetching changed value") +} + func TestExtender(t *testing.T) { suite.Run(t, new(ExtenderTestSuite)) } diff --git a/pkg/extras/extendertype_string.go b/pkg/extras/extendertype_string.go index 4188249..0988f23 100644 --- a/pkg/extras/extendertype_string.go +++ b/pkg/extras/extendertype_string.go @@ -13,11 +13,12 @@ func _() { _ = x[Base64Extender-2] _ = x[RegexExtender-3] _ = x[JsonExtender-4] + _ = x[TomlExtender-5] } -const _ExtenderType_name = "UnknownYamlExtenderBase64ExtenderRegexExtenderJsonExtender" +const _ExtenderType_name = "UnknownYamlExtenderBase64ExtenderRegexExtenderJsonExtenderTomlExtender" -var _ExtenderType_index = [...]uint8{0, 7, 19, 33, 46, 58} +var _ExtenderType_index = [...]uint8{0, 7, 19, 33, 46, 58, 70} func (i ExtenderType) String() string { if i < 0 || i >= ExtenderType(len(_ExtenderType_index)-1) { From 8c83f71b0d5229b6e88976a5cb3d6ab239f64a10 Mon Sep 17 00:00:00 2001 From: Antoine Martin Date: Tue, 24 Jan 2023 18:45:41 +0000 Subject: [PATCH 04/10] =?UTF-8?q?=E2=9C=A8=20Added=20ini=20file=20extender?= =?UTF-8?q?=20support?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- go.mod | 1 + go.sum | 2 ++ pkg/extras/extender.go | 59 +++++++++++++++++++++++++++++++ pkg/extras/extender_test.go | 45 +++++++++++++++++++++++ pkg/extras/extendertype_string.go | 5 +-- 5 files changed, 110 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 388b4b0..fa7aa89 100644 --- a/go.mod +++ b/go.mod @@ -28,6 +28,7 @@ require ( github.com/go-errors/errors v1.4.2 // indirect github.com/go-git/gcfg v1.5.0 // indirect github.com/go-git/go-billy/v5 v5.4.0 // indirect + github.com/go-ini/ini v1.67.0 github.com/go-openapi/jsonpointer v0.19.5 // indirect github.com/go-openapi/jsonreference v0.19.5 // indirect github.com/go-openapi/swag v0.22.3 // indirect diff --git a/go.sum b/go.sum index eb4f259..c88ca1f 100644 --- a/go.sum +++ b/go.sum @@ -50,6 +50,8 @@ github.com/go-git/go-git-fixtures/v4 v4.3.1 h1:y5z6dd3qi8Hl+stezc8p3JxDkoTRqMAlK github.com/go-git/go-git-fixtures/v4 v4.3.1/go.mod h1:8LHG1a3SRW71ettAD/jW13h8c6AqjVSeL11RAdgaqpo= github.com/go-git/go-git/v5 v5.5.2 h1:v8lgZa5k9ylUw+OR/roJHTxR4QItsNFI5nKtAXFuynw= github.com/go-git/go-git/v5 v5.5.2/go.mod h1:BE5hUJ5yaV2YMxhmaP4l6RBQ08kMxKSPD4BlxtH7OjI= +github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= +github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= diff --git a/pkg/extras/extender.go b/pkg/extras/extender.go index 2853c3c..fd451ae 100644 --- a/pkg/extras/extender.go +++ b/pkg/extras/extender.go @@ -9,6 +9,7 @@ import ( "strconv" "strings" + "github.com/go-ini/ini" "github.com/pelletier/go-toml/v2" "golang.org/x/text/cases" "golang.org/x/text/language" @@ -50,6 +51,7 @@ const ( RegexExtender JsonExtender TomlExtender + IniExtender ) var stringToExtenderTypeMap map[string]ExtenderType @@ -442,6 +444,62 @@ func NewTomlExtender() Extender { return &tomlExtender{} } +/////// +// TOML +/////// + +type iniExtender struct { + file *ini.File +} + +func (e *iniExtender) SetPayload(payload []byte) (err error) { + + e.file, err = ini.Load(payload) + return err +} + +func (e *iniExtender) GetPayload() ([]byte, error) { + var b bytes.Buffer + _, err := e.file.WriteTo(&b) + return b.Bytes(), err +} + +func (e *iniExtender) keyFromPath(path []string) (*ini.Key, error) { + if len(path) < 1 || len(path) > 2 { + return nil, fmt.Errorf("invalid path length: %d", len(path)) + } + section := "" + key := path[0] + if len(path) == 2 { + section = key + key = path[1] + } + return e.file.Section(section).Key(key), nil +} + +func (e *iniExtender) Get(path []string) ([]byte, error) { + k, err := e.keyFromPath(path) + if err != nil { + return nil, fmt.Errorf("while getting key at path %s", strings.Join(path, ".")) + } + return []byte(k.String()), nil +} + +func (e *iniExtender) Set(path []string, value any) error { + k, err := e.keyFromPath(path) + if err != nil { + return fmt.Errorf("while getting key at path %s", strings.Join(path, ".")) + } + + k.SetValue(string(GetByteValue(value))) + + return nil +} + +func NewIniExtender() Extender { + return &iniExtender{} +} + //////////// // Factories //////////// @@ -452,6 +510,7 @@ var ExtenderFactories = map[ExtenderType]func() Extender{ RegexExtender: NewRegexExtender, JsonExtender: NewJsonExtender, TomlExtender: NewTomlExtender, + IniExtender: NewIniExtender, } func (path *ExtendedSegment) Extender(payload []byte) (Extender, error) { diff --git a/pkg/extras/extender_test.go b/pkg/extras/extender_test.go index a9661c6..88bdb60 100644 --- a/pkg/extras/extender_test.go +++ b/pkg/extras/extender_test.go @@ -418,6 +418,51 @@ targetRevision = 'deploy/citest' require.Equal("deploy/citest", string(value), "error fetching changed value") } +func (s *ExtenderTestSuite) TestIniExtender() { + require := s.Require() + source := ` +uninode = true +[common] +targetRevision = main +[apps] +enabled = true +` + expected := `uninode = true + +[common] +targetRevision = deploy/citest + +[apps] +enabled = true +` + + p := `!!ini.common.targetRevision` + path := kyaml_utils.SmarterPathSplitter(p, ".") + + extensions := []*ExtendedSegment{} + prefix, err := splitExtendedPath(path, &extensions) + require.NoError(err) + require.Len(prefix, 0, "There should be no prefix") + require.Len(extensions, 1, "There should be 2 extensions") + require.Equal("ini", extensions[0].Encoding, "The first extension should be ini") + + iniXP := extensions[0] + iniExt, err := iniXP.Extender([]byte(source)) + require.NoError(err) + value, err := iniExt.Get(iniXP.Path) + require.NoError(err) + require.Equal("main", string(value), "error fetching value") + require.NoError(iniExt.Set(iniXP.Path, []byte("deploy/citest"))) + + modified, err := iniExt.GetPayload() + require.NoError(err) + require.Equal(expected, string(modified), "final ini") + + value, err = iniExt.Get(iniXP.Path) + require.NoError(err) + require.Equal("deploy/citest", string(value), "error fetching changed value") +} + func TestExtender(t *testing.T) { suite.Run(t, new(ExtenderTestSuite)) } diff --git a/pkg/extras/extendertype_string.go b/pkg/extras/extendertype_string.go index 0988f23..76aaeda 100644 --- a/pkg/extras/extendertype_string.go +++ b/pkg/extras/extendertype_string.go @@ -14,11 +14,12 @@ func _() { _ = x[RegexExtender-3] _ = x[JsonExtender-4] _ = x[TomlExtender-5] + _ = x[IniExtender-6] } -const _ExtenderType_name = "UnknownYamlExtenderBase64ExtenderRegexExtenderJsonExtenderTomlExtender" +const _ExtenderType_name = "UnknownYamlExtenderBase64ExtenderRegexExtenderJsonExtenderTomlExtenderIniExtender" -var _ExtenderType_index = [...]uint8{0, 7, 19, 33, 46, 58, 70} +var _ExtenderType_index = [...]uint8{0, 7, 19, 33, 46, 58, 70, 81} func (i ExtenderType) String() string { if i < 0 || i >= ExtenderType(len(_ExtenderType_index)-1) { From daa49c6cfe70816cf4f4bd158b42076b4323cdf8 Mon Sep 17 00:00:00 2001 From: Antoine Martin Date: Wed, 25 Jan 2023 00:31:48 +0000 Subject: [PATCH 05/10] =?UTF-8?q?=F0=9F=93=9D=20Document=20extended=20path?= =?UTF-8?q?s=20(go=20doc)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- go.mod | 2 +- main.go | 40 +---- pkg/extras/GitConfigMapGenerator.go | 17 ++ pkg/extras/doc.go | 49 +++++ pkg/extras/extender.go | 270 +++++++++++++++++++++++----- pkg/extras/extender_test.go | 4 +- pkg/extras/replacement.go | 64 ++++--- pkg/plugins/doc.go | 4 + pkg/utils/constants.go | 15 ++ pkg/utils/doc.go | 5 + pkg/utils/utils.go | 32 ++++ 11 files changed, 391 insertions(+), 111 deletions(-) create mode 100644 pkg/extras/doc.go create mode 100644 pkg/plugins/doc.go create mode 100644 pkg/utils/constants.go create mode 100644 pkg/utils/doc.go create mode 100644 pkg/utils/utils.go diff --git a/go.mod b/go.mod index fa7aa89..72116b0 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,6 @@ require ( github.com/go-git/go-git/v5 v5.5.2 github.com/lithammer/dedent v1.1.0 github.com/stretchr/testify v1.8.1 - golang.org/x/text v0.6.0 golang.org/x/tools v0.5.0 sigs.k8s.io/kustomize/api v0.12.1 sigs.k8s.io/kustomize/kyaml v0.13.10 @@ -56,6 +55,7 @@ require ( golang.org/x/mod v0.7.0 // indirect golang.org/x/net v0.5.0 // indirect golang.org/x/sys v0.4.0 // indirect + golang.org/x/text v0.6.0 // indirect google.golang.org/protobuf v1.28.0 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect diff --git a/main.go b/main.go index 922370d..0915f19 100644 --- a/main.go +++ b/main.go @@ -5,8 +5,8 @@ import ( "os" "github.com/kaweezle/krmfnbuiltin/pkg/plugins" + "github.com/kaweezle/krmfnbuiltin/pkg/utils" - "sigs.k8s.io/kustomize/api/konfig" "sigs.k8s.io/kustomize/api/resmap" "sigs.k8s.io/kustomize/api/resource" "sigs.k8s.io/kustomize/kyaml/errors" @@ -18,42 +18,6 @@ import ( "sigs.k8s.io/kustomize/kyaml/yaml" ) -const ( - // build annotations - BuildAnnotationPreviousKinds = konfig.ConfigAnnoDomain + "/previousKinds" - BuildAnnotationPreviousNames = konfig.ConfigAnnoDomain + "/previousNames" - BuildAnnotationPrefixes = konfig.ConfigAnnoDomain + "/prefixes" - BuildAnnotationSuffixes = konfig.ConfigAnnoDomain + "/suffixes" - BuildAnnotationPreviousNamespaces = konfig.ConfigAnnoDomain + "/previousNamespaces" - BuildAnnotationsRefBy = konfig.ConfigAnnoDomain + "/refBy" - BuildAnnotationsGenBehavior = konfig.ConfigAnnoDomain + "/generatorBehavior" - BuildAnnotationsGenAddHashSuffix = konfig.ConfigAnnoDomain + "/needsHashSuffix" -) - -var BuildAnnotations = []string{ - BuildAnnotationPreviousKinds, - BuildAnnotationPreviousNames, - BuildAnnotationPrefixes, - BuildAnnotationSuffixes, - BuildAnnotationPreviousNamespaces, - BuildAnnotationsRefBy, - BuildAnnotationsGenBehavior, - BuildAnnotationsGenAddHashSuffix, -} - -func RemoveBuildAnnotations(r *resource.Resource) { - annotations := r.GetAnnotations() - if len(annotations) == 0 { - return - } - for _, a := range BuildAnnotations { - delete(annotations, a) - } - if err := r.SetAnnotations(annotations); err != nil { - panic(err) - } -} - func main() { var processor framework.ResourceListProcessorFunc = func(rl *framework.ResourceList) error { @@ -95,7 +59,7 @@ func main() { } for _, r := range rm.Resources() { - RemoveBuildAnnotations(r) + utils.RemoveBuildAnnotations(r) } rl.Items = rm.ToRNodeSlice() diff --git a/pkg/extras/GitConfigMapGenerator.go b/pkg/extras/GitConfigMapGenerator.go index 5ee2665..b67f0c8 100644 --- a/pkg/extras/GitConfigMapGenerator.go +++ b/pkg/extras/GitConfigMapGenerator.go @@ -11,13 +11,28 @@ import ( "sigs.k8s.io/yaml" ) +// GitConfigMapGeneratorPlugin Generates a config map that includes two +// properties of the current git repository: +// +// - repoURL contains the URL or the remote specified by remoteName. by +// default, it takes the URL of the remote named "origin". +// - targetRevision contains the name of the current branch. +// +// This generator is useful in transformations that use those values, like for +// instance Argo CD application customization. +// +// Information about the configuration can be found in the [kustomize doc]. +// +// [kustomize doc]: https://kubectl.docs.kubernetes.io/references/kustomize/builtins/#_configmapgenerator_ type GitConfigMapGeneratorPlugin struct { h *resmap.PluginHelpers types.ObjectMeta `json:"metadata,omitempty" yaml:"metadata,omitempty" protobuf:"bytes,1,opt,name=metadata"` types.ConfigMapArgs + // The name of the remote which URL to include. defaults to "origin". RemoteName string `json:"remoteName,omitempty" yaml:"remoteName,omitempty"` } +// Config configures the generator with the functionConfig passed in config. func (p *GitConfigMapGeneratorPlugin) Config(h *resmap.PluginHelpers, config []byte) (err error) { p.ConfigMapArgs = types.ConfigMapArgs{} err = yaml.Unmarshal(config, p) @@ -31,6 +46,7 @@ func (p *GitConfigMapGeneratorPlugin) Config(h *resmap.PluginHelpers, config []b return } +// Generate generates the config map func (p *GitConfigMapGeneratorPlugin) Generate() (resmap.ResMap, error) { // Add git repository properties @@ -63,6 +79,7 @@ func (p *GitConfigMapGeneratorPlugin) Generate() (resmap.ResMap, error) { kv.NewLoader(p.h.Loader(), p.h.Validator()), p.ConfigMapArgs) } +// NewGitConfigMapGeneratorPlugin returns a newly created GitConfigMapGenerator. func NewGitConfigMapGeneratorPlugin() resmap.GeneratorPlugin { return &GitConfigMapGeneratorPlugin{} } diff --git a/pkg/extras/doc.go b/pkg/extras/doc.go new file mode 100644 index 0000000..9ee51ba --- /dev/null +++ b/pkg/extras/doc.go @@ -0,0 +1,49 @@ +/* +Package extras contains additional utility transformers and generators. + +[GitConfigMapGeneratorPlugin] is identical to ConfigMapGeneratorPlugin +but automatically creates two properties when run inside a git repository: + + - repoURL gives the URL of the origin remote. + - targetRevision gives the current branch. + +[ExtendedReplacementTransformerPlugin] is a copy of ReplacementTransformerPlugin +that provides extended target paths into embedded data structures. For instance, +consider the following resource snippet: + + helm: + parameters: + - name: common.targetRevision + # This resource is accessible by traditional transformer + value: deploy/citest + - name: common.repoURL + value: https://github.com/antoinemartin/autocloud.git + values: | + uninode: true + apps: + enabled: true + common: + # This embedded resource is not accessible + targetRevision: deploy/citest + repoURL: https://github.com/antoinemartin/autocloud.git + +In the above, the common.targetRevision property of the yaml embedded in the +spec.source.helm.values property is not accessible with the traditional +ReplacementTransformerPlugin. With the extended transformer, you can target +it with: + + fieldPaths: + - spec.source.helm.parameters.[name=common.targetRevision].value + - spec.source.helm.values.!!yaml.common.targetRevision + +Note the use of !!yaml to designate the encoding of the embedded structure. The +extended transformer supports the following encodings: + + - YAML + - JSON + - TOML + - INI + - base64 + - Plain text (with Regexp) +*/ +package extras diff --git a/pkg/extras/extender.go b/pkg/extras/extender.go index fd451ae..ccd50ca 100644 --- a/pkg/extras/extender.go +++ b/pkg/extras/extender.go @@ -11,26 +11,46 @@ import ( "github.com/go-ini/ini" "github.com/pelletier/go-toml/v2" - "golang.org/x/text/cases" - "golang.org/x/text/language" "sigs.k8s.io/kustomize/kyaml/errors" "sigs.k8s.io/kustomize/kyaml/kio" "sigs.k8s.io/kustomize/kyaml/kio/kioutil" "sigs.k8s.io/kustomize/kyaml/yaml" ) +// Extender allows both traversing and modifying hierarchical opaque data +// structures like yaml, toml or ini files. +// Part of the structure is addressed through a path that is an array of string +// symbols. +// +// - It is first initialized with SetPayload with the data structure payload. +// - Traversal is done with Get +// - Modification of part of the structure is done through Set +// - After modification, the modified payload is retrieved with GetPayload type Extender interface { + // SetPayload initialize the embedded data structure with payload. SetPayload(payload []byte) error + // GetPayload returns the current data structure in the appropriate encoding. GetPayload() ([]byte, error) + // Get returns the subset of the structure at path in the appropriate encoding. Get(path []string) ([]byte, error) + // Set modifies the data structure at path with value. Value can either be + // in the appropriate encoding or can be encoded by the Extender. Please + // see the Extender documentation to see how the the value is treated. Set(path []string, value any) error } +// ExtendedSegment contains the path segment of a resource inside an embedded +// data structure. type ExtendedSegment struct { - Encoding string - Path []string + Encoding string // The encoding of the embedded data structure + Path []string // The path inside the embedded data structure } +// String returns a string representation of the ExtendedSegment. +// +// For instance: +// +// !!yaml.common.targetRevision func (e *ExtendedSegment) String() string { if len(e.Path) > 0 { return fmt.Sprintf("!!%s", e.Encoding) @@ -41,6 +61,8 @@ func (e *ExtendedSegment) String() string { type any interface{} +// ExtenderType enumerates the existing extender types. +// //go:generate go run golang.org/x/tools/cmd/stringer -type=ExtenderType type ExtenderType int @@ -54,13 +76,15 @@ const ( IniExtender ) +// stringToExtenderTypeMap maps encoding names to the corresponding extender var stringToExtenderTypeMap map[string]ExtenderType func init() { //nolint:gochecknoinits stringToExtenderTypeMap = makeStringToExtenderTypeMap() } -func GetByteValue(value any) []byte { +// getByteValue returns value encoded as a byte array. +func getByteValue(value any) []byte { switch v := value.(type) { case *yaml.Node: return []byte(v.Value) @@ -72,16 +96,20 @@ func GetByteValue(value any) []byte { return []byte{} } +// makeStringToExtenderTypeMap makes a map to get the appropriate +// [ExtenderType] given its name. func makeStringToExtenderTypeMap() (result map[string]ExtenderType) { result = make(map[string]ExtenderType, 3) for k := range ExtenderFactories { - result[k.String()] = k + result[strings.Replace(strings.ToLower(k.String()), "extender", "", 1)] = k } return } -func GetExtenderType(n string) ExtenderType { - result, ok := stringToExtenderTypeMap[n] +// getExtenderType returns the appropriate [ExtenderType] for the passed +// extender type name +func getExtenderType(n string) ExtenderType { + result, ok := stringToExtenderTypeMap[strings.ToLower(n)] if ok { return result } @@ -92,11 +120,18 @@ func GetExtenderType(n string) ExtenderType { // YAML Extender //////////////// +// yamlExtender manages embedded YAML in KRM resources. +// +// Internally, it uses a RNode. It avoids additional dependencies and preserves +// ordering and comments. type yamlExtender struct { node *yaml.RNode } -func readPayload(payload []byte) (*yaml.RNode, error) { +// parsePayload parses payload into a RNode. +// +// The payload can either by in YAML or JSON format. +func parsePayload(payload []byte) (*yaml.RNode, error) { nodes, err := (&kio.ByteReader{ Reader: bytes.NewBuffer(payload), OmitReaderAnnotations: false, @@ -110,22 +145,26 @@ func readPayload(payload []byte) (*yaml.RNode, error) { return nodes[0], nil } +// SetPayload parses payload an sets the extender internal state func (e *yamlExtender) SetPayload(payload []byte) (err error) { - e.node, err = readPayload(payload) + e.node, err = parsePayload(payload) return } -func getNodeBytes(nodes []*yaml.RNode) ([]byte, error) { +// serializeNodes serialize nodes into YAML +func serializeNodes(nodes []*yaml.RNode) ([]byte, error) { var b bytes.Buffer err := (&kio.ByteWriter{Writer: &b}).Write(nodes) return b.Bytes(), err } +// GetPayload returns the current payload in the proper encoding func (e *yamlExtender) GetPayload() ([]byte, error) { - return getNodeBytes([]*yaml.RNode{e.node}) + return serializeNodes([]*yaml.RNode{e.node}) } -func LookupNode(node *yaml.RNode) *yaml.RNode { +// unwrapSeqNode unwraps node if it is a Wrapped Bare Seq Node +func unwrapSeqNode(node *yaml.RNode) *yaml.RNode { seqNode, err := node.Pipe(yaml.Lookup(yaml.BareSeqNodeWrappingKey)) if err == nil && !seqNode.IsNilOrEmpty() { return seqNode @@ -133,8 +172,10 @@ func LookupNode(node *yaml.RNode) *yaml.RNode { return node } +// Lookup looks for the specified path in node and return the matching nodes. func Lookup(node *yaml.RNode, path []string) ([]*yaml.RNode, error) { - node, err := LookupNode(node).Pipe(&yaml.PathMatcher{Path: path}) + // TODO: consider using yaml.PathGetter instead + node, err := unwrapSeqNode(node).Pipe(&yaml.PathMatcher{Path: path}) if err != nil { return nil, errors.WrapPrefixf(err, "while getting path %s", strings.Join(path, ".")) } @@ -142,6 +183,7 @@ func Lookup(node *yaml.RNode, path []string) ([]*yaml.RNode, error) { return node.Elements() } +// Get returns the encoded payload at the specified path func (e *yamlExtender) Get(path []string) ([]byte, error) { targetFields, err := Lookup(e.node, path) if err != nil { @@ -152,9 +194,10 @@ func (e *yamlExtender) Get(path []string) ([]byte, error) { return []byte(targetFields[0].YNode().Value), nil } - return getNodeBytes(targetFields) + return serializeNodes(targetFields) } +// Set modifies the current payload with value at the specified path. func (e *yamlExtender) Set(path []string, value any) error { targetFields, err := Lookup(e.node, path) @@ -164,7 +207,7 @@ func (e *yamlExtender) Set(path []string, value any) error { for _, t := range targetFields { if t.YNode().Kind == yaml.ScalarNode { - t.YNode().Value = string(GetByteValue(value)) + t.YNode().Value = string(getByteValue(value)) } else { if v, ok := value.(*yaml.Node); ok { t.SetYNode(v) @@ -176,6 +219,10 @@ func (e *yamlExtender) Set(path []string, value any) error { return nil } +// NewYamlExtender returns a newly created YAML [Extender]. +// +// With this encoding, you can set scalar values (strings, numbers) as well +// as mapping values. func NewYamlExtender() Extender { return &yamlExtender{} } @@ -184,10 +231,12 @@ func NewYamlExtender() Extender { // Base64 ///////// +// base64Extender manages embedded base64 in KRM resources. type base64Extender struct { - decoded []byte + decoded []byte // The base64 decoded payload } +// SetPayload decodes the payload and stores in internal state. func (e *base64Extender) SetPayload(payload []byte) error { decoded, err := base64.StdEncoding.DecodeString(string(payload)) if err != nil { @@ -197,10 +246,14 @@ func (e *base64Extender) SetPayload(payload []byte) error { return nil } +// GetPayload returns the current payload as base64 func (e *base64Extender) GetPayload() ([]byte, error) { - return e.decoded, nil + return []byte(base64.StdEncoding.EncodeToString(e.decoded)), nil } +// Get returns the current base64 decoded payload. +// +// An error is returned if the path is not empty. func (e *base64Extender) Get(path []string) ([]byte, error) { if len(path) > 0 { return nil, fmt.Errorf("path is invalid for base64: %s", strings.Join(path, ".")) @@ -208,14 +261,24 @@ func (e *base64Extender) Get(path []string) ([]byte, error) { return e.decoded, nil } +// Set stores value in the current payload. path must be empty. func (e *base64Extender) Set(path []string, value any) error { if len(path) > 0 { return fmt.Errorf("path is invalid for base64: %s", strings.Join(path, ".")) } - e.decoded = []byte(base64.StdEncoding.EncodeToString(GetByteValue(value))) + e.decoded = getByteValue(value) return nil } +// NewBase64Extender returns a newly created Base64 extender. +// +// This extender doesn't allow structured traversal and modification. It just +// passes its decoded payload downstream. Example of usage: +// +// prefix.!!base64.!!yaml.inside.path +// +// The above means that we want to modify inside.path in the YAML payload that +// is stored in base64 in the prefix property. func NewBase64Extender() Extender { return &base64Extender{} } @@ -224,19 +287,26 @@ func NewBase64Extender() Extender { // Regex //////// +// regexExtender allows text replacement in pure text properties. +// +// see [NewRegexExtender] type regexExtender struct { text []byte } +// SetPayload store the plain payload internally func (e *regexExtender) SetPayload(payload []byte) error { e.text = payload return nil } +// GetPayload returns the text payload func (e *regexExtender) GetPayload() ([]byte, error) { return []byte(e.text), nil } +// Get returns the text matched by the regexp contained in the first segment of +// path. func (e *regexExtender) Get(path []string) ([]byte, error) { if len(path) < 1 { return nil, fmt.Errorf("path for regex should at least be one") @@ -248,6 +318,18 @@ func (e *regexExtender) Get(path []string) ([]byte, error) { return re.Find(e.text), nil } +// Set modifies the inner text inserting value in the capture group specified by +// path[1] of the Regexp specified by path[0]. +// +// Example paths: +// +// [`^\s+HostName\s+(\S+)\s*$`, `1`] +// +// Changes the value after HostName with value. +// +// [`^\s+HostName\s+\S+\s*$`, `0`] +// +// Replace the whole line with value. func (e *regexExtender) Set(path []string, value any) error { if len(path) != 2 { return fmt.Errorf("path for regex should at least be one") @@ -271,7 +353,7 @@ func (e *regexExtender) Set(path []string, value any) error { startIndex := group * 2 b.Write(e.text[start:v[startIndex]]) - b.Write(GetByteValue(value)) + b.Write(getByteValue(value)) start = v[startIndex+1] } @@ -285,6 +367,27 @@ func (e *regexExtender) Set(path []string, value any) error { return nil } +// NewRegexExtender returns a newly created Regexp [Extender]. +// +// This extender allows text replacement in pure text properties. It is useful +// in the case the content of the KRM property is not structured. +// +// We don't recommend using it too much as it weakens the transformation. +// +// The paths to use with this extender are always composed of two elements: +// +// - The regexp to look for in the text. +// - The capture group index to replace with the source value. +// +// Examples: +// +// ^\s+HostName\s+(\S+)\s*$.1 +// +// Changes the value after HostName with value. +// +// ^\s+HostName\s+\S+\s*$.0 +// +// Replace the whole line with value. func NewRegexExtender() Extender { return ®exExtender{} } @@ -293,15 +396,23 @@ func NewRegexExtender() Extender { // JSON /////// +// jsonExtender is an [Extender] allowing modifications in JSON content. +// +// It is close to [yamlExtender] as kyaml knows to read and write JSON files. type jsonExtender struct { node *yaml.RNode } +// SetPayload parses the JSON payload and stores it internally as a yaml.RNode. func (e *jsonExtender) SetPayload(payload []byte) (err error) { - e.node, err = readPayload(payload) + e.node, err = parsePayload(payload) return } +// getJSONPayload returns the JSON payload for the passed node. +// +// There is a small issue in kio.ByteWriter preventing the JSON serialization of +// a wrapped JSON array. func getJSONPayload(node *yaml.RNode) ([]byte, error) { var b bytes.Buffer if node.YNode().Kind == yaml.MappingNode { @@ -318,10 +429,12 @@ func getJSONPayload(node *yaml.RNode) ([]byte, error) { return b.Bytes(), err } +// GetPayload returns the payload as a serialized JSON object func (e *jsonExtender) GetPayload() ([]byte, error) { - return getJSONPayload(LookupNode(e.node)) + return getJSONPayload(unwrapSeqNode(e.node)) } +// Get returns the sub JSON specified by path. func (e *jsonExtender) Get(path []string) ([]byte, error) { targetFields, err := Lookup(e.node, path) if err != nil { @@ -340,6 +453,7 @@ func (e *jsonExtender) Get(path []string) ([]byte, error) { return getJSONPayload(target) } +// Set modifies the inner JSON at path with value func (e *jsonExtender) Set(path []string, value any) error { targetFields, err := Lookup(e.node, path) @@ -349,7 +463,7 @@ func (e *jsonExtender) Set(path []string, value any) error { for _, t := range targetFields { if t.YNode().Kind == yaml.ScalarNode { - t.YNode().Value = string(GetByteValue(value)) + t.YNode().Value = string(getByteValue(value)) } else { if v, ok := value.(*yaml.Node); ok { t.SetYNode(v) @@ -361,6 +475,10 @@ func (e *jsonExtender) Set(path []string, value any) error { return nil } +// NewJsonExtender returns a newly created [Extender] to modify JSON content. +// +// As with the YAML extender (see [NewYamlExtender]), modifications are not +// limited to scalar values but the source can be a mapping or a sequence. func NewJsonExtender() Extender { return &jsonExtender{} } @@ -369,10 +487,13 @@ func NewJsonExtender() Extender { // TOML /////// +// tomlExtender is an [Extender] allowing the structured modification of a TOML +// property. type tomlExtender struct { node *yaml.RNode } +// SetPayload sets the internal state with the TOML source payload. func (e *tomlExtender) SetPayload(payload []byte) error { m := map[string]interface{}{} @@ -389,6 +510,9 @@ func (e *tomlExtender) SetPayload(payload []byte) error { return nil } +// getTOMLPayload returns the TOML representation of the specified node. +// +// The node must be a mapping node. func getTOMLPayload(node *yaml.RNode) ([]byte, error) { m, err := node.Map() if err != nil { @@ -397,10 +521,12 @@ func getTOMLPayload(node *yaml.RNode) ([]byte, error) { return toml.Marshal(m) } +// GetPayload return the current payload as a TOML snippet. func (e *tomlExtender) GetPayload() ([]byte, error) { return getTOMLPayload(e.node) } +// Get returns the TOML representation of the sub element at path. func (e *tomlExtender) Get(path []string) ([]byte, error) { targetFields, err := Lookup(e.node, path) if err != nil { @@ -419,6 +545,7 @@ func (e *tomlExtender) Get(path []string) ([]byte, error) { return getTOMLPayload(target) } +// Set modifies the current payload at path with value. func (e *tomlExtender) Set(path []string, value any) error { targetFields, err := Lookup(e.node, path) @@ -428,7 +555,7 @@ func (e *tomlExtender) Set(path []string, value any) error { for _, t := range targetFields { if t.YNode().Kind == yaml.ScalarNode { - t.YNode().Value = string(GetByteValue(value)) + t.YNode().Value = string(getByteValue(value)) } else { if v, ok := value.(*yaml.Node); ok { t.SetYNode(v) @@ -440,30 +567,39 @@ func (e *tomlExtender) Set(path []string, value any) error { return nil } +// NewTomlExtender returns a newly created [Extender] for modifying properties +// containing TOML. +// +// Please be aware that this [Extender] doesn't preserve the source ordering +// nor the comments in the content. func NewTomlExtender() Extender { return &tomlExtender{} } -/////// -// TOML -/////// +////// +// INI +////// +// iniExtender allows structured modification of ini file based properties. type iniExtender struct { file *ini.File } +// SetPayload parses payload as a INI file and set the internal state. func (e *iniExtender) SetPayload(payload []byte) (err error) { e.file, err = ini.Load(payload) return err } +// GetPayload returns the current state as an ini file. func (e *iniExtender) GetPayload() ([]byte, error) { var b bytes.Buffer _, err := e.file.WriteTo(&b) return b.Bytes(), err } +// keyFromPath returns the INI key at path. func (e *iniExtender) keyFromPath(path []string) (*ini.Key, error) { if len(path) < 1 || len(path) > 2 { return nil, fmt.Errorf("invalid path length: %d", len(path)) @@ -477,6 +613,7 @@ func (e *iniExtender) keyFromPath(path []string) (*ini.Key, error) { return e.file.Section(section).Key(key), nil } +// Get returns the content of the key specified by path. func (e *iniExtender) Get(path []string) ([]byte, error) { k, err := e.keyFromPath(path) if err != nil { @@ -485,17 +622,29 @@ func (e *iniExtender) Get(path []string) ([]byte, error) { return []byte(k.String()), nil } +// Set sets the value of the key specified by path with value. func (e *iniExtender) Set(path []string, value any) error { k, err := e.keyFromPath(path) if err != nil { return fmt.Errorf("while getting key at path %s", strings.Join(path, ".")) } - k.SetValue(string(GetByteValue(value))) + k.SetValue(string(getByteValue(value))) return nil } +// NewIniExtender returns a newly created [Extender] for modifying INI files +// like properties. +// +// Some tools may use ini type configuration files. This extender allows +// modification of the values. At this point, it doesn't allow inserting +// complete sections. If paths have one element, it will set the corresponding +// property at the root level. If path have two elements, the first one contains +// the section name and the second the property name. +// +// Please be aware that this [Extender] doesn't preserve the source ordering +// nor the comments in the content. func NewIniExtender() Extender { return &iniExtender{} } @@ -504,6 +653,8 @@ func NewIniExtender() Extender { // Factories //////////// +// ExtenderFactories register the [Extender] factory functions for each +// [ExtenderType]. var ExtenderFactories = map[ExtenderType]func() Extender{ YamlExtender: NewYamlExtender, Base64Extender: NewBase64Extender, @@ -513,8 +664,10 @@ var ExtenderFactories = map[ExtenderType]func() Extender{ IniExtender: NewIniExtender, } +// Extender returns a newly created [Extender] for the appropriate encoding. +// uses [ExtenderFactories]. func (path *ExtendedSegment) Extender(payload []byte) (Extender, error) { - bpt := GetExtenderType(cases.Title(language.English, cases.NoLower).String(path.Encoding) + "Extender") + bpt := getExtenderType(path.Encoding) if f, ok := ExtenderFactories[bpt]; ok { result := f() if err := result.SetPayload(payload); err != nil { @@ -530,6 +683,8 @@ func (path *ExtendedSegment) Extender(payload []byte) (Extender, error) { // ExtendedPath /////////////// +// splitExtendedPath fills extensions with the ExtendedSegments found in path +// and returns the path prefix. This method is used by [NewExtendedPath] func splitExtendedPath(path []string, extensions *[]*ExtendedSegment) (basePath []string, err error) { if len(path) == 0 { @@ -559,11 +714,30 @@ func splitExtendedPath(path []string, extensions *[]*ExtendedSegment) (basePath return } +// ExtendedPath contains all the paths segments of a path. +// The path is composed by: +// +// - a KRM resource path, the prefix (ResourcePath) +// - 0 or more [ExtendedSegment]s. +// +// For instance, for the following path: +// +// data.secretConfiguration.!!base64.!!yaml.common.URL +// +// ResourcePath would be ["data", "secretConfiguration"] and ExtendedSegments: +// +// *[]*ExtendedSegment{ +// &ExtendedSegment{Encoding: "base64", Path: []string{}}, +// &ExtendedSegment{Encoding: "yaml", Path: []string{"common", "URL"}}, +// } type ExtendedPath struct { - resourcePath []string - extendedSegments *[]*ExtendedSegment + // ResourcePath is The KRM portion of the path + ResourcePath []string + // ExtendedSegments contains all extended path segments + ExtendedSegments *[]*ExtendedSegment } +// NewExtendedPath creates an [ExtendedPath] from the split path segments in paths. func NewExtendedPath(path []string) (*ExtendedPath, error) { extensions := []*ExtendedSegment{} prefix, err := splitExtendedPath(path, &extensions) @@ -571,18 +745,20 @@ func NewExtendedPath(path []string) (*ExtendedPath, error) { return nil, errors.WrapPrefixf(err, "while getting extended path") } - return &ExtendedPath{resourcePath: prefix, extendedSegments: &extensions}, nil + return &ExtendedPath{ResourcePath: prefix, ExtendedSegments: &extensions}, nil } +// HasExtensions returns true if the path contains extended segments. func (ep *ExtendedPath) HasExtensions() bool { - return len(*ep.extendedSegments) > 0 + return len(*ep.ExtendedSegments) > 0 } +// String returns a string representation of the extended path. func (ep *ExtendedPath) String() string { - out := strings.Join(ep.resourcePath, ".") - if len(*ep.extendedSegments) > 0 { + out := strings.Join(ep.ResourcePath, ".") + if len(*ep.ExtendedSegments) > 0 { segmentStrings := []string{} - for _, s := range *ep.extendedSegments { + for _, s := range *ep.ExtendedSegments { segmentStrings = append(segmentStrings, s.String()) } out = fmt.Sprintf("%s.%s", out, strings.Join(segmentStrings, ".")) @@ -590,18 +766,19 @@ func (ep *ExtendedPath) String() string { return out } -func (ep *ExtendedPath) ApplyIndex(index int, input []byte, value *yaml.Node) ([]byte, error) { - if index >= len(*ep.extendedSegments) || index < 0 { +// applyIndex applies value to input starting at the extended path index. +func (ep *ExtendedPath) applyIndex(index int, input []byte, value *yaml.Node) ([]byte, error) { + if index >= len(*ep.ExtendedSegments) || index < 0 { return nil, fmt.Errorf("invalid extended path index: %d", index) } - segment := (*ep.extendedSegments)[index] + segment := (*ep.ExtendedSegments)[index] extender, err := segment.Extender(input) if err != nil { return nil, errors.WrapPrefixf(err, "creating extender at index: %d", index) } - if index == len(*ep.extendedSegments)-1 { + if index == len(*ep.ExtendedSegments)-1 { err := extender.Set(segment.Path, value) if err != nil { return nil, errors.WrapPrefixf(err, "setting value on path %s", segment.String()) @@ -611,7 +788,7 @@ func (ep *ExtendedPath) ApplyIndex(index int, input []byte, value *yaml.Node) ([ if err != nil { return nil, errors.WrapPrefixf(err, "getting value on path %s", segment.String()) } - newValue, err := ep.ApplyIndex(index+1, nextInput, value) + newValue, err := ep.applyIndex(index+1, nextInput, value) if err != nil { return nil, err } @@ -624,15 +801,22 @@ func (ep *ExtendedPath) ApplyIndex(index int, input []byte, value *yaml.Node) ([ return extender.GetPayload() } +// Apply applies value to target. target is the KRM resource specified by +// ResourcePrefix. +// +// Apply creates the appropriate [Extender] for each extended segment and +// traverse it until the last. When reaching the last, it sets value +// in the appropriate path. It then unwinds the paths and save the modified +// value in the target. func (ep *ExtendedPath) Apply(target *yaml.RNode, value *yaml.RNode) error { if target.YNode().Kind != yaml.ScalarNode { return fmt.Errorf("extended path only works on scalar nodes") } outValue := value.YNode().Value - if len(*ep.extendedSegments) > 0 { + if len(*ep.ExtendedSegments) > 0 { input := []byte(target.YNode().Value) - output, err := ep.ApplyIndex(0, input, value.YNode()) + output, err := ep.applyIndex(0, input, value.YNode()) if err != nil { return errors.WrapPrefixf(err, "applying value on extended segment %s", ep.String()) } diff --git a/pkg/extras/extender_test.go b/pkg/extras/extender_test.go index 88bdb60..16d0ee6 100644 --- a/pkg/extras/extender_test.go +++ b/pkg/extras/extender_test.go @@ -252,11 +252,11 @@ common: path := kyaml_utils.SmarterPathSplitter(p, ".") e, err := NewExtendedPath(path) require.NoError(err) - require.Len(e.resourcePath, 1, "no resource path") + require.Len(e.ResourcePath, 1, "no resource path") sourcePath := []string{"common"} - target, err := source.Pipe(&yaml.PathGetter{Path: e.resourcePath}) + target, err := source.Pipe(&yaml.PathGetter{Path: e.ResourcePath}) require.NoError(err) value, err := replacement.Pipe(&yaml.PathGetter{Path: sourcePath}) diff --git a/pkg/extras/replacement.go b/pkg/extras/replacement.go index 578a87c..2bd9312 100644 --- a/pkg/extras/replacement.go +++ b/pkg/extras/replacement.go @@ -9,7 +9,7 @@ import ( "reflect" "strings" - "sigs.k8s.io/kustomize/api/konfig" + "github.com/kaweezle/krmfnbuiltin/pkg/utils" "sigs.k8s.io/kustomize/api/resmap" "sigs.k8s.io/kustomize/api/resource" "sigs.k8s.io/kustomize/api/types" @@ -18,12 +18,12 @@ import ( "sigs.k8s.io/kustomize/kyaml/yaml" ) -type Filter struct { +type extendedFilter struct { Replacements []types.Replacement `json:"replacements,omitempty" yaml:"replacements,omitempty"` } // Filter replaces values of targets with values from sources -func (f Filter) Filter(nodes []*yaml.RNode) ([]*yaml.RNode, error) { +func (f extendedFilter) Filter(nodes []*yaml.RNode) ([]*yaml.RNode, error) { for i, r := range f.Replacements { if r.Source == nil || r.Targets == nil { return nil, fmt.Errorf("replacements must specify a source and at least one target") @@ -67,7 +67,7 @@ func getReplacement(nodes []*yaml.RNode, r *types.Replacement) (*yaml.RNode, err func selectSourceNode(nodes []*yaml.RNode, selector *types.SourceSelector) (*yaml.RNode, error) { var matches []*yaml.RNode for _, n := range nodes { - ids, err := MakeResIds(n) + ids, err := makeResIds(n) if err != nil { return nil, fmt.Errorf("error getting node IDs: %w", err) } @@ -113,7 +113,7 @@ func applyReplacement(nodes []*yaml.RNode, value *yaml.RNode, targetSelectors [] selector.FieldPaths = []string{types.DefaultReplacementFieldPath} } for _, possibleTarget := range nodes { - ids, err := MakeResIds(possibleTarget) + ids, err := makeResIds(possibleTarget) if err != nil { return nil, err } @@ -186,21 +186,21 @@ func copyValueToTarget(target *yaml.RNode, value *yaml.RNode, selector *types.Ta if err != nil { return err } - create, err := shouldCreateField(selector.Options, extendedPath.resourcePath) + create, err := shouldCreateField(selector.Options, extendedPath.ResourcePath) if err != nil { return err } var targetFields []*yaml.RNode if create { - createdField, createErr := target.Pipe(yaml.LookupCreate(value.YNode().Kind, extendedPath.resourcePath...)) + createdField, createErr := target.Pipe(yaml.LookupCreate(value.YNode().Kind, extendedPath.ResourcePath...)) if createErr != nil { return fmt.Errorf("error creating replacement node: %w", createErr) } targetFields = append(targetFields, createdField) } else { // may return multiple fields, always wrapped in a sequence node - foundFieldSequence, lookupErr := target.Pipe(&yaml.PathMatcher{Path: extendedPath.resourcePath}) + foundFieldSequence, lookupErr := target.Pipe(&yaml.PathMatcher{Path: extendedPath.ResourcePath}) if lookupErr != nil { return fmt.Errorf("error finding field in replacement target: %w", lookupErr) } @@ -271,16 +271,8 @@ func shouldCreateField(options *types.FieldOptions, fieldPath []string) (bool, e // Copied -const ( - BuildAnnotationPreviousKinds = konfig.ConfigAnnoDomain + "/previousKinds" - BuildAnnotationPreviousNames = konfig.ConfigAnnoDomain + "/previousNames" - BuildAnnotationPrefixes = konfig.ConfigAnnoDomain + "/prefixes" - BuildAnnotationSuffixes = konfig.ConfigAnnoDomain + "/suffixes" - BuildAnnotationPreviousNamespaces = konfig.ConfigAnnoDomain + "/previousNamespaces" -) - -// MakeResIds returns all of an RNode's current and previous Ids -func MakeResIds(n *yaml.RNode) ([]resid.ResId, error) { +// makeResIds returns all of an RNode's current and previous Ids +func makeResIds(n *yaml.RNode) ([]resid.ResId, error) { var result []resid.ResId apiVersion := n.Field(yaml.APIVersionField) var group, version string @@ -290,7 +282,7 @@ func MakeResIds(n *yaml.RNode) ([]resid.ResId, error) { result = append(result, resid.NewResIdWithNamespace( resid.Gvk{Group: group, Version: version, Kind: n.GetKind()}, n.GetName(), n.GetNamespace()), ) - prevIds, err := PrevIds(n) + prevIds, err := prevIds(n) if err != nil { return nil, err } @@ -298,18 +290,18 @@ func MakeResIds(n *yaml.RNode) ([]resid.ResId, error) { return result, nil } -// PrevIds returns all of an RNode's previous Ids -func PrevIds(n *yaml.RNode) ([]resid.ResId, error) { +// prevIds returns all of an RNode's previous Ids +func prevIds(n *yaml.RNode) ([]resid.ResId, error) { var ids []resid.ResId // TODO: merge previous names and namespaces into one list of // pairs on one annotation so there is no chance of error annotations := n.GetAnnotations() - if _, ok := annotations[BuildAnnotationPreviousNames]; !ok { + if _, ok := annotations[utils.BuildAnnotationPreviousNames]; !ok { return nil, nil } - names := strings.Split(annotations[BuildAnnotationPreviousNames], ",") - ns := strings.Split(annotations[BuildAnnotationPreviousNamespaces], ",") - kinds := strings.Split(annotations[BuildAnnotationPreviousKinds], ",") + names := strings.Split(annotations[utils.BuildAnnotationPreviousNames], ",") + ns := strings.Split(annotations[utils.BuildAnnotationPreviousNamespaces], ",") + kinds := strings.Split(annotations[utils.BuildAnnotationPreviousKinds], ",") // This should never happen if len(names) != len(ns) || len(names) != len(kinds) { return nil, fmt.Errorf( @@ -336,12 +328,28 @@ func PrevIds(n *yaml.RNode) ([]resid.ResId, error) { // plugin -// Replace values in targets with values from a source +// Replace values in targets with values from a source. This transformer is +// "extended" because it allows structured replacement in properties +// containing a string representation of some structured content. It currently +// supports the following structured formats: +// +// - Yaml +// - Json +// - Toml +// - Ini +// +// It also provides helpers for changing content in base64 encoded properties +// as well as a simple regexp based replacer for edge cases. +// +// Configuration of replacements can be found in the [kustomize doc]. +// +// [kustomize doc]: https://kubectl.docs.kubernetes.io/references/kustomize/kustomization/replacements/ type ExtendedReplacementTransformerPlugin struct { ReplacementList []types.ReplacementField `json:"replacements,omitempty" yaml:"replacements,omitempty"` Replacements []types.Replacement `json:"omitempty" yaml:"omitempty"` } +// Config configures the plugin func (p *ExtendedReplacementTransformerPlugin) Config( h *resmap.PluginHelpers, c []byte) (err error) { p.ReplacementList = []types.ReplacementField{} @@ -390,12 +398,14 @@ func (p *ExtendedReplacementTransformerPlugin) Config( return nil } +// Transform performs the configured replacements in the specified resource map func (p *ExtendedReplacementTransformerPlugin) Transform(m resmap.ResMap) (err error) { - return m.ApplyFilter(Filter{ + return m.ApplyFilter(extendedFilter{ Replacements: p.Replacements, }) } +// NewExtendedReplacementTransformerPlugin returns a newly created [ExtendedReplacementTransformerPlugin] func NewExtendedReplacementTransformerPlugin() resmap.TransformerPlugin { return &ExtendedReplacementTransformerPlugin{} } diff --git a/pkg/plugins/doc.go b/pkg/plugins/doc.go new file mode 100644 index 0000000..9a11804 --- /dev/null +++ b/pkg/plugins/doc.go @@ -0,0 +1,4 @@ +/* +Package plugins is a copy of the Kustomize standard plugins public factory. +*/ +package plugins diff --git a/pkg/utils/constants.go b/pkg/utils/constants.go new file mode 100644 index 0000000..de315fa --- /dev/null +++ b/pkg/utils/constants.go @@ -0,0 +1,15 @@ +package utils + +import "sigs.k8s.io/kustomize/api/konfig" + +const ( + // build annotations + BuildAnnotationPreviousKinds = konfig.ConfigAnnoDomain + "/previousKinds" + BuildAnnotationPreviousNames = konfig.ConfigAnnoDomain + "/previousNames" + BuildAnnotationPrefixes = konfig.ConfigAnnoDomain + "/prefixes" + BuildAnnotationSuffixes = konfig.ConfigAnnoDomain + "/suffixes" + BuildAnnotationPreviousNamespaces = konfig.ConfigAnnoDomain + "/previousNamespaces" + BuildAnnotationsRefBy = konfig.ConfigAnnoDomain + "/refBy" + BuildAnnotationsGenBehavior = konfig.ConfigAnnoDomain + "/generatorBehavior" + BuildAnnotationsGenAddHashSuffix = konfig.ConfigAnnoDomain + "/needsHashSuffix" +) diff --git a/pkg/utils/doc.go b/pkg/utils/doc.go new file mode 100644 index 0000000..eff2858 --- /dev/null +++ b/pkg/utils/doc.go @@ -0,0 +1,5 @@ +/* +Package utils contains functions and constants located in kustomize internal +packages and that are needed by krmfnbuiltin. +*/ +package utils diff --git a/pkg/utils/utils.go b/pkg/utils/utils.go new file mode 100644 index 0000000..a5365a3 --- /dev/null +++ b/pkg/utils/utils.go @@ -0,0 +1,32 @@ +package utils + +import "sigs.k8s.io/kustomize/api/resource" + +var buildAnnotations = []string{ + BuildAnnotationPreviousKinds, + BuildAnnotationPreviousNames, + BuildAnnotationPrefixes, + BuildAnnotationSuffixes, + BuildAnnotationPreviousNamespaces, + BuildAnnotationsRefBy, + BuildAnnotationsGenBehavior, + BuildAnnotationsGenAddHashSuffix, +} + +// RemoveBuildAnnotations removes kustomize build annotations from r. +// +// Contrary to the method available in resource.Resource, this method doesn't +// remove the file name related annotations, as this would prevent modification +// of the source file. +func RemoveBuildAnnotations(r *resource.Resource) { + annotations := r.GetAnnotations() + if len(annotations) == 0 { + return + } + for _, a := range buildAnnotations { + delete(annotations, a) + } + if err := r.SetAnnotations(annotations); err != nil { + panic(err) + } +} From a03948e6bdefabccf5f3c6d96fc803f97670a783 Mon Sep 17 00:00:00 2001 From: Antoine Martin Date: Wed, 25 Jan 2023 15:12:27 +0000 Subject: [PATCH 06/10] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20Allow=20node=20addit?= =?UTF-8?q?ion=20and=20factorize=20code?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/extras/GitConfigMapGenerator.go | 2 +- pkg/extras/extender.go | 138 ++++++++++------------------ 2 files changed, 48 insertions(+), 92 deletions(-) diff --git a/pkg/extras/GitConfigMapGenerator.go b/pkg/extras/GitConfigMapGenerator.go index b67f0c8..d448c57 100644 --- a/pkg/extras/GitConfigMapGenerator.go +++ b/pkg/extras/GitConfigMapGenerator.go @@ -11,7 +11,7 @@ import ( "sigs.k8s.io/yaml" ) -// GitConfigMapGeneratorPlugin Generates a config map that includes two +// GitConfigMapGeneratorPlugin generates a config map that includes two // properties of the current git repository: // // - repoURL contains the URL or the remote specified by remoteName. by diff --git a/pkg/extras/extender.go b/pkg/extras/extender.go index ccd50ca..cd77f03 100644 --- a/pkg/extras/extender.go +++ b/pkg/extras/extender.go @@ -151,16 +151,16 @@ func (e *yamlExtender) SetPayload(payload []byte) (err error) { return } -// serializeNodes serialize nodes into YAML -func serializeNodes(nodes []*yaml.RNode) ([]byte, error) { +// serializeNode serialize one node into YAML +func serializeNode(node *yaml.RNode) ([]byte, error) { var b bytes.Buffer - err := (&kio.ByteWriter{Writer: &b}).Write(nodes) + err := (&kio.ByteWriter{Writer: &b}).Write([]*yaml.RNode{node}) return b.Bytes(), err } // GetPayload returns the current payload in the proper encoding func (e *yamlExtender) GetPayload() ([]byte, error) { - return serializeNodes([]*yaml.RNode{e.node}) + return serializeNode(e.node) } // unwrapSeqNode unwraps node if it is a Wrapped Bare Seq Node @@ -172,53 +172,71 @@ func unwrapSeqNode(node *yaml.RNode) *yaml.RNode { return node } -// Lookup looks for the specified path in node and return the matching nodes. -func Lookup(node *yaml.RNode, path []string) ([]*yaml.RNode, error) { +// Lookup looks for the specified path in node and return the matching node. If +// kind is a valid node kind and the node doesn't exist, create it. +func Lookup(node *yaml.RNode, path []string, kind yaml.Kind) (*yaml.RNode, error) { // TODO: consider using yaml.PathGetter instead - node, err := unwrapSeqNode(node).Pipe(&yaml.PathMatcher{Path: path}) + node, err := unwrapSeqNode(node).Pipe(&yaml.PathGetter{Path: path, Create: kind}) if err != nil { return nil, errors.WrapPrefixf(err, "while getting path %s", strings.Join(path, ".")) } - return node.Elements() + return node, nil } -// Get returns the encoded payload at the specified path -func (e *yamlExtender) Get(path []string) ([]byte, error) { - targetFields, err := Lookup(e.node, path) +// nodeSerializer is a RNode serializer function +type nodeSerializer func(*yaml.RNode) ([]byte, error) + +// getNodePath returns the value of the node at path serialized with serializer. +func getNodePath(node *yaml.RNode, path []string, serializer nodeSerializer) ([]byte, error) { + node, err := Lookup(node, path, 0) if err != nil { return nil, fmt.Errorf("error fetching elements in replacement target: %w", err) } - if len(targetFields) == 1 && targetFields[0].YNode().Kind == yaml.ScalarNode { - return []byte(targetFields[0].YNode().Value), nil + if node.YNode().Kind == yaml.ScalarNode { + return []byte(node.YNode().Value), nil } - return serializeNodes(targetFields) + return serializer(node) } -// Set modifies the current payload with value at the specified path. -func (e *yamlExtender) Set(path []string, value any) error { +// Get returns the encoded payload at the specified path +func (e *yamlExtender) Get(path []string) ([]byte, error) { + return getNodePath(e.node, path, serializeNode) +} + +// setValue sets value at path on node +func setValue(node *yaml.RNode, path []string, value any) error { + + kind := yaml.ScalarNode + if v, ok := value.(*yaml.Node); ok { + kind = v.Kind + } - targetFields, err := Lookup(e.node, path) + target, err := Lookup(node, path, kind) if err != nil { return fmt.Errorf("error fetching elements in replacement target: %w", err) } - for _, t := range targetFields { - if t.YNode().Kind == yaml.ScalarNode { - t.YNode().Value = string(getByteValue(value)) + if target.YNode().Kind == yaml.ScalarNode { + target.YNode().Value = string(getByteValue(value)) + } else { + if target.YNode().Kind == kind { + v, _ := value.(*yaml.Node) + target.SetYNode(v) } else { - if v, ok := value.(*yaml.Node); ok { - t.SetYNode(v) - } else { - return fmt.Errorf("setting non yaml object in place of object of type %s at path %s", t.YNode().Tag, strings.Join(path, ".")) - } + return fmt.Errorf("setting non yaml object in place of object of type %s at path %s", target.YNode().Tag, strings.Join(path, ".")) } } return nil } +// Set modifies the current payload with value at the specified path. +func (e *yamlExtender) Set(path []string, value any) error { + return setValue(e.node, path, value) +} + // NewYamlExtender returns a newly created YAML [Extender]. // // With this encoding, you can set scalar values (strings, numbers) as well @@ -436,43 +454,12 @@ func (e *jsonExtender) GetPayload() ([]byte, error) { // Get returns the sub JSON specified by path. func (e *jsonExtender) Get(path []string) ([]byte, error) { - targetFields, err := Lookup(e.node, path) - if err != nil { - return nil, errors.WrapPrefixf(err, "error fetching elements in replacement target") - } - - if len(targetFields) > 1 { - return nil, fmt.Errorf("path %s returned %d items. Expected one", strings.Join(path, "."), len(targetFields)) - } - - target := targetFields[0] - if target.YNode().Kind == yaml.ScalarNode { - return []byte(target.YNode().Value), nil - } - - return getJSONPayload(target) + return getNodePath(e.node, path, getJSONPayload) } // Set modifies the inner JSON at path with value func (e *jsonExtender) Set(path []string, value any) error { - - targetFields, err := Lookup(e.node, path) - if err != nil { - return fmt.Errorf("error fetching elements in replacement target: %w", err) - } - - for _, t := range targetFields { - if t.YNode().Kind == yaml.ScalarNode { - t.YNode().Value = string(getByteValue(value)) - } else { - if v, ok := value.(*yaml.Node); ok { - t.SetYNode(v) - } else { - return fmt.Errorf("setting non json object in place of object of type %s at path %s", t.YNode().Tag, strings.Join(path, ".")) - } - } - } - return nil + return setValue(e.node, path, value) } // NewJsonExtender returns a newly created [Extender] to modify JSON content. @@ -528,43 +515,12 @@ func (e *tomlExtender) GetPayload() ([]byte, error) { // Get returns the TOML representation of the sub element at path. func (e *tomlExtender) Get(path []string) ([]byte, error) { - targetFields, err := Lookup(e.node, path) - if err != nil { - return nil, errors.WrapPrefixf(err, "error fetching elements in replacement target") - } - - if len(targetFields) > 1 { - return nil, fmt.Errorf("path %s returned %d items. Expected one", strings.Join(path, "."), len(targetFields)) - } - - target := targetFields[0] - if target.YNode().Kind == yaml.ScalarNode { - return []byte(target.YNode().Value), nil - } - - return getTOMLPayload(target) + return getNodePath(e.node, path, getTOMLPayload) } // Set modifies the current payload at path with value. func (e *tomlExtender) Set(path []string, value any) error { - - targetFields, err := Lookup(e.node, path) - if err != nil { - return fmt.Errorf("error fetching elements in replacement target: %w", err) - } - - for _, t := range targetFields { - if t.YNode().Kind == yaml.ScalarNode { - t.YNode().Value = string(getByteValue(value)) - } else { - if v, ok := value.(*yaml.Node); ok { - t.SetYNode(v) - } else { - return fmt.Errorf("setting non toml object in place of object of type %s at path %s", t.YNode().Tag, strings.Join(path, ".")) - } - } - } - return nil + return setValue(e.node, path, value) } // NewTomlExtender returns a newly created [Extender] for modifying properties From 6044659a9c373ee578e9f3cbab22073b575ad9f9 Mon Sep 17 00:00:00 2001 From: Antoine Martin Date: Wed, 25 Jan 2023 16:40:18 +0000 Subject: [PATCH 07/10] =?UTF-8?q?=E2=9C=A8=20Added=20the=20no-op=20functio?= =?UTF-8?q?n=20that=20just=20injects=20its=20content?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Setting the config.kubernetes.io/inject-config annotation will add the functionConfig to the set or resources with the config.kubernetes.io/local-config annotation. This is meant to inject whatever resource we want for replacement instead of relying on flat configmaps. --- main.go | 22 ++++++++++++++++------ pkg/utils/constants.go | 17 +++++++++++++++++ pkg/utils/utils.go | 19 ++++++++++++++++++- 3 files changed, 51 insertions(+), 7 deletions(-) diff --git a/main.go b/main.go index 0915f19..23e61bc 100644 --- a/main.go +++ b/main.go @@ -13,7 +13,6 @@ import ( "sigs.k8s.io/kustomize/kyaml/fn/framework" "sigs.k8s.io/kustomize/kyaml/fn/framework/command" "sigs.k8s.io/kustomize/kyaml/kio/filters" - "sigs.k8s.io/kustomize/kyaml/kio/kioutil" "sigs.k8s.io/kustomize/kyaml/resid" "sigs.k8s.io/kustomize/kyaml/yaml" ) @@ -28,7 +27,20 @@ func main() { plugin, err := plugins.MakeBuiltinPlugin(resid.GvkFromNode(config)) if err != nil { - return errors.WrapPrefixf(err, "creating plugin") + // Check if config asks us to inject it + if _, ok := config.GetAnnotations()[utils.FunctionAnnotationInjectLocal]; ok { + injected := config.Copy() + + err := utils.MakeResourceLocal(injected) + if err != nil { + return errors.WrapPrefixf( + err, "Error while mangling annotations on %s fails configuration", res.OrgId()) + } + rl.Items = append(rl.Items, injected) + return nil + } else { + return errors.WrapPrefixf(err, "creating plugin") + } } yamlNode := config.YNode() @@ -68,7 +80,7 @@ func main() { // As it always add a filename by default, the local resources keep saved. // To avoid this, an annotation `config.kubernetes.io/prune-local` present in a // transformer makes all the local resources disappear. - if _, ok := config.GetAnnotations()["config.kubernetes.io/prune-local"]; ok { + if _, ok := config.GetAnnotations()[utils.FunctionAnnotationPruneLocal]; ok { filter := &filters.IsLocalConfig{IncludeLocalConfig: false, ExcludeNonLocalConfig: false} err = rl.Filter(filter) if err != nil { @@ -95,9 +107,7 @@ func main() { // do that on functions. So we have added a special annotation // `config.kubernetes.io/prune-local` to add on the last transformer. // We set the filename of the generated resource in case it is forgotten. - r.Pipe(yaml.SetAnnotation(filters.LocalConfigAnnotation, "true")) - r.Pipe(yaml.SetAnnotation(kioutil.PathAnnotation, ".generated.yaml")) - r.Pipe(yaml.SetAnnotation(kioutil.LegacyPathAnnotation, ".generated.yaml")) + utils.MakeResourceLocal(&r.RNode) } rl.Items = append(rl.Items, rm.ToRNodeSlice()...) diff --git a/pkg/utils/constants.go b/pkg/utils/constants.go index de315fa..58811f2 100644 --- a/pkg/utils/constants.go +++ b/pkg/utils/constants.go @@ -12,4 +12,21 @@ const ( BuildAnnotationsRefBy = konfig.ConfigAnnoDomain + "/refBy" BuildAnnotationsGenBehavior = konfig.ConfigAnnoDomain + "/generatorBehavior" BuildAnnotationsGenAddHashSuffix = konfig.ConfigAnnoDomain + "/needsHashSuffix" + + // ConfigurationAnnotationDomain is the domain of function configuration + // annotations + ConfigurationAnnotationDomain = "config.kubernetes.io" + + // Function configuration annotation + FunctionAnnotationFunction = ConfigurationAnnotationDomain + "/function" + + // true when the resource is part of the local configuration + FunctionAnnotationLocalConfig = ConfigurationAnnotationDomain + "/local-config" + + // Setting to true means we want this function configuration to be injected as a + // local configuration resource (local-config) + FunctionAnnotationInjectLocal = ConfigurationAnnotationDomain + "/inject-local" + + // if set, the transformation will remove all the resources marked as local-config + FunctionAnnotationPruneLocal = ConfigurationAnnotationDomain + "/prune-local" ) diff --git a/pkg/utils/utils.go b/pkg/utils/utils.go index a5365a3..a7b5977 100644 --- a/pkg/utils/utils.go +++ b/pkg/utils/utils.go @@ -1,6 +1,11 @@ package utils -import "sigs.k8s.io/kustomize/api/resource" +import ( + "sigs.k8s.io/kustomize/api/resource" + "sigs.k8s.io/kustomize/kyaml/kio/filters" + "sigs.k8s.io/kustomize/kyaml/kio/kioutil" + "sigs.k8s.io/kustomize/kyaml/yaml" +) var buildAnnotations = []string{ BuildAnnotationPreviousKinds, @@ -30,3 +35,15 @@ func RemoveBuildAnnotations(r *resource.Resource) { panic(err) } } + +func MakeResourceLocal(r *yaml.RNode) error { + annotations := r.GetAnnotations() + + annotations[filters.LocalConfigAnnotation] = "true" + annotations[kioutil.PathAnnotation] = ".generated.yaml" + annotations[kioutil.LegacyPathAnnotation] = ".generated.yaml" + delete(annotations, FunctionAnnotationInjectLocal) + delete(annotations, FunctionAnnotationFunction) + + return r.SetAnnotations(annotations) +} From 1c07fc03261d3e06aded701450a82d09a7c626fe Mon Sep 17 00:00:00 2001 From: Antoine Martin Date: Wed, 25 Jan 2023 23:46:19 +0000 Subject: [PATCH 08/10] =?UTF-8?q?=E2=9C=A8=20Allow=20heredoc=20resources?= =?UTF-8?q?=20and=20local=20save.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Added some annotation management to add features: - `config.kaweeze.com/inject-local` injects the function configuration as is, as if it had been generated. - `config.kaweezle.com/keep-local` allows saving a generated resource along with the other configuration files. - `config.kaweezle.com/(path|index)` allow specifying the local file and position where to save the generated resource. The astute observer have probably already seen that the specific annotations domain has been changed from `config.kubernetes.io` to `config.kaweezle.com`. --- main.go | 10 +++++++--- pkg/utils/constants.go | 10 ++++++++-- pkg/utils/utils.go | 33 +++++++++++++++++++++++++++++++-- 3 files changed, 46 insertions(+), 7 deletions(-) diff --git a/main.go b/main.go index 23e61bc..7c71ae6 100644 --- a/main.go +++ b/main.go @@ -78,9 +78,13 @@ func main() { // kustomize fn don't remove config.kubernetes.io/local-config resources upon completion. // As it always add a filename by default, the local resources keep saved. - // To avoid this, an annotation `config.kubernetes.io/prune-local` present in a + // To avoid this, an annotation `config.kaweezle.com/prune-local` present in a // transformer makes all the local resources disappear. if _, ok := config.GetAnnotations()[utils.FunctionAnnotationPruneLocal]; ok { + err = rl.Filter(utils.UnLocal) + if err != nil { + return errors.WrapPrefixf(err, "Removing local from keep-local resources") + } filter := &filters.IsLocalConfig{IncludeLocalConfig: false, ExcludeNonLocalConfig: false} err = rl.Filter(filter) if err != nil { @@ -101,11 +105,11 @@ func main() { } for _, r := range rm.Resources() { - r.RemoveBuildAnnotations() + utils.RemoveBuildAnnotations(r) // We add the annotation config.kubernetes.io/local-config to be able to delete // The generated resource at the end of the process. Unfortunately, kustomize doesn't // do that on functions. So we have added a special annotation - // `config.kubernetes.io/prune-local` to add on the last transformer. + // `config.kaweezle.com/prune-local` to add on the last transformer. // We set the filename of the generated resource in case it is forgotten. utils.MakeResourceLocal(&r.RNode) } diff --git a/pkg/utils/constants.go b/pkg/utils/constants.go index 58811f2..ae81ca4 100644 --- a/pkg/utils/constants.go +++ b/pkg/utils/constants.go @@ -17,6 +17,8 @@ const ( // annotations ConfigurationAnnotationDomain = "config.kubernetes.io" + LocalConfigurationAnnotationDomain = "config.kaweezle.com" + // Function configuration annotation FunctionAnnotationFunction = ConfigurationAnnotationDomain + "/function" @@ -25,8 +27,12 @@ const ( // Setting to true means we want this function configuration to be injected as a // local configuration resource (local-config) - FunctionAnnotationInjectLocal = ConfigurationAnnotationDomain + "/inject-local" + FunctionAnnotationInjectLocal = LocalConfigurationAnnotationDomain + "/inject-local" // if set, the transformation will remove all the resources marked as local-config - FunctionAnnotationPruneLocal = ConfigurationAnnotationDomain + "/prune-local" + FunctionAnnotationPruneLocal = LocalConfigurationAnnotationDomain + "/prune-local" + // if set on a Generated resource, it won't be pruned + FunctionAnnotationKeepLocal = LocalConfigurationAnnotationDomain + "/keep-local" + FunctionAnnotationPath = LocalConfigurationAnnotationDomain + "/path" + FunctionAnnotationIndex = LocalConfigurationAnnotationDomain + "/index" ) diff --git a/pkg/utils/utils.go b/pkg/utils/utils.go index a7b5977..e6ce47c 100644 --- a/pkg/utils/utils.go +++ b/pkg/utils/utils.go @@ -2,6 +2,7 @@ package utils import ( "sigs.k8s.io/kustomize/api/resource" + "sigs.k8s.io/kustomize/kyaml/kio" "sigs.k8s.io/kustomize/kyaml/kio/filters" "sigs.k8s.io/kustomize/kyaml/kio/kioutil" "sigs.k8s.io/kustomize/kyaml/yaml" @@ -40,10 +41,38 @@ func MakeResourceLocal(r *yaml.RNode) error { annotations := r.GetAnnotations() annotations[filters.LocalConfigAnnotation] = "true" - annotations[kioutil.PathAnnotation] = ".generated.yaml" - annotations[kioutil.LegacyPathAnnotation] = ".generated.yaml" + if _, ok := annotations[kioutil.PathAnnotation]; !ok { + annotations[kioutil.PathAnnotation] = ".generated.yaml" + } + if _, ok := annotations[kioutil.LegacyPathAnnotation]; !ok { + annotations[kioutil.LegacyPathAnnotation] = ".generated.yaml" + } delete(annotations, FunctionAnnotationInjectLocal) delete(annotations, FunctionAnnotationFunction) return r.SetAnnotations(annotations) } + +func unLocal(list []*yaml.RNode) ([]*yaml.RNode, error) { + for _, r := range list { + annotations := r.GetAnnotations() + if _, ok := annotations[FunctionAnnotationKeepLocal]; ok { + delete(annotations, FunctionAnnotationKeepLocal) + delete(annotations, FunctionAnnotationLocalConfig) + if path, ok := annotations[FunctionAnnotationPath]; ok { + annotations[kioutil.LegacyPathAnnotation] = path + annotations[kioutil.PathAnnotation] = path + delete(annotations, FunctionAnnotationPath) + } + if index, ok := annotations[FunctionAnnotationIndex]; ok { + annotations[kioutil.LegacyIndexAnnotation] = index + annotations[kioutil.IndexAnnotation] = index + delete(annotations, FunctionAnnotationIndex) + } + r.SetAnnotations(annotations) + } + } + return list, nil +} + +var UnLocal kio.FilterFunc = unLocal From 5709f7105b5ccc3a612c6481650adb3955795638 Mon Sep 17 00:00:00 2001 From: Antoine Martin Date: Wed, 25 Jan 2023 23:48:00 +0000 Subject: [PATCH 09/10] =?UTF-8?q?=F0=9F=93=9D=20documentation=20of=20the?= =?UTF-8?q?=20add=20features?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 551 +++++++++++++++++- tests/.gitignore | 1 + .../expected/sish-client.yaml | 25 + tests/multi-replacement/expected/traefik.yaml | 62 ++ .../functions/multi-transformation.yaml | 74 +++ .../original/sish-client.yaml | 26 + tests/multi-replacement/original/traefik.yaml | 60 ++ .../functions/02_replacement-transformer.yaml | 2 +- tests/test_krmfnbuiltin.sh | 12 +- 9 files changed, 779 insertions(+), 34 deletions(-) create mode 100644 tests/.gitignore create mode 100644 tests/multi-replacement/expected/sish-client.yaml create mode 100644 tests/multi-replacement/expected/traefik.yaml create mode 100644 tests/multi-replacement/functions/multi-transformation.yaml create mode 100644 tests/multi-replacement/original/sish-client.yaml create mode 100644 tests/multi-replacement/original/traefik.yaml diff --git a/README.md b/README.md index b570752..bbea0f0 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,8 @@ krmfnbuiltin is a [kustomize plugin](https://kubectl.docs.kubernetes.io/guides/extending_kustomize/) -that you can use to perform in place transformation in your kustomize projects. +providing a set of [KRM Functions] that you can use to perform in place +transformation in your kustomize projects. @@ -15,6 +16,14 @@ that you can use to perform in place transformation in your kustomize projects.
  • Rationale
  • Usage Example
  • Use of generators
    • +
  • Keeping or deleting generated resources
    • +
  • Extensions + +
  • Installation
  • Argo CD integration
  • Related projects
    • @@ -25,20 +34,21 @@ that you can use to perform in place transformation in your kustomize projects. ## Rationale `kustomize fn run` allows performing _in place_ transformation of KRM -(kubernetes Resource Model) resources. This is handy to perform modification -operations on GitOps repositories (see the [functions tutorial]). Unfortunately, -the builtin transformers are not available to `kustomize fn run`, as it expects -a `container` or `exec` annotation in the transformer resource pointing to a krm -function docker image or executable. +(Kubernetes Resource Model) resources. This is handy to perform structured +modification operations on GitOps repositories (aka _shift left_, see the +[functions tutorial] and the [KRM Functions Specification][krm functions]). +Unfortunately, the builtin transformers are not available to `kustomize fn run`, +as it expects the function to be contained in an external `container` or +`exec`utable . `krmfnbuiltin` provides both the image and executable allowing the use of any -builtin transformer or generator. +kustomize builtin transformer or generator, along with some additional goodies. ## Usage Example -Let's imagine that you have a GitOps repository containing in the `applications` -folder a list of **10** Argo CD applications. The following is the manifest for -one of them: +Let's imagine that you have a GitOps repository containing **10** Argo CD +applications in the `applications` folder. The following is the manifest for one +of them: ```yaml apiVersion: argoproj.io/v1alpha1 @@ -80,14 +90,14 @@ source: ``` Let's imagine now that you want to fork this repository for developing on -another cluster. Now you get a new repository, +another cluster. You obtain a new repository, `https://github.com/myname/autocloud.git`, on which you create a branch named `feature/experiment` for development. For the deployment to the development cluster to use the right repository and branch, you need to change `repoURL` and `targetRevision` for all the applications. You can do that by hand, but this is -**error prone**. +cumbersome and **error prone**. -This is where KRM functions shine. on a Kustomization, you would have done: +On a Kustomization, you would have done: ```yaml patches: @@ -107,8 +117,9 @@ patches: ``` But here you don't want to add a new kustomization nesting level. You just want -to modify the actual application manifests on your branch. To do that, you can -write a function: +to modify the actual application manifests on your branch. This is where KRM +functions shine. To do that, you can write a function file in a `functions` +directory: ```yaml # functions/fn-change-repo-and-branch.yaml @@ -122,7 +133,7 @@ metadata: path: krmfnbuiltin # Can also be: # container: - # image: ghcr.io/kaweezle/krmfnbuiltin:v0.0.2 + # image: ghcr.io/kaweezle/krmfnbuiltin:v0.2.0 patch: |- - op: replace path: /spec/source/repoURL @@ -138,7 +149,7 @@ target: annotationSelector: "autocloud/local-application=true" ``` -And then you can apply your modification with the following: +And then you can apply your modification with the following command: ```console > kustomize fn run --enable-exec --fn-path functions applications @@ -160,6 +171,9 @@ applications. ## Use of generators +`krmfnbuiltin` provides all the +[builtin generators](https://kubectl.docs.kubernetes.io/references/kustomize/builtins/). + Let's imagine that one or more of your applications use an Helm chart that in turn creates applications. You pass the repo URL and target branch as values to the Helm Chart with the following: @@ -227,7 +241,7 @@ metadata: namespace: argocd annotations: # Put this annotation in the last transformation to remove generated resources - config.kubernetes.io/prune-local: "true" + config.kaweezle.com/prune-local: "true" config.kubernetes.io/function: | exec: path: krmfnbuiltin @@ -257,15 +271,18 @@ replacements: Some remarks: -- ✔️ The actual values (repo url and revision) are only specified once. -- ✔️ `spec.source.helm.parameters.[name=common.repoURL].value` is path more - specific than `/spec/source/helm/parameters/1/value`. +- ✔️ The actual values (repo url and revision) are only specified once in the + config map generator. +- ✔️ `spec.source.helm.parameters.[name=common.repoURL].value` is a path more + specific than `/spec/source/helm/parameters/1/value`. The transformation would + survive reordering. - ✔️ The functions file names are prefixed with a number prefix (`01_`, `02_`) - in order to ensure that the functions are executed in the right order. + in order to ensure that the functions are executed in the right order. Note + that you can group the two functions in one file separated by `---`. - ✔️ In the last transformation, we add the following annotation: ```yaml - config.kubernetes.io/prune-local: "true" + config.kaweezle.com/prune-local: "true" ``` In order to avoid saving the generated resources. This is due to an issue in @@ -273,9 +290,476 @@ Some remarks: `config.kubernetes.io/local-config` in the case you are using `kustomize fn run` (although it works with `kustomize build`). -As a convenience, for this specific use case, we have added a -`GitConfigMapGenerator` that automatically adds the relevant resources, while -some people may consider this overkill. +## Keeping or deleting generated resources + +As said above, generated resources are saved by default beside being marked with +the `config.kubernetes.io/local-config` annotation. To prevent that, adding: + +```yaml +config.kaweezle.com/prune-local: "true" +``` + +On the last transformation will remove those resources. If the annotation is not +present, all the generated resources will be saved in a file named +`.generated.yaml` located in the configuration directory. You may want to add +this file name to your `.gitignore` file in order to avoid committing it. + +In some cases however, we want to _inject_ new resources in the configuration. +This can be done by adding the following annotations to the generator: + +- `config.kaweezle.com/keep-local` prevents the deletion of the resource when + reaching the transformation annotated with `config.kaweezle.com/prune-local`. +- `config.kaweezle.com/path` allows specifying the filename of the saved file. +- `config.kaweezle.com/index` allows specifying the position of the resource in + the file. + +Example: + +```yaml +apiVersion: builtin +kind: ConfigMapGenerator +metadata: + name: configuration-map + annotations: + config.kaweezle.com/keep-local: "true" + config.kaweezle.com/path: local-config.yaml + config.kubernetes.io/function: | + exec: + path: krmfnbuiltin +``` + +With these annotations, the generated config map will be saved in the +`local-config.yaml` file in the configuration directory. + +## Extensions + +### ConfigMap generator with git properties + +`GitConfigMapGenerator` work identically to `ConfigMapGenerator` except it adds +two properties of the current git repository to the generated config map: + +- `repoURL` contains the URL or the remote specified by `remoteName`. by + default, it takes the URL of the remote named `origin`. +- `targetRevision` contains the name of the current branch. + +This generator is useful in transformations that use those values, like for +instance Argo CD application customization. Information about the configuration +of the generator can be found in the [ConfigMapGenerator kustomize +documentation]. + +The following function configuration: + +```yaml +# 01_configmap-generator.yaml +apiVersion: builtin +kind: GitConfigMapGenerator +metadata: + name: configuration-map + annotations: + config.kubernetes.io/function: | + exec: + path: krmfnbuiltin +remoteName: origin # default +``` + +produces the following config map (comments mine): + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: configuration-map + namespace: argocd + annotations: + # add config.kaweezle.com/prune-local: "true" to last transformer to remove + config.kubernetes.io/local-config: "true" + # Add .generated.yaml to .gitignore to avoid mistakes + internal.config.kubernetes.io/path: .generated.yaml + config.kubernetes.io/path: .generated.yaml +data: + repoURL: git@github.com:kaweezle/krmfnbuiltin.git + targetRevision: feature/extended-replacement-transformer +``` + +### Heredoc generator + +Using `ConfigMapGenerator` to _inject_ values in the transformation is fine but +has some limitations, due to its _flat nature_. It cannot be used for _object_ +replacement and it's difficult to organize replacement variables. For object +replacement, you can use `PatchStrategicMergeTransformer` but then you loose the +`ReplacementTransformer` advantage of using the same source for several targets. + +`krmfnbuiltin` allows injecting any KRM resource in the transformation. Just add +the `config.kaweezle.com/inject-local: "true"` annotation. For instance: + +```yaml +apiVersion: builtin +kind: LocalConfiguration +metadata: + name: traefik-customization + annotations: + # This will inject this resource. like a ConfigMapGenerator, but with hierarchical + # properties + config.kaweezle.com/inject-local: "true" + config.kubernetes.io/function: | + exec: + path: krmfnbuiltin +data: + # kustomization + traefik: + dashboard_enabled: true + expose: true + sish: + # New properties + server: target.link + hostname: myhost.target.link + host_key: AAAAGnNrLXNzaC1lZDI1NTE5QG9wZW5zc2guY29tAAAAID+4/eqtPTLC18TE8ZP7NeF4ZP68/wnY2d7mhH/KVs79AAAABHNzaDo= +``` + +When the function configuration contains the `config.kaweezle.com/inject-local`, +annotation, `krmfnbuiltin` bypasses the generation/transformation process for +this function and return the content of the function config _as if_ it had been +generated. Its content can then be used in the following transformations, in +particular in replacements, and be deleted by the last transformation (with the +help of the `config.kaweezle.com/prune-local` annotation). + +This injection mechanism, along with the `config.kaweezle.com/keep-local` +annotation (see +[Keeping or deleting generated resources](#keeping-or-deleting-generated-resources)) +allows adding new resources to an existing configuration. + +### Extended replacement in structured content + +The `ReplacementTransformer` provided in `krmfnbuiltin` is _extended_ compared +to the standard one because it allows structured replacements in properties +containing a string representation of some structured content. It currently +supports the following structured formats: + +- YAML +- JSON +- TOML +- INI + +It also provides helpers for changing content in base64 encoded properties as +well as a simple regexp based replacer for edge cases. The standard +configuration of the transformer can be found in the [replacements kustomize +documentation]. + +The typical use case for this is when you have an Argo CD application using a +Helm chart as source with some custom values: + +```yaml +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: traefik + namespace: argocd + finalizers: + - resources-finalizer.argocd.argoproj.io +spec: + destination: + namespace: traefik + server: https://kubernetes.default.svc + project: default + source: + chart: traefik + repoURL: https://helm.traefik.io/traefik + targetRevision: "10.19.5" + helm: + parameters: [] + values: |- + ingressClass: + enabled: true + isDefaultClass: true + ingressRoute: + dashboard: + enabled: false + providers: + kubernetesCRD: + allowCrossNamespace: true + allowExternalNameServices: true + kubernetesIngress: + allowExternalNameServices: true + publishedService: + enabled: true + logs: + general: + level: ERROR + access: + enabled: true + tracing: + instana: false + gobalArguments: {} + # BEWARE: use only for debugging + additionalArguments: + - --api.insecure=false + ports: + # BEWARE: use only for debugging + # traefik: + # expose: false + web: + redirectTo: websecure + websecure: + tls: + enabled: true + syncPolicy: + automated: + prune: true + selfHeal: true + syncOptions: + - CreateNamespace=true + ignoreDifferences: [] +``` + +And that you want your KRM function to personalize the values of the Helm chart. +What you would want is having your replacement path _follow inside_ the values +property by specifying: + +```yaml +- spec.source.helm.values..ingressRoute.dashboard.enabled +``` + +This is not possible with the standard `ReplacementTransformer`, but this is is +possible with the one provided by `krmfnbuiltin`. Consider the following +function configurations: + +```yaml +# fn-traefik-customization.yaml +apiVersion: builtin +kind: LocalConfiguration +metadata: + name: traefik-customization + annotations: + # This will inject this resource. like a ConfigMapGenerator, but with hierarchical + # properties + config.kaweezle.com/inject-local: "true" + config.kubernetes.io/function: | + exec: + path: krmfnbuiltin +data: + # kustomization + traefik: + dashboard_enabled: true + expose: true +--- +apiVersion: builtin +kind: ReplacementTransformer +metadata: + name: replacement-transformer + annotations: + # remove LocalConfiguration after + config.kaweezle.com/prune-local: "true" + config.kubernetes.io/function: | + exec: + path: krmfnbuiltin +replacements: + - source: + kind: LocalConfiguration + fieldPath: data.traefik.dashboard_enabled + targets: + - select: + kind: Application + name: traefik + fieldPaths: + # !!yaml tells the transformer that the property contains YAML + - spec.source.helm.values.!!yaml.ingressRoute.dashboard.enabled + - source: + kind: LocalConfiguration + fieldPath: data.traefik.expose + targets: + - select: + kind: Application + name: traefik + fieldPaths: + - spec.source.helm.values.!!yaml.ports.traefik.expose +``` + +If you apply this to the directory containing the application, you will obtain a +new application: + +```yaml +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: traefik + namespace: argocd + finalizers: + - resources-finalizer.argocd.argoproj.io + annotations: + config.kubernetes.io/path: traefik.yaml + internal.config.kubernetes.io/path: traefik.yaml +spec: + destination: + namespace: traefik + server: https://kubernetes.default.svc + project: default + source: + chart: traefik + helm: + parameters: [] + values: | + ... + ingressRoute: + dashboard: + enabled: true + ... + ports: + # BEWARE: use only for debugging + # traefik: + # expose: false + web: + redirectTo: websecure + websecure: + tls: + enabled: true + traefik: + expose: true + repoURL: https://helm.traefik.io/traefik + targetRevision: "10.19.5" + syncPolicy: + automated: + prune: true + selfHeal: true + syncOptions: + - CreateNamespace=true + ignoreDifferences: [] +``` + +As you can see, inside the `values` property, the yaml has been modified. +`ingressRoute.dashboard.enabled` is now `true` and `port.traefik.expose` is also +`true`. Notice that this last property, also present as a comment, has been +inserted at the end of the `ports` section. + +Now for a more _extreme_ use case involving regular expressions, imagine you +have the following configuration map defining two files: + +```yaml +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: sish-client + namespace: traefik + labels: + app.kubernetes.io/name: "sish-client" + app.kubernetes.io/component: edge + app.kubernetes.io/part-of: autocloud +data: + # ~/.ssh/config file + config: | + PubkeyAcceptedKeyTypes +ssh-rsa + Host sishserver + HostName holepunch.in + Port 2222 + BatchMode yes + IdentityFile ~/.ssh_keys/id_rsa + IdentitiesOnly yes + LogLevel ERROR + ServerAliveInterval 10 + ServerAliveCountMax 2 + RemoteCommand sni-proxy=true + RemoteForward citest.holepunch.in:443 traefik.traefik.svc:443 + # ~/.ssh/known_hosts with the server key + known_hosts: | + [holepunch.in]:2222 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAID+3abW2y3T5dodnI5O1Z/2KlIdH3bwnbGDvCFf13zlh +``` + +And imagine you want to modify it to access a different server on another domain +name. You need to change: + +- `HostName` in `~/.ssh/config` from `holepunch.in` to the new server address. +- `RemoteForward` in `~/.ssh/config` by changing the address forwarded from + `citest.holepunch.in` to the new address. +- In `~/.ssh/known_hosts` the name of the host and the key fingerprint of the + new server. + +You can do this by hand, but you may forget something now and the next time. +This is where the regexp transformer comes into play with the following +configuration: + +```yaml +apiVersion: builtin +kind: LocalConfiguration +metadata: + name: configuration-map + annotations: + config.kaweezle.com/inject-local: "true" + config.kubernetes.io/function: | + exec: + path: ../../krmfnbuiltin +data: + sish: + # New properties + server: target.link + hostname: myhost.target.link + host_key: AAAAGnNrLXNzaC1lZDI1NTE5QG9wZW5zc2guY29tAAAAID+4/eqtPTLC18TE8ZP7NeF4ZP68/wnY2d7mhH/KVs79AAAABHNzaDo= +--- +apiVersion: builtin +kind: ReplacementTransformer +metadata: + name: replacement-transformer + annotations: + config.kaweezle.com/prune-local: "true" + config.kubernetes.io/function: | + exec: + path: ../../krmfnbuiltin +replacements: + - source: + kind: LocalConfiguration + fieldPath: data.sish.server + targets: + - select: + kind: ConfigMap + name: sish-client + fieldPaths: + - data.config.!!regex.^\s+HostName\s+(\S+)\s*$.1 + - data.known_hosts.!!regex.^\[(\S+)\].1 + - source: + kind: LocalConfiguration + fieldPath: data.sish.hostname + targets: + - select: + kind: ConfigMap + name: sish-client + fieldPaths: + - data.config.!!regex.^\s+RemoteForward\s+(\S+):.1 + - source: + kind: LocalConfiguration + fieldPath: data.sish.host_key + targets: + - select: + kind: ConfigMap + name: sish-client + fieldPaths: + - data.known_hosts.!!regex.ssh-ed25519\s(\S+).1 +``` + +The _path_ after `!!regex` is composed of two elements. The first one is the +regexp to match. The second one is the the capture group that needs to be +replaced with the source. In the first replacement, the regexp: + +```regexp +^\s+HostName\s+(\S+)\s*$ +``` + +can be interpreted as: + +> a line starting with one or more spaces followed by `HostName`, then one or +> more spaces and a sequence of non space characters, captured as a group; then +> optional spaces till the end of the line. + +The second part of the path, `1`, tells to replace the first capturing group +with the source. With the above, the line: + +```sshconfig + HostName holepunch.in +``` + +will become + +```sshconfig + HostName target.link +``` ## Installation @@ -284,14 +768,15 @@ provide binaries for most platforms as well as Alpine based packages. Typically, you would install it on linux with the following command: ```console -> KRMFNBUILTIN_VERSION="v0.1.0" +> KRMFNBUILTIN_VERSION="v0.2.0" > curl -sLo /usr/local/bin/krmfnbuiltin https://github.com/kaweezle/krmfnbuiltin/releases/download/${KRMFNBUILTIN_VERSION}/krmfnbuiltin_${KRMFNBUILTIN_VERSION}_linux_amd64 ``` ## Argo CD integration `krmfnbuiltin` is **NOT** primarily meant to be used inside Argo CD, but instead -to perform _structural_ modifications to the source **BEFORE** the commit. +to perform _structural_ modifications to the configuration **BEFORE** it's +committed and provided to GitOps. Anyway, to use `krmfnbuiltin` with Argo CD, you need to: @@ -312,7 +797,7 @@ summarize: ```Dockerfile FROM argoproj/argocd:latest -ARG KRMFNBUILTIN_VERSION=v0.1.0 +ARG KRMFNBUILTIN_VERSION=v0.2.0 # Switch to root for the ability to perform install USER root @@ -352,9 +837,17 @@ new configuration after transformation. While it has not been tested, krmfnbuiltin should work with [kpt]. +[knot8] lenses have provided the idea of extended paths. + +[KRM Functions]: https://github.com/kubernetes-sigs/kustomize/blob/master/cmd/config/docs/api-conventions/functions-spec.md [kpt]: https://kpt.dev/guides/rationale [functions tutorial]: https://github.com/kubernetes-sigs/kustomize/blob/master/cmd/config/docs/tutorials/function-basics.md +[knot8]: https://knot8.io/ +[ConfigMapGenerator kustomize documentation]: + https://kubectl.docs.kubernetes.io/references/kustomize/builtins/#_configmapgenerator_ +[replacements kustomize documentation]: https://kubectl.docs.kubernetes.io/references/kustomize/kustomization/replacements/ + diff --git a/tests/.gitignore b/tests/.gitignore new file mode 100644 index 0000000..6109723 --- /dev/null +++ b/tests/.gitignore @@ -0,0 +1 @@ +applications/ diff --git a/tests/multi-replacement/expected/sish-client.yaml b/tests/multi-replacement/expected/sish-client.yaml new file mode 100644 index 0000000..885cb26 --- /dev/null +++ b/tests/multi-replacement/expected/sish-client.yaml @@ -0,0 +1,25 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: sish-client + namespace: traefik + labels: + app.kubernetes.io/name: "sish-client" + app.kubernetes.io/component: edge + app.kubernetes.io/part-of: autocloud +data: + config: | + PubkeyAcceptedKeyTypes +ssh-rsa + Host sishserver + HostName target.link + Port 2222 + BatchMode yes + IdentityFile ~/.ssh_keys/id_rsa + IdentitiesOnly yes + LogLevel ERROR + ServerAliveInterval 10 + ServerAliveCountMax 2 + RemoteCommand sni-proxy=true + RemoteForward myhost.target.link:443 traefik.traefik.svc:443 + known_hosts: | + [target.link]:2222 ssh-ed25519 AAAAGnNrLXNzaC1lZDI1NTE5QG9wZW5zc2guY29tAAAAID+4/eqtPTLC18TE8ZP7NeF4ZP68/wnY2d7mhH/KVs79AAAABHNzaDo= diff --git a/tests/multi-replacement/expected/traefik.yaml b/tests/multi-replacement/expected/traefik.yaml new file mode 100644 index 0000000..65073ae --- /dev/null +++ b/tests/multi-replacement/expected/traefik.yaml @@ -0,0 +1,62 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: traefik + namespace: argocd + finalizers: + - resources-finalizer.argocd.argoproj.io +spec: + destination: + namespace: traefik + server: https://kubernetes.default.svc + project: default + source: + chart: traefik + helm: + parameters: [] + values: | + ingressClass: + enabled: true + isDefaultClass: true + ingressRoute: + dashboard: + enabled: true + providers: + kubernetesCRD: + allowCrossNamespace: true + allowExternalNameServices: true + kubernetesIngress: + allowExternalNameServices: true + publishedService: + enabled: true + logs: + general: + level: ERROR + access: + enabled: true + tracing: + instana: false + gobalArguments: {} + # BEWARE: use only for debugging + additionalArguments: + - --api.insecure=false + ports: + # BEWARE: use only for debugging + # traefik: + # expose: false + web: + redirectTo: websecure + websecure: + tls: + enabled: true + traefik: + expose: true + repoURL: https://helm.traefik.io/traefik + targetRevision: "10.19.5" + syncPolicy: + automated: + prune: true + selfHeal: true + syncOptions: + - CreateNamespace=true + ignoreDifferences: [] diff --git a/tests/multi-replacement/functions/multi-transformation.yaml b/tests/multi-replacement/functions/multi-transformation.yaml new file mode 100644 index 0000000..a29ede4 --- /dev/null +++ b/tests/multi-replacement/functions/multi-transformation.yaml @@ -0,0 +1,74 @@ +apiVersion: builtin +kind: LocalConfiguration +metadata: + name: configuration-map + annotations: + config.kaweezle.com/inject-local: "true" + config.kubernetes.io/function: | + exec: + path: ../../krmfnbuiltin +data: + sish: + server: target.link + hostname: myhost.target.link + host_key: AAAAGnNrLXNzaC1lZDI1NTE5QG9wZW5zc2guY29tAAAAID+4/eqtPTLC18TE8ZP7NeF4ZP68/wnY2d7mhH/KVs79AAAABHNzaDo= + traefik: + dashboard_enabled: true + expose: true +--- +apiVersion: builtin +kind: ReplacementTransformer +metadata: + name: replacement-transformer + annotations: + config.kubernetes.io/prune-local: "true" + config.kubernetes.io/function: | + exec: + path: ../../krmfnbuiltin +replacements: + - source: + kind: LocalConfiguration + fieldPath: data.sish.server + targets: + - select: + kind: ConfigMap + name: sish-client + fieldPaths: + - data.config.!!regex.^\s+HostName\s+(\S+)\s*$.1 + - data.known_hosts.!!regex.^\[(\S+)\].1 + - source: + kind: LocalConfiguration + fieldPath: data.sish.hostname + targets: + - select: + kind: ConfigMap + name: sish-client + fieldPaths: + - data.config.!!regex.^\s+RemoteForward\s+(\S+):.1 + - source: + kind: LocalConfiguration + fieldPath: data.sish.host_key + targets: + - select: + kind: ConfigMap + name: sish-client + fieldPaths: + - data.known_hosts.!!regex.ssh-ed25519\s(\S+).1 + - source: + kind: LocalConfiguration + fieldPath: data.traefik.dashboard_enabled + targets: + - select: + kind: Application + name: traefik + fieldPaths: + - spec.source.helm.values.!!yaml.ingressRoute.dashboard.enabled + - source: + kind: LocalConfiguration + fieldPath: data.traefik.expose + targets: + - select: + kind: Application + name: traefik + fieldPaths: + - spec.source.helm.values.!!yaml.ports.traefik.expose diff --git a/tests/multi-replacement/original/sish-client.yaml b/tests/multi-replacement/original/sish-client.yaml new file mode 100644 index 0000000..8ea4619 --- /dev/null +++ b/tests/multi-replacement/original/sish-client.yaml @@ -0,0 +1,26 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: sish-client + namespace: traefik + labels: + app.kubernetes.io/name: "sish-client" + app.kubernetes.io/component: edge + app.kubernetes.io/part-of: autocloud +data: + config: | + PubkeyAcceptedKeyTypes +ssh-rsa + Host sishserver + HostName holepunch.in + Port 2222 + BatchMode yes + IdentityFile ~/.ssh_keys/id_rsa + IdentitiesOnly yes + LogLevel ERROR + ServerAliveInterval 10 + ServerAliveCountMax 2 + RemoteCommand sni-proxy=true + RemoteForward citest.holepunch.in:443 traefik.traefik.svc:443 + known_hosts: | + [holepunch.in]:2222 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAID+3abW2y3T5dodnI5O1Z/2KlIdH3bwnbGDvCFf13zlh diff --git a/tests/multi-replacement/original/traefik.yaml b/tests/multi-replacement/original/traefik.yaml new file mode 100644 index 0000000..4d40f31 --- /dev/null +++ b/tests/multi-replacement/original/traefik.yaml @@ -0,0 +1,60 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: traefik + namespace: argocd + finalizers: + - resources-finalizer.argocd.argoproj.io +spec: + destination: + namespace: traefik + server: https://kubernetes.default.svc + project: default + source: + chart: traefik + helm: + parameters: [] + values: |- + ingressClass: + enabled: true + isDefaultClass: true + ingressRoute: + dashboard: + enabled: false + providers: + kubernetesCRD: + allowCrossNamespace: true + allowExternalNameServices: true + kubernetesIngress: + allowExternalNameServices: true + publishedService: + enabled: true + logs: + general: + level: ERROR + access: + enabled: true + tracing: + instana: false + gobalArguments: {} + # BEWARE: use only for debugging + additionalArguments: + - --api.insecure=false + ports: + # BEWARE: use only for debugging + # traefik: + # expose: false + web: + redirectTo: websecure + websecure: + tls: + enabled: true + repoURL: https://helm.traefik.io/traefik + targetRevision: "10.19.5" + syncPolicy: + automated: + prune: true + selfHeal: true + syncOptions: + - CreateNamespace=true + ignoreDifferences: [] diff --git a/tests/replacement/functions/02_replacement-transformer.yaml b/tests/replacement/functions/02_replacement-transformer.yaml index 1f56b17..264810f 100644 --- a/tests/replacement/functions/02_replacement-transformer.yaml +++ b/tests/replacement/functions/02_replacement-transformer.yaml @@ -4,7 +4,7 @@ metadata: name: replacement-transformer namespace: argocd annotations: - config.kubernetes.io/prune-local: "true" + config.kaweezle.com/prune-local: "true" config.kubernetes.io/function: | exec: path: ../../krmfnbuiltin diff --git a/tests/test_krmfnbuiltin.sh b/tests/test_krmfnbuiltin.sh index ff78ecb..cf90fd7 100755 --- a/tests/test_krmfnbuiltin.sh +++ b/tests/test_krmfnbuiltin.sh @@ -7,16 +7,20 @@ #set -uexo pipefail set -e pipefail -trap "rm -rf {patch,replacement}/applications" EXIT +trap "find . -type d -name 'applications' -exec rm -rf {} +" EXIT -for d in patch replacement; do +for d in $(ls -d */); do echo "Running Test in $d..." cd $d - rm -rf appllications + rm -rf applications cp -r original applications + echo " > Performing kustomizations..." kustomize fn run --enable-exec --fn-path functions applications - diff <(yq eval -P expected/argocd.yaml) <(yq eval -P applications/argocd.yaml) + for f in $(ls -1 expected); do + echo " > Checking $f..." + diff <(yq eval -P expected/$f) <(yq eval -P applications/$f) + done cd .. done echo "Done ok 🎉" From d7f3b821b3f527fa3b937f9467d1d2c045ffdb57 Mon Sep 17 00:00:00 2001 From: Antoine Martin Date: Thu, 26 Jan 2023 00:28:42 +0000 Subject: [PATCH 10/10] =?UTF-8?q?=F0=9F=94=96=20Prepare=20version=200.2.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.go b/main.go index 7c71ae6..763602f 100644 --- a/main.go +++ b/main.go @@ -124,7 +124,7 @@ func main() { cmd := command.Build(processor, command.StandaloneDisabled, false) command.AddGenerateDockerfile(cmd) - cmd.Version = "v0.1.0" // <---VERSION---> + cmd.Version = "v0.2.0" // <---VERSION---> if err := cmd.Execute(); err != nil { os.Exit(1)