diff --git a/go.mod b/go.mod index fa7aa89..72116b0 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,6 @@ require ( github.com/go-git/go-git/v5 v5.5.2 github.com/lithammer/dedent v1.1.0 github.com/stretchr/testify v1.8.1 - golang.org/x/text v0.6.0 golang.org/x/tools v0.5.0 sigs.k8s.io/kustomize/api v0.12.1 sigs.k8s.io/kustomize/kyaml v0.13.10 @@ -56,6 +55,7 @@ require ( golang.org/x/mod v0.7.0 // indirect golang.org/x/net v0.5.0 // indirect golang.org/x/sys v0.4.0 // indirect + golang.org/x/text v0.6.0 // indirect google.golang.org/protobuf v1.28.0 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect diff --git a/main.go b/main.go index 922370d..0915f19 100644 --- a/main.go +++ b/main.go @@ -5,8 +5,8 @@ import ( "os" "github.com/kaweezle/krmfnbuiltin/pkg/plugins" + "github.com/kaweezle/krmfnbuiltin/pkg/utils" - "sigs.k8s.io/kustomize/api/konfig" "sigs.k8s.io/kustomize/api/resmap" "sigs.k8s.io/kustomize/api/resource" "sigs.k8s.io/kustomize/kyaml/errors" @@ -18,42 +18,6 @@ import ( "sigs.k8s.io/kustomize/kyaml/yaml" ) -const ( - // build annotations - BuildAnnotationPreviousKinds = konfig.ConfigAnnoDomain + "/previousKinds" - BuildAnnotationPreviousNames = konfig.ConfigAnnoDomain + "/previousNames" - BuildAnnotationPrefixes = konfig.ConfigAnnoDomain + "/prefixes" - BuildAnnotationSuffixes = konfig.ConfigAnnoDomain + "/suffixes" - BuildAnnotationPreviousNamespaces = konfig.ConfigAnnoDomain + "/previousNamespaces" - BuildAnnotationsRefBy = konfig.ConfigAnnoDomain + "/refBy" - BuildAnnotationsGenBehavior = konfig.ConfigAnnoDomain + "/generatorBehavior" - BuildAnnotationsGenAddHashSuffix = konfig.ConfigAnnoDomain + "/needsHashSuffix" -) - -var BuildAnnotations = []string{ - BuildAnnotationPreviousKinds, - BuildAnnotationPreviousNames, - BuildAnnotationPrefixes, - BuildAnnotationSuffixes, - BuildAnnotationPreviousNamespaces, - BuildAnnotationsRefBy, - BuildAnnotationsGenBehavior, - BuildAnnotationsGenAddHashSuffix, -} - -func RemoveBuildAnnotations(r *resource.Resource) { - annotations := r.GetAnnotations() - if len(annotations) == 0 { - return - } - for _, a := range BuildAnnotations { - delete(annotations, a) - } - if err := r.SetAnnotations(annotations); err != nil { - panic(err) - } -} - func main() { var processor framework.ResourceListProcessorFunc = func(rl *framework.ResourceList) error { @@ -95,7 +59,7 @@ func main() { } for _, r := range rm.Resources() { - RemoveBuildAnnotations(r) + utils.RemoveBuildAnnotations(r) } rl.Items = rm.ToRNodeSlice() diff --git a/pkg/extras/GitConfigMapGenerator.go b/pkg/extras/GitConfigMapGenerator.go index 5ee2665..b67f0c8 100644 --- a/pkg/extras/GitConfigMapGenerator.go +++ b/pkg/extras/GitConfigMapGenerator.go @@ -11,13 +11,28 @@ import ( "sigs.k8s.io/yaml" ) +// GitConfigMapGeneratorPlugin Generates a config map that includes two +// properties of the current git repository: +// +// - repoURL contains the URL or the remote specified by remoteName. by +// default, it takes the URL of the remote named "origin". +// - targetRevision contains the name of the current branch. +// +// This generator is useful in transformations that use those values, like for +// instance Argo CD application customization. +// +// Information about the configuration can be found in the [kustomize doc]. +// +// [kustomize doc]: https://kubectl.docs.kubernetes.io/references/kustomize/builtins/#_configmapgenerator_ type GitConfigMapGeneratorPlugin struct { h *resmap.PluginHelpers types.ObjectMeta `json:"metadata,omitempty" yaml:"metadata,omitempty" protobuf:"bytes,1,opt,name=metadata"` types.ConfigMapArgs + // The name of the remote which URL to include. defaults to "origin". RemoteName string `json:"remoteName,omitempty" yaml:"remoteName,omitempty"` } +// Config configures the generator with the functionConfig passed in config. func (p *GitConfigMapGeneratorPlugin) Config(h *resmap.PluginHelpers, config []byte) (err error) { p.ConfigMapArgs = types.ConfigMapArgs{} err = yaml.Unmarshal(config, p) @@ -31,6 +46,7 @@ func (p *GitConfigMapGeneratorPlugin) Config(h *resmap.PluginHelpers, config []b return } +// Generate generates the config map func (p *GitConfigMapGeneratorPlugin) Generate() (resmap.ResMap, error) { // Add git repository properties @@ -63,6 +79,7 @@ func (p *GitConfigMapGeneratorPlugin) Generate() (resmap.ResMap, error) { kv.NewLoader(p.h.Loader(), p.h.Validator()), p.ConfigMapArgs) } +// NewGitConfigMapGeneratorPlugin returns a newly created GitConfigMapGenerator. func NewGitConfigMapGeneratorPlugin() resmap.GeneratorPlugin { return &GitConfigMapGeneratorPlugin{} } diff --git a/pkg/extras/doc.go b/pkg/extras/doc.go new file mode 100644 index 0000000..9ee51ba --- /dev/null +++ b/pkg/extras/doc.go @@ -0,0 +1,49 @@ +/* +Package extras contains additional utility transformers and generators. + +[GitConfigMapGeneratorPlugin] is identical to ConfigMapGeneratorPlugin +but automatically creates two properties when run inside a git repository: + + - repoURL gives the URL of the origin remote. + - targetRevision gives the current branch. + +[ExtendedReplacementTransformerPlugin] is a copy of ReplacementTransformerPlugin +that provides extended target paths into embedded data structures. For instance, +consider the following resource snippet: + + helm: + parameters: + - name: common.targetRevision + # This resource is accessible by traditional transformer + value: deploy/citest + - name: common.repoURL + value: https://github.com/antoinemartin/autocloud.git + values: | + uninode: true + apps: + enabled: true + common: + # This embedded resource is not accessible + targetRevision: deploy/citest + repoURL: https://github.com/antoinemartin/autocloud.git + +In the above, the common.targetRevision property of the yaml embedded in the +spec.source.helm.values property is not accessible with the traditional +ReplacementTransformerPlugin. With the extended transformer, you can target +it with: + + fieldPaths: + - spec.source.helm.parameters.[name=common.targetRevision].value + - spec.source.helm.values.!!yaml.common.targetRevision + +Note the use of !!yaml to designate the encoding of the embedded structure. The +extended transformer supports the following encodings: + + - YAML + - JSON + - TOML + - INI + - base64 + - Plain text (with Regexp) +*/ +package extras diff --git a/pkg/extras/extender.go b/pkg/extras/extender.go index fd451ae..ccd50ca 100644 --- a/pkg/extras/extender.go +++ b/pkg/extras/extender.go @@ -11,26 +11,46 @@ import ( "github.com/go-ini/ini" "github.com/pelletier/go-toml/v2" - "golang.org/x/text/cases" - "golang.org/x/text/language" "sigs.k8s.io/kustomize/kyaml/errors" "sigs.k8s.io/kustomize/kyaml/kio" "sigs.k8s.io/kustomize/kyaml/kio/kioutil" "sigs.k8s.io/kustomize/kyaml/yaml" ) +// Extender allows both traversing and modifying hierarchical opaque data +// structures like yaml, toml or ini files. +// Part of the structure is addressed through a path that is an array of string +// symbols. +// +// - It is first initialized with SetPayload with the data structure payload. +// - Traversal is done with Get +// - Modification of part of the structure is done through Set +// - After modification, the modified payload is retrieved with GetPayload type Extender interface { + // SetPayload initialize the embedded data structure with payload. SetPayload(payload []byte) error + // GetPayload returns the current data structure in the appropriate encoding. GetPayload() ([]byte, error) + // Get returns the subset of the structure at path in the appropriate encoding. Get(path []string) ([]byte, error) + // Set modifies the data structure at path with value. Value can either be + // in the appropriate encoding or can be encoded by the Extender. Please + // see the Extender documentation to see how the the value is treated. Set(path []string, value any) error } +// ExtendedSegment contains the path segment of a resource inside an embedded +// data structure. type ExtendedSegment struct { - Encoding string - Path []string + Encoding string // The encoding of the embedded data structure + Path []string // The path inside the embedded data structure } +// String returns a string representation of the ExtendedSegment. +// +// For instance: +// +// !!yaml.common.targetRevision func (e *ExtendedSegment) String() string { if len(e.Path) > 0 { return fmt.Sprintf("!!%s", e.Encoding) @@ -41,6 +61,8 @@ func (e *ExtendedSegment) String() string { type any interface{} +// ExtenderType enumerates the existing extender types. +// //go:generate go run golang.org/x/tools/cmd/stringer -type=ExtenderType type ExtenderType int @@ -54,13 +76,15 @@ const ( IniExtender ) +// stringToExtenderTypeMap maps encoding names to the corresponding extender var stringToExtenderTypeMap map[string]ExtenderType func init() { //nolint:gochecknoinits stringToExtenderTypeMap = makeStringToExtenderTypeMap() } -func GetByteValue(value any) []byte { +// getByteValue returns value encoded as a byte array. +func getByteValue(value any) []byte { switch v := value.(type) { case *yaml.Node: return []byte(v.Value) @@ -72,16 +96,20 @@ func GetByteValue(value any) []byte { return []byte{} } +// makeStringToExtenderTypeMap makes a map to get the appropriate +// [ExtenderType] given its name. func makeStringToExtenderTypeMap() (result map[string]ExtenderType) { result = make(map[string]ExtenderType, 3) for k := range ExtenderFactories { - result[k.String()] = k + result[strings.Replace(strings.ToLower(k.String()), "extender", "", 1)] = k } return } -func GetExtenderType(n string) ExtenderType { - result, ok := stringToExtenderTypeMap[n] +// getExtenderType returns the appropriate [ExtenderType] for the passed +// extender type name +func getExtenderType(n string) ExtenderType { + result, ok := stringToExtenderTypeMap[strings.ToLower(n)] if ok { return result } @@ -92,11 +120,18 @@ func GetExtenderType(n string) ExtenderType { // YAML Extender //////////////// +// yamlExtender manages embedded YAML in KRM resources. +// +// Internally, it uses a RNode. It avoids additional dependencies and preserves +// ordering and comments. type yamlExtender struct { node *yaml.RNode } -func readPayload(payload []byte) (*yaml.RNode, error) { +// parsePayload parses payload into a RNode. +// +// The payload can either by in YAML or JSON format. +func parsePayload(payload []byte) (*yaml.RNode, error) { nodes, err := (&kio.ByteReader{ Reader: bytes.NewBuffer(payload), OmitReaderAnnotations: false, @@ -110,22 +145,26 @@ func readPayload(payload []byte) (*yaml.RNode, error) { return nodes[0], nil } +// SetPayload parses payload an sets the extender internal state func (e *yamlExtender) SetPayload(payload []byte) (err error) { - e.node, err = readPayload(payload) + e.node, err = parsePayload(payload) return } -func getNodeBytes(nodes []*yaml.RNode) ([]byte, error) { +// serializeNodes serialize nodes into YAML +func serializeNodes(nodes []*yaml.RNode) ([]byte, error) { var b bytes.Buffer err := (&kio.ByteWriter{Writer: &b}).Write(nodes) return b.Bytes(), err } +// GetPayload returns the current payload in the proper encoding func (e *yamlExtender) GetPayload() ([]byte, error) { - return getNodeBytes([]*yaml.RNode{e.node}) + return serializeNodes([]*yaml.RNode{e.node}) } -func LookupNode(node *yaml.RNode) *yaml.RNode { +// unwrapSeqNode unwraps node if it is a Wrapped Bare Seq Node +func unwrapSeqNode(node *yaml.RNode) *yaml.RNode { seqNode, err := node.Pipe(yaml.Lookup(yaml.BareSeqNodeWrappingKey)) if err == nil && !seqNode.IsNilOrEmpty() { return seqNode @@ -133,8 +172,10 @@ func LookupNode(node *yaml.RNode) *yaml.RNode { return node } +// Lookup looks for the specified path in node and return the matching nodes. func Lookup(node *yaml.RNode, path []string) ([]*yaml.RNode, error) { - node, err := LookupNode(node).Pipe(&yaml.PathMatcher{Path: path}) + // TODO: consider using yaml.PathGetter instead + node, err := unwrapSeqNode(node).Pipe(&yaml.PathMatcher{Path: path}) if err != nil { return nil, errors.WrapPrefixf(err, "while getting path %s", strings.Join(path, ".")) } @@ -142,6 +183,7 @@ func Lookup(node *yaml.RNode, path []string) ([]*yaml.RNode, error) { return node.Elements() } +// Get returns the encoded payload at the specified path func (e *yamlExtender) Get(path []string) ([]byte, error) { targetFields, err := Lookup(e.node, path) if err != nil { @@ -152,9 +194,10 @@ func (e *yamlExtender) Get(path []string) ([]byte, error) { return []byte(targetFields[0].YNode().Value), nil } - return getNodeBytes(targetFields) + return serializeNodes(targetFields) } +// Set modifies the current payload with value at the specified path. func (e *yamlExtender) Set(path []string, value any) error { targetFields, err := Lookup(e.node, path) @@ -164,7 +207,7 @@ func (e *yamlExtender) Set(path []string, value any) error { for _, t := range targetFields { if t.YNode().Kind == yaml.ScalarNode { - t.YNode().Value = string(GetByteValue(value)) + t.YNode().Value = string(getByteValue(value)) } else { if v, ok := value.(*yaml.Node); ok { t.SetYNode(v) @@ -176,6 +219,10 @@ func (e *yamlExtender) Set(path []string, value any) error { return nil } +// NewYamlExtender returns a newly created YAML [Extender]. +// +// With this encoding, you can set scalar values (strings, numbers) as well +// as mapping values. func NewYamlExtender() Extender { return &yamlExtender{} } @@ -184,10 +231,12 @@ func NewYamlExtender() Extender { // Base64 ///////// +// base64Extender manages embedded base64 in KRM resources. type base64Extender struct { - decoded []byte + decoded []byte // The base64 decoded payload } +// SetPayload decodes the payload and stores in internal state. func (e *base64Extender) SetPayload(payload []byte) error { decoded, err := base64.StdEncoding.DecodeString(string(payload)) if err != nil { @@ -197,10 +246,14 @@ func (e *base64Extender) SetPayload(payload []byte) error { return nil } +// GetPayload returns the current payload as base64 func (e *base64Extender) GetPayload() ([]byte, error) { - return e.decoded, nil + return []byte(base64.StdEncoding.EncodeToString(e.decoded)), nil } +// Get returns the current base64 decoded payload. +// +// An error is returned if the path is not empty. func (e *base64Extender) Get(path []string) ([]byte, error) { if len(path) > 0 { return nil, fmt.Errorf("path is invalid for base64: %s", strings.Join(path, ".")) @@ -208,14 +261,24 @@ func (e *base64Extender) Get(path []string) ([]byte, error) { return e.decoded, nil } +// Set stores value in the current payload. path must be empty. func (e *base64Extender) Set(path []string, value any) error { if len(path) > 0 { return fmt.Errorf("path is invalid for base64: %s", strings.Join(path, ".")) } - e.decoded = []byte(base64.StdEncoding.EncodeToString(GetByteValue(value))) + e.decoded = getByteValue(value) return nil } +// NewBase64Extender returns a newly created Base64 extender. +// +// This extender doesn't allow structured traversal and modification. It just +// passes its decoded payload downstream. Example of usage: +// +// prefix.!!base64.!!yaml.inside.path +// +// The above means that we want to modify inside.path in the YAML payload that +// is stored in base64 in the prefix property. func NewBase64Extender() Extender { return &base64Extender{} } @@ -224,19 +287,26 @@ func NewBase64Extender() Extender { // Regex //////// +// regexExtender allows text replacement in pure text properties. +// +// see [NewRegexExtender] type regexExtender struct { text []byte } +// SetPayload store the plain payload internally func (e *regexExtender) SetPayload(payload []byte) error { e.text = payload return nil } +// GetPayload returns the text payload func (e *regexExtender) GetPayload() ([]byte, error) { return []byte(e.text), nil } +// Get returns the text matched by the regexp contained in the first segment of +// path. func (e *regexExtender) Get(path []string) ([]byte, error) { if len(path) < 1 { return nil, fmt.Errorf("path for regex should at least be one") @@ -248,6 +318,18 @@ func (e *regexExtender) Get(path []string) ([]byte, error) { return re.Find(e.text), nil } +// Set modifies the inner text inserting value in the capture group specified by +// path[1] of the Regexp specified by path[0]. +// +// Example paths: +// +// [`^\s+HostName\s+(\S+)\s*$`, `1`] +// +// Changes the value after HostName with value. +// +// [`^\s+HostName\s+\S+\s*$`, `0`] +// +// Replace the whole line with value. func (e *regexExtender) Set(path []string, value any) error { if len(path) != 2 { return fmt.Errorf("path for regex should at least be one") @@ -271,7 +353,7 @@ func (e *regexExtender) Set(path []string, value any) error { startIndex := group * 2 b.Write(e.text[start:v[startIndex]]) - b.Write(GetByteValue(value)) + b.Write(getByteValue(value)) start = v[startIndex+1] } @@ -285,6 +367,27 @@ func (e *regexExtender) Set(path []string, value any) error { return nil } +// NewRegexExtender returns a newly created Regexp [Extender]. +// +// This extender allows text replacement in pure text properties. It is useful +// in the case the content of the KRM property is not structured. +// +// We don't recommend using it too much as it weakens the transformation. +// +// The paths to use with this extender are always composed of two elements: +// +// - The regexp to look for in the text. +// - The capture group index to replace with the source value. +// +// Examples: +// +// ^\s+HostName\s+(\S+)\s*$.1 +// +// Changes the value after HostName with value. +// +// ^\s+HostName\s+\S+\s*$.0 +// +// Replace the whole line with value. func NewRegexExtender() Extender { return ®exExtender{} } @@ -293,15 +396,23 @@ func NewRegexExtender() Extender { // JSON /////// +// jsonExtender is an [Extender] allowing modifications in JSON content. +// +// It is close to [yamlExtender] as kyaml knows to read and write JSON files. type jsonExtender struct { node *yaml.RNode } +// SetPayload parses the JSON payload and stores it internally as a yaml.RNode. func (e *jsonExtender) SetPayload(payload []byte) (err error) { - e.node, err = readPayload(payload) + e.node, err = parsePayload(payload) return } +// getJSONPayload returns the JSON payload for the passed node. +// +// There is a small issue in kio.ByteWriter preventing the JSON serialization of +// a wrapped JSON array. func getJSONPayload(node *yaml.RNode) ([]byte, error) { var b bytes.Buffer if node.YNode().Kind == yaml.MappingNode { @@ -318,10 +429,12 @@ func getJSONPayload(node *yaml.RNode) ([]byte, error) { return b.Bytes(), err } +// GetPayload returns the payload as a serialized JSON object func (e *jsonExtender) GetPayload() ([]byte, error) { - return getJSONPayload(LookupNode(e.node)) + return getJSONPayload(unwrapSeqNode(e.node)) } +// Get returns the sub JSON specified by path. func (e *jsonExtender) Get(path []string) ([]byte, error) { targetFields, err := Lookup(e.node, path) if err != nil { @@ -340,6 +453,7 @@ func (e *jsonExtender) Get(path []string) ([]byte, error) { return getJSONPayload(target) } +// Set modifies the inner JSON at path with value func (e *jsonExtender) Set(path []string, value any) error { targetFields, err := Lookup(e.node, path) @@ -349,7 +463,7 @@ func (e *jsonExtender) Set(path []string, value any) error { for _, t := range targetFields { if t.YNode().Kind == yaml.ScalarNode { - t.YNode().Value = string(GetByteValue(value)) + t.YNode().Value = string(getByteValue(value)) } else { if v, ok := value.(*yaml.Node); ok { t.SetYNode(v) @@ -361,6 +475,10 @@ func (e *jsonExtender) Set(path []string, value any) error { return nil } +// NewJsonExtender returns a newly created [Extender] to modify JSON content. +// +// As with the YAML extender (see [NewYamlExtender]), modifications are not +// limited to scalar values but the source can be a mapping or a sequence. func NewJsonExtender() Extender { return &jsonExtender{} } @@ -369,10 +487,13 @@ func NewJsonExtender() Extender { // TOML /////// +// tomlExtender is an [Extender] allowing the structured modification of a TOML +// property. type tomlExtender struct { node *yaml.RNode } +// SetPayload sets the internal state with the TOML source payload. func (e *tomlExtender) SetPayload(payload []byte) error { m := map[string]interface{}{} @@ -389,6 +510,9 @@ func (e *tomlExtender) SetPayload(payload []byte) error { return nil } +// getTOMLPayload returns the TOML representation of the specified node. +// +// The node must be a mapping node. func getTOMLPayload(node *yaml.RNode) ([]byte, error) { m, err := node.Map() if err != nil { @@ -397,10 +521,12 @@ func getTOMLPayload(node *yaml.RNode) ([]byte, error) { return toml.Marshal(m) } +// GetPayload return the current payload as a TOML snippet. func (e *tomlExtender) GetPayload() ([]byte, error) { return getTOMLPayload(e.node) } +// Get returns the TOML representation of the sub element at path. func (e *tomlExtender) Get(path []string) ([]byte, error) { targetFields, err := Lookup(e.node, path) if err != nil { @@ -419,6 +545,7 @@ func (e *tomlExtender) Get(path []string) ([]byte, error) { return getTOMLPayload(target) } +// Set modifies the current payload at path with value. func (e *tomlExtender) Set(path []string, value any) error { targetFields, err := Lookup(e.node, path) @@ -428,7 +555,7 @@ func (e *tomlExtender) Set(path []string, value any) error { for _, t := range targetFields { if t.YNode().Kind == yaml.ScalarNode { - t.YNode().Value = string(GetByteValue(value)) + t.YNode().Value = string(getByteValue(value)) } else { if v, ok := value.(*yaml.Node); ok { t.SetYNode(v) @@ -440,30 +567,39 @@ func (e *tomlExtender) Set(path []string, value any) error { return nil } +// NewTomlExtender returns a newly created [Extender] for modifying properties +// containing TOML. +// +// Please be aware that this [Extender] doesn't preserve the source ordering +// nor the comments in the content. func NewTomlExtender() Extender { return &tomlExtender{} } -/////// -// TOML -/////// +////// +// INI +////// +// iniExtender allows structured modification of ini file based properties. type iniExtender struct { file *ini.File } +// SetPayload parses payload as a INI file and set the internal state. func (e *iniExtender) SetPayload(payload []byte) (err error) { e.file, err = ini.Load(payload) return err } +// GetPayload returns the current state as an ini file. func (e *iniExtender) GetPayload() ([]byte, error) { var b bytes.Buffer _, err := e.file.WriteTo(&b) return b.Bytes(), err } +// keyFromPath returns the INI key at path. func (e *iniExtender) keyFromPath(path []string) (*ini.Key, error) { if len(path) < 1 || len(path) > 2 { return nil, fmt.Errorf("invalid path length: %d", len(path)) @@ -477,6 +613,7 @@ func (e *iniExtender) keyFromPath(path []string) (*ini.Key, error) { return e.file.Section(section).Key(key), nil } +// Get returns the content of the key specified by path. func (e *iniExtender) Get(path []string) ([]byte, error) { k, err := e.keyFromPath(path) if err != nil { @@ -485,17 +622,29 @@ func (e *iniExtender) Get(path []string) ([]byte, error) { return []byte(k.String()), nil } +// Set sets the value of the key specified by path with value. func (e *iniExtender) Set(path []string, value any) error { k, err := e.keyFromPath(path) if err != nil { return fmt.Errorf("while getting key at path %s", strings.Join(path, ".")) } - k.SetValue(string(GetByteValue(value))) + k.SetValue(string(getByteValue(value))) return nil } +// NewIniExtender returns a newly created [Extender] for modifying INI files +// like properties. +// +// Some tools may use ini type configuration files. This extender allows +// modification of the values. At this point, it doesn't allow inserting +// complete sections. If paths have one element, it will set the corresponding +// property at the root level. If path have two elements, the first one contains +// the section name and the second the property name. +// +// Please be aware that this [Extender] doesn't preserve the source ordering +// nor the comments in the content. func NewIniExtender() Extender { return &iniExtender{} } @@ -504,6 +653,8 @@ func NewIniExtender() Extender { // Factories //////////// +// ExtenderFactories register the [Extender] factory functions for each +// [ExtenderType]. var ExtenderFactories = map[ExtenderType]func() Extender{ YamlExtender: NewYamlExtender, Base64Extender: NewBase64Extender, @@ -513,8 +664,10 @@ var ExtenderFactories = map[ExtenderType]func() Extender{ IniExtender: NewIniExtender, } +// Extender returns a newly created [Extender] for the appropriate encoding. +// uses [ExtenderFactories]. func (path *ExtendedSegment) Extender(payload []byte) (Extender, error) { - bpt := GetExtenderType(cases.Title(language.English, cases.NoLower).String(path.Encoding) + "Extender") + bpt := getExtenderType(path.Encoding) if f, ok := ExtenderFactories[bpt]; ok { result := f() if err := result.SetPayload(payload); err != nil { @@ -530,6 +683,8 @@ func (path *ExtendedSegment) Extender(payload []byte) (Extender, error) { // ExtendedPath /////////////// +// splitExtendedPath fills extensions with the ExtendedSegments found in path +// and returns the path prefix. This method is used by [NewExtendedPath] func splitExtendedPath(path []string, extensions *[]*ExtendedSegment) (basePath []string, err error) { if len(path) == 0 { @@ -559,11 +714,30 @@ func splitExtendedPath(path []string, extensions *[]*ExtendedSegment) (basePath return } +// ExtendedPath contains all the paths segments of a path. +// The path is composed by: +// +// - a KRM resource path, the prefix (ResourcePath) +// - 0 or more [ExtendedSegment]s. +// +// For instance, for the following path: +// +// data.secretConfiguration.!!base64.!!yaml.common.URL +// +// ResourcePath would be ["data", "secretConfiguration"] and ExtendedSegments: +// +// *[]*ExtendedSegment{ +// &ExtendedSegment{Encoding: "base64", Path: []string{}}, +// &ExtendedSegment{Encoding: "yaml", Path: []string{"common", "URL"}}, +// } type ExtendedPath struct { - resourcePath []string - extendedSegments *[]*ExtendedSegment + // ResourcePath is The KRM portion of the path + ResourcePath []string + // ExtendedSegments contains all extended path segments + ExtendedSegments *[]*ExtendedSegment } +// NewExtendedPath creates an [ExtendedPath] from the split path segments in paths. func NewExtendedPath(path []string) (*ExtendedPath, error) { extensions := []*ExtendedSegment{} prefix, err := splitExtendedPath(path, &extensions) @@ -571,18 +745,20 @@ func NewExtendedPath(path []string) (*ExtendedPath, error) { return nil, errors.WrapPrefixf(err, "while getting extended path") } - return &ExtendedPath{resourcePath: prefix, extendedSegments: &extensions}, nil + return &ExtendedPath{ResourcePath: prefix, ExtendedSegments: &extensions}, nil } +// HasExtensions returns true if the path contains extended segments. func (ep *ExtendedPath) HasExtensions() bool { - return len(*ep.extendedSegments) > 0 + return len(*ep.ExtendedSegments) > 0 } +// String returns a string representation of the extended path. func (ep *ExtendedPath) String() string { - out := strings.Join(ep.resourcePath, ".") - if len(*ep.extendedSegments) > 0 { + out := strings.Join(ep.ResourcePath, ".") + if len(*ep.ExtendedSegments) > 0 { segmentStrings := []string{} - for _, s := range *ep.extendedSegments { + for _, s := range *ep.ExtendedSegments { segmentStrings = append(segmentStrings, s.String()) } out = fmt.Sprintf("%s.%s", out, strings.Join(segmentStrings, ".")) @@ -590,18 +766,19 @@ func (ep *ExtendedPath) String() string { return out } -func (ep *ExtendedPath) ApplyIndex(index int, input []byte, value *yaml.Node) ([]byte, error) { - if index >= len(*ep.extendedSegments) || index < 0 { +// applyIndex applies value to input starting at the extended path index. +func (ep *ExtendedPath) applyIndex(index int, input []byte, value *yaml.Node) ([]byte, error) { + if index >= len(*ep.ExtendedSegments) || index < 0 { return nil, fmt.Errorf("invalid extended path index: %d", index) } - segment := (*ep.extendedSegments)[index] + segment := (*ep.ExtendedSegments)[index] extender, err := segment.Extender(input) if err != nil { return nil, errors.WrapPrefixf(err, "creating extender at index: %d", index) } - if index == len(*ep.extendedSegments)-1 { + if index == len(*ep.ExtendedSegments)-1 { err := extender.Set(segment.Path, value) if err != nil { return nil, errors.WrapPrefixf(err, "setting value on path %s", segment.String()) @@ -611,7 +788,7 @@ func (ep *ExtendedPath) ApplyIndex(index int, input []byte, value *yaml.Node) ([ if err != nil { return nil, errors.WrapPrefixf(err, "getting value on path %s", segment.String()) } - newValue, err := ep.ApplyIndex(index+1, nextInput, value) + newValue, err := ep.applyIndex(index+1, nextInput, value) if err != nil { return nil, err } @@ -624,15 +801,22 @@ func (ep *ExtendedPath) ApplyIndex(index int, input []byte, value *yaml.Node) ([ return extender.GetPayload() } +// Apply applies value to target. target is the KRM resource specified by +// ResourcePrefix. +// +// Apply creates the appropriate [Extender] for each extended segment and +// traverse it until the last. When reaching the last, it sets value +// in the appropriate path. It then unwinds the paths and save the modified +// value in the target. func (ep *ExtendedPath) Apply(target *yaml.RNode, value *yaml.RNode) error { if target.YNode().Kind != yaml.ScalarNode { return fmt.Errorf("extended path only works on scalar nodes") } outValue := value.YNode().Value - if len(*ep.extendedSegments) > 0 { + if len(*ep.ExtendedSegments) > 0 { input := []byte(target.YNode().Value) - output, err := ep.ApplyIndex(0, input, value.YNode()) + output, err := ep.applyIndex(0, input, value.YNode()) if err != nil { return errors.WrapPrefixf(err, "applying value on extended segment %s", ep.String()) } diff --git a/pkg/extras/extender_test.go b/pkg/extras/extender_test.go index 88bdb60..16d0ee6 100644 --- a/pkg/extras/extender_test.go +++ b/pkg/extras/extender_test.go @@ -252,11 +252,11 @@ common: path := kyaml_utils.SmarterPathSplitter(p, ".") e, err := NewExtendedPath(path) require.NoError(err) - require.Len(e.resourcePath, 1, "no resource path") + require.Len(e.ResourcePath, 1, "no resource path") sourcePath := []string{"common"} - target, err := source.Pipe(&yaml.PathGetter{Path: e.resourcePath}) + target, err := source.Pipe(&yaml.PathGetter{Path: e.ResourcePath}) require.NoError(err) value, err := replacement.Pipe(&yaml.PathGetter{Path: sourcePath}) diff --git a/pkg/extras/replacement.go b/pkg/extras/replacement.go index 578a87c..2bd9312 100644 --- a/pkg/extras/replacement.go +++ b/pkg/extras/replacement.go @@ -9,7 +9,7 @@ import ( "reflect" "strings" - "sigs.k8s.io/kustomize/api/konfig" + "github.com/kaweezle/krmfnbuiltin/pkg/utils" "sigs.k8s.io/kustomize/api/resmap" "sigs.k8s.io/kustomize/api/resource" "sigs.k8s.io/kustomize/api/types" @@ -18,12 +18,12 @@ import ( "sigs.k8s.io/kustomize/kyaml/yaml" ) -type Filter struct { +type extendedFilter struct { Replacements []types.Replacement `json:"replacements,omitempty" yaml:"replacements,omitempty"` } // Filter replaces values of targets with values from sources -func (f Filter) Filter(nodes []*yaml.RNode) ([]*yaml.RNode, error) { +func (f extendedFilter) Filter(nodes []*yaml.RNode) ([]*yaml.RNode, error) { for i, r := range f.Replacements { if r.Source == nil || r.Targets == nil { return nil, fmt.Errorf("replacements must specify a source and at least one target") @@ -67,7 +67,7 @@ func getReplacement(nodes []*yaml.RNode, r *types.Replacement) (*yaml.RNode, err func selectSourceNode(nodes []*yaml.RNode, selector *types.SourceSelector) (*yaml.RNode, error) { var matches []*yaml.RNode for _, n := range nodes { - ids, err := MakeResIds(n) + ids, err := makeResIds(n) if err != nil { return nil, fmt.Errorf("error getting node IDs: %w", err) } @@ -113,7 +113,7 @@ func applyReplacement(nodes []*yaml.RNode, value *yaml.RNode, targetSelectors [] selector.FieldPaths = []string{types.DefaultReplacementFieldPath} } for _, possibleTarget := range nodes { - ids, err := MakeResIds(possibleTarget) + ids, err := makeResIds(possibleTarget) if err != nil { return nil, err } @@ -186,21 +186,21 @@ func copyValueToTarget(target *yaml.RNode, value *yaml.RNode, selector *types.Ta if err != nil { return err } - create, err := shouldCreateField(selector.Options, extendedPath.resourcePath) + create, err := shouldCreateField(selector.Options, extendedPath.ResourcePath) if err != nil { return err } var targetFields []*yaml.RNode if create { - createdField, createErr := target.Pipe(yaml.LookupCreate(value.YNode().Kind, extendedPath.resourcePath...)) + createdField, createErr := target.Pipe(yaml.LookupCreate(value.YNode().Kind, extendedPath.ResourcePath...)) if createErr != nil { return fmt.Errorf("error creating replacement node: %w", createErr) } targetFields = append(targetFields, createdField) } else { // may return multiple fields, always wrapped in a sequence node - foundFieldSequence, lookupErr := target.Pipe(&yaml.PathMatcher{Path: extendedPath.resourcePath}) + foundFieldSequence, lookupErr := target.Pipe(&yaml.PathMatcher{Path: extendedPath.ResourcePath}) if lookupErr != nil { return fmt.Errorf("error finding field in replacement target: %w", lookupErr) } @@ -271,16 +271,8 @@ func shouldCreateField(options *types.FieldOptions, fieldPath []string) (bool, e // Copied -const ( - BuildAnnotationPreviousKinds = konfig.ConfigAnnoDomain + "/previousKinds" - BuildAnnotationPreviousNames = konfig.ConfigAnnoDomain + "/previousNames" - BuildAnnotationPrefixes = konfig.ConfigAnnoDomain + "/prefixes" - BuildAnnotationSuffixes = konfig.ConfigAnnoDomain + "/suffixes" - BuildAnnotationPreviousNamespaces = konfig.ConfigAnnoDomain + "/previousNamespaces" -) - -// MakeResIds returns all of an RNode's current and previous Ids -func MakeResIds(n *yaml.RNode) ([]resid.ResId, error) { +// makeResIds returns all of an RNode's current and previous Ids +func makeResIds(n *yaml.RNode) ([]resid.ResId, error) { var result []resid.ResId apiVersion := n.Field(yaml.APIVersionField) var group, version string @@ -290,7 +282,7 @@ func MakeResIds(n *yaml.RNode) ([]resid.ResId, error) { result = append(result, resid.NewResIdWithNamespace( resid.Gvk{Group: group, Version: version, Kind: n.GetKind()}, n.GetName(), n.GetNamespace()), ) - prevIds, err := PrevIds(n) + prevIds, err := prevIds(n) if err != nil { return nil, err } @@ -298,18 +290,18 @@ func MakeResIds(n *yaml.RNode) ([]resid.ResId, error) { return result, nil } -// PrevIds returns all of an RNode's previous Ids -func PrevIds(n *yaml.RNode) ([]resid.ResId, error) { +// prevIds returns all of an RNode's previous Ids +func prevIds(n *yaml.RNode) ([]resid.ResId, error) { var ids []resid.ResId // TODO: merge previous names and namespaces into one list of // pairs on one annotation so there is no chance of error annotations := n.GetAnnotations() - if _, ok := annotations[BuildAnnotationPreviousNames]; !ok { + if _, ok := annotations[utils.BuildAnnotationPreviousNames]; !ok { return nil, nil } - names := strings.Split(annotations[BuildAnnotationPreviousNames], ",") - ns := strings.Split(annotations[BuildAnnotationPreviousNamespaces], ",") - kinds := strings.Split(annotations[BuildAnnotationPreviousKinds], ",") + names := strings.Split(annotations[utils.BuildAnnotationPreviousNames], ",") + ns := strings.Split(annotations[utils.BuildAnnotationPreviousNamespaces], ",") + kinds := strings.Split(annotations[utils.BuildAnnotationPreviousKinds], ",") // This should never happen if len(names) != len(ns) || len(names) != len(kinds) { return nil, fmt.Errorf( @@ -336,12 +328,28 @@ func PrevIds(n *yaml.RNode) ([]resid.ResId, error) { // plugin -// Replace values in targets with values from a source +// Replace values in targets with values from a source. This transformer is +// "extended" because it allows structured replacement in properties +// containing a string representation of some structured content. It currently +// supports the following structured formats: +// +// - Yaml +// - Json +// - Toml +// - Ini +// +// It also provides helpers for changing content in base64 encoded properties +// as well as a simple regexp based replacer for edge cases. +// +// Configuration of replacements can be found in the [kustomize doc]. +// +// [kustomize doc]: https://kubectl.docs.kubernetes.io/references/kustomize/kustomization/replacements/ type ExtendedReplacementTransformerPlugin struct { ReplacementList []types.ReplacementField `json:"replacements,omitempty" yaml:"replacements,omitempty"` Replacements []types.Replacement `json:"omitempty" yaml:"omitempty"` } +// Config configures the plugin func (p *ExtendedReplacementTransformerPlugin) Config( h *resmap.PluginHelpers, c []byte) (err error) { p.ReplacementList = []types.ReplacementField{} @@ -390,12 +398,14 @@ func (p *ExtendedReplacementTransformerPlugin) Config( return nil } +// Transform performs the configured replacements in the specified resource map func (p *ExtendedReplacementTransformerPlugin) Transform(m resmap.ResMap) (err error) { - return m.ApplyFilter(Filter{ + return m.ApplyFilter(extendedFilter{ Replacements: p.Replacements, }) } +// NewExtendedReplacementTransformerPlugin returns a newly created [ExtendedReplacementTransformerPlugin] func NewExtendedReplacementTransformerPlugin() resmap.TransformerPlugin { return &ExtendedReplacementTransformerPlugin{} } diff --git a/pkg/plugins/doc.go b/pkg/plugins/doc.go new file mode 100644 index 0000000..9a11804 --- /dev/null +++ b/pkg/plugins/doc.go @@ -0,0 +1,4 @@ +/* +Package plugins is a copy of the Kustomize standard plugins public factory. +*/ +package plugins diff --git a/pkg/utils/constants.go b/pkg/utils/constants.go new file mode 100644 index 0000000..de315fa --- /dev/null +++ b/pkg/utils/constants.go @@ -0,0 +1,15 @@ +package utils + +import "sigs.k8s.io/kustomize/api/konfig" + +const ( + // build annotations + BuildAnnotationPreviousKinds = konfig.ConfigAnnoDomain + "/previousKinds" + BuildAnnotationPreviousNames = konfig.ConfigAnnoDomain + "/previousNames" + BuildAnnotationPrefixes = konfig.ConfigAnnoDomain + "/prefixes" + BuildAnnotationSuffixes = konfig.ConfigAnnoDomain + "/suffixes" + BuildAnnotationPreviousNamespaces = konfig.ConfigAnnoDomain + "/previousNamespaces" + BuildAnnotationsRefBy = konfig.ConfigAnnoDomain + "/refBy" + BuildAnnotationsGenBehavior = konfig.ConfigAnnoDomain + "/generatorBehavior" + BuildAnnotationsGenAddHashSuffix = konfig.ConfigAnnoDomain + "/needsHashSuffix" +) diff --git a/pkg/utils/doc.go b/pkg/utils/doc.go new file mode 100644 index 0000000..eff2858 --- /dev/null +++ b/pkg/utils/doc.go @@ -0,0 +1,5 @@ +/* +Package utils contains functions and constants located in kustomize internal +packages and that are needed by krmfnbuiltin. +*/ +package utils diff --git a/pkg/utils/utils.go b/pkg/utils/utils.go new file mode 100644 index 0000000..a5365a3 --- /dev/null +++ b/pkg/utils/utils.go @@ -0,0 +1,32 @@ +package utils + +import "sigs.k8s.io/kustomize/api/resource" + +var buildAnnotations = []string{ + BuildAnnotationPreviousKinds, + BuildAnnotationPreviousNames, + BuildAnnotationPrefixes, + BuildAnnotationSuffixes, + BuildAnnotationPreviousNamespaces, + BuildAnnotationsRefBy, + BuildAnnotationsGenBehavior, + BuildAnnotationsGenAddHashSuffix, +} + +// RemoveBuildAnnotations removes kustomize build annotations from r. +// +// Contrary to the method available in resource.Resource, this method doesn't +// remove the file name related annotations, as this would prevent modification +// of the source file. +func RemoveBuildAnnotations(r *resource.Resource) { + annotations := r.GetAnnotations() + if len(annotations) == 0 { + return + } + for _, a := range buildAnnotations { + delete(annotations, a) + } + if err := r.SetAnnotations(annotations); err != nil { + panic(err) + } +}