287 splitting state files according to mainsitecomponent (#324)

<!--

Describe in detail the changes you are proposing, and the rationale.

-->

<!--

Link all GitHub issues fixed by this PR, and add references to prior
related PRs.

-->

Fixes #

### NEW FEATURES | UPGRADE NOTES | ENHANCEMENTS | BUG FIXES |
EXPERIMENTS

<!--

Write a short description of your changes. Examples:

- Fixed a bug
- Added a new feature
- Updated documentation

--> 

-

.changes/unreleased/Added-20231109-164830.yaml

@@ -0,0 +1,3 @@
+kind: Added
+body: Reworked state handling to allow for separate state per component
+time: 2023-11-09T16:48:30.51867373+01:00

.changes/v2.12.3.md

@@ -0,0 +1,5 @@
+## v2.12.3 (2023-10-25)
+### Updated
+* Updated dependencies and added support for go 1.21.x
+### Fixed
+* Fix resolving plugin executables on Windows

CHANGELOG.md

@@ -6,6 +6,11 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html),
 and is generated by [Changie](https://github.com/miniscruff/changie).
 
 
+## v2.12.3 (2023-10-25)
+### Updated
+* Updated dependencies and added support for go 1.21.x
+### Fixed
+* Fix resolving plugin executables on Windows
 ## v2.12.2 (2023-09-29)
 ### Fixed
 * Fixed usage of root directory when loading referenced files

cmd/mach-composer/apply.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"github.com/mach-composer/mach-composer-cli/internal/dependency"
 	"github.com/spf13/cobra"
 
 	"github.com/mach-composer/mach-composer-cli/internal/generator"
@@ -12,6 +13,7 @@ var applyFlags struct {
 	autoApprove bool
 	destroy     bool
 	components  []string
+	numWorkers  int
 }
 
 var applyCmd = &cobra.Command{
@@ -28,10 +30,11 @@ var applyCmd = &cobra.Command{
 
 func init() {
 	registerGenerateFlags(applyCmd)
-	applyCmd.Flags().BoolVarP(&applyFlags.reuse, "reuse", "", false, "Supress a terraform init for improved speed (not recommended for production usage)")
-	applyCmd.Flags().BoolVarP(&applyFlags.autoApprove, "auto-approve", "", false, "Supress a terraform init for improved speed (not recommended for production usage)")
+	applyCmd.Flags().BoolVarP(&applyFlags.reuse, "reuse", "", false, "Suppress a terraform init for improved speed (not recommended for production usage)")
+	applyCmd.Flags().BoolVarP(&applyFlags.autoApprove, "auto-approve", "", false, "Suppress a terraform init for improved speed (not recommended for production usage)")
 	applyCmd.Flags().BoolVarP(&applyFlags.destroy, "destroy", "", false, "Destroy option is a convenient way to destroy all remote objects managed by this mach config")
 	applyCmd.Flags().StringArrayVarP(&applyFlags.components, "component", "c", []string{}, "")
+	applyCmd.Flags().IntVarP(&applyFlags.numWorkers, "workers", "w", 1, "The number of workers to use")
 }
 
 func applyFunc(cmd *cobra.Command, args []string) error {
@@ -41,18 +44,23 @@ func applyFunc(cmd *cobra.Command, args []string) error {
 
 	generateFlags.ValidateSite(cfg)
 
+	dg, err := dependency.ToDeploymentGraph(cfg)
+	if err != nil {
+		return err
+	}
+
 	// Note that we do this in multiple passes to minimize ending up with
 	// half broken runs. We could in the future also run some parts in parallel
 
-	paths, err := generator.WriteFiles(ctx, cfg, &generator.GenerateOptions{
+	err = generator.Write(ctx, cfg, dg, &generator.GenerateOptions{
 		OutputPath: generateFlags.outputPath,
 		Site:       generateFlags.siteName,
 	})
 	if err != nil {
 		return err
 	}
 
-	return runner.TerraformApply(ctx, cfg, paths, &runner.ApplyOptions{
+	return runner.TerraformApply(ctx, cfg, dg, &runner.ApplyOptions{
 		Destroy:     applyFlags.destroy,
 		Reuse:       applyFlags.reuse,
 		AutoApprove: applyFlags.autoApprove,

cmd/mach-composer/common.go

@@ -26,22 +26,22 @@ var generateFlags GenerateFlags
 
 func (gf GenerateFlags) ValidateSite(cfg *config.MachConfig) {
 	if gf.siteName != "" && !cfg.HasSite(gf.siteName) {
-		fmt.Fprintf(os.Stderr, "No site found with identifier: %s\n", gf.siteName)
+		_, _ = fmt.Fprintf(os.Stderr, "No site found with identifier: %s\n", gf.siteName)
 		os.Exit(1)
 	}
 }
 
 func registerGenerateFlags(cmd *cobra.Command) {
 	cmd.Flags().StringVarP(&generateFlags.configFile, "file", "f", "main.yml", "YAML file to parse.")
 	cmd.Flags().StringVarP(&generateFlags.varFile, "var-file", "", "", "Use a variable file to parse the configuration with.")
-	cmd.Flags().StringVarP(&generateFlags.siteName, "site", "s", "", "Site to parse. If not set parse all sites.")
+	cmd.Flags().StringVarP(&generateFlags.siteName, "site", "s", "", "DeploymentSite to parse. If not set parse all sites.")
 	cmd.Flags().BoolVarP(&generateFlags.ignoreVersion, "ignore-version", "", false, "Skip MACH composer version check")
 	cmd.Flags().StringVarP(&generateFlags.outputPath, "output-path", "", "", "Output path, defaults to `cwd`/deployments.")
 
 	handleError(cmd.MarkFlagFilename("var-file", "yml", "yaml"))
 	handleError(cmd.MarkFlagFilename("file", "yml", "yaml"))
 
-	cmd.RegisterFlagCompletionFunc("site", AutocompleteSiteName)
+	_ = cmd.RegisterFlagCompletionFunc("site", AutocompleteSiteName)
 }
 
 func preprocessGenerateFlags() {
@@ -74,14 +74,15 @@ func preprocessGenerateFlags() {
 
 func handleError(err error) {
 	if err != nil {
-		cli.PrintExitError("An error occured:", err.Error())
+		cli.PrintExitError("An error occurred:", err.Error())
 	}
 }
 
 // loadConfig parses and validates the given config file path.
 func loadConfig(cmd *cobra.Command, resolveVars bool) *config.MachConfig {
 	opts := &config.ConfigOptions{
 		NoResolveVars: !resolveVars,
+		Validate:      true,
 	}
 	if generateFlags.varFile != "" {
 		opts.VarFilenames = []string{generateFlags.varFile}

cmd/mach-composer/generate.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"github.com/mach-composer/mach-composer-cli/internal/dependency"
 	"github.com/spf13/cobra"
 
 	"github.com/mach-composer/mach-composer-cli/internal/generator"
@@ -13,15 +14,15 @@ var generateCmd = &cobra.Command{
 		preprocessGenerateFlags()
 	},
 	RunE: func(cmd *cobra.Command, args []string) error {
-		return generateFunc(cmd, args)
+		return generateFunc(cmd)
 	},
 }
 
 func init() {
 	registerGenerateFlags(generateCmd)
 }
 
-func generateFunc(cmd *cobra.Command, args []string) error {
+func generateFunc(cmd *cobra.Command) error {
 	cfg := loadConfig(cmd, true)
 	defer cfg.Close()
 
@@ -32,7 +33,12 @@ func generateFunc(cmd *cobra.Command, args []string) error {
 		Site:       generateFlags.siteName,
 	}
 
-	_, err := generator.WriteFiles(cmd.Context(), cfg, genOptions)
+	gd, err := dependency.ToDeploymentGraph(cfg)
+	if err != nil {
+		return err
+	}
+
+	err = generator.Write(cmd.Context(), cfg, gd, genOptions)
 	if err != nil {
 		return err
 	}

cmd/mach-composer/graph.go

@@ -2,24 +2,22 @@ package main
 
 import (
 	"bytes"
-	"fmt"
 	"github.com/dominikbraun/graph/draw"
 	"github.com/mach-composer/mach-composer-cli/internal/dependency"
+	"github.com/rs/zerolog/log"
 	"github.com/spf13/cobra"
+
+	"github.com/goccy/go-graphviz"
 )
 
+var graphFlags struct {
+	output           string
+	outputDeployment string
+}
+
 var graphCmd = &cobra.Command{
 	Use:   "graph",
 	Short: "Print the execution graph for this project",
-	Long: `
-Print the execution graph for this project. Note that the output will be in the DOT Language (https://graphviz.org/about/).
-
-This output can be used to generate actual files with the dependency graph. 
-A tool like graphviz can be used to make this transformation:
-  
-  'mach-composer graph -f main.yml | dot -Tpng -o image.png'
-	
-	`,
 	PreRun: func(cmd *cobra.Command, args []string) {
 		preprocessGenerateFlags()
 	},
@@ -30,24 +28,57 @@ A tool like graphviz can be used to make this transformation:
 
 func init() {
 	registerGenerateFlags(graphCmd)
+	graphCmd.Flags().StringVarP(&graphFlags.output, "output", "", "./graph.png", "output file for the deployment image")
+	graphCmd.Flags().StringVarP(&graphFlags.outputDeployment, "output-deployment", "", "./deployment-graph.png",
+		"output file for the deployment image")
 }
 
+// TODO: turn both into single graph, where nested site components are also visible within the site terraform as a whole.
+//
+//	This should more clearly show the dependencies between the different sites and components.
 func graphFunc(cmd *cobra.Command, _ []string) error {
 	cfg := loadConfig(cmd, true)
 	defer cfg.Close()
 
-	t, _, err := dependency.FromConfig(cfg)
+	g, err := dependency.ToDependencyGraph(cfg)
 	if err != nil {
 		return err
 	}
 
 	var buff bytes.Buffer
-	err = draw.DOT(t, &buff, draw.GraphAttribute("label", cfg.Filename))
+	err = draw.DOT(g.NodeGraph, &buff)
+	if err != nil {
+		return err
+	}
+
+	gv := graphviz.New()
+	ggv, _ := graphviz.ParseBytes(buff.Bytes())
+
+	if err := gv.RenderFilename(ggv, graphviz.PNG, graphFlags.output); err != nil {
+		log.Fatal().Err(err)
+	}
+
+	log.Info().Msgf("Graph written to %s", graphFlags.output)
+
+	dg, err := dependency.ToDeploymentGraph(cfg)
 	if err != nil {
 		return err
 	}
 
-	fmt.Println(buff.String())
+	var buff2 bytes.Buffer
+	err = draw.DOT(dg.NodeGraph, &buff2)
+	if err != nil {
+		return err
+	}
+
+	gv2 := graphviz.New()
+	ggv2, _ := graphviz.ParseBytes(buff2.Bytes())
+
+	if err := gv2.RenderFilename(ggv2, graphviz.PNG, graphFlags.outputDeployment); err != nil {
+		log.Fatal().Err(err)
+	}
+
+	log.Info().Msgf("Deployment graph written to %s", graphFlags.outputDeployment)
 
 	return nil
 }

cmd/mach-composer/init.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"github.com/mach-composer/mach-composer-cli/internal/dependency"
 	"github.com/spf13/cobra"
 
 	"github.com/mach-composer/mach-composer-cli/internal/cli"
@@ -34,15 +35,20 @@ func initFunc(cmd *cobra.Command, args []string) error {
 
 	generateFlags.ValidateSite(cfg)
 
-	paths, err := generator.WriteFiles(ctx, cfg, &generator.GenerateOptions{
+	dg, err := dependency.ToDeploymentGraph(cfg)
+	if err != nil {
+		return err
+	}
+
+	err = generator.Write(ctx, cfg, dg, &generator.GenerateOptions{
 		OutputPath: generateFlags.outputPath,
 		Site:       generateFlags.siteName,
 	})
 	if err != nil {
 		return err
 	}
 
-	return runner.TerraformInit(ctx, cfg, paths, &runner.InitOptions{
+	return runner.TerraformInit(ctx, cfg, dg, &runner.InitOptions{
 		Site: generateFlags.siteName,
 	})
 }

cmd/mach-composer/plan.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"github.com/mach-composer/mach-composer-cli/internal/dependency"
 	"github.com/spf13/cobra"
 
 	"github.com/mach-composer/mach-composer-cli/internal/generator"
@@ -37,15 +38,20 @@ func planFunc(cmd *cobra.Command, args []string) error {
 
 	generateFlags.ValidateSite(cfg)
 
-	paths, err := generator.WriteFiles(ctx, cfg, &generator.GenerateOptions{
+	dg, err := dependency.ToDeploymentGraph(cfg)
+	if err != nil {
+		return err
+	}
+
+	err = generator.Write(ctx, cfg, dg, &generator.GenerateOptions{
 		OutputPath: generateFlags.outputPath,
 		Site:       generateFlags.siteName,
 	})
 	if err != nil {
 		return err
 	}
 
-	return runner.TerraformPlan(ctx, cfg, paths, &runner.PlanOptions{
+	return runner.TerraformPlan(ctx, cfg, dg, &runner.PlanOptions{
 		Reuse: planFlags.reuse,
 		Site:  generateFlags.siteName,
 	})

cmd/mach-composer/show-plan.go

@@ -1,9 +1,9 @@
 package main
 
 import (
+	"github.com/mach-composer/mach-composer-cli/internal/dependency"
 	"github.com/spf13/cobra"
 
-	"github.com/mach-composer/mach-composer-cli/internal/generator"
 	"github.com/mach-composer/mach-composer-cli/internal/runner"
 )
 
@@ -35,12 +35,12 @@ func showPlanFunc(cmd *cobra.Command, args []string) error {
 
 	generateFlags.ValidateSite(cfg)
 
-	paths := generator.FileLocations(cfg, &generator.GenerateOptions{
-		OutputPath: generateFlags.outputPath,
-		Site:       generateFlags.siteName,
-	})
+	dg, err := dependency.ToDeploymentGraph(cfg)
+	if err != nil {
+		return err
+	}
 
-	return runner.TerraformShow(ctx, cfg, paths, &runner.ShowPlanOptions{
+	return runner.TerraformShow(ctx, cfg, dg, &runner.ShowPlanOptions{
 		NoColor: showPlanFlags.noColor,
 		Site:    generateFlags.siteName,
 	})

cmd/mach-composer/terraform.go

@@ -1,9 +1,9 @@
 package main
 
 import (
+	"github.com/mach-composer/mach-composer-cli/internal/dependency"
 	"github.com/spf13/cobra"
 
-	"github.com/mach-composer/mach-composer-cli/internal/generator"
 	"github.com/mach-composer/mach-composer-cli/internal/runner"
 )
 
@@ -28,12 +28,12 @@ func terraformFunc(cmd *cobra.Command, args []string) error {
 
 	generateFlags.ValidateSite(cfg)
 
-	paths := generator.FileLocations(cfg, &generator.GenerateOptions{
-		OutputPath: generateFlags.outputPath,
-		Site:       generateFlags.siteName,
-	})
+	dg, err := dependency.ToDeploymentGraph(cfg)
+	if err != nil {
+		return err
+	}
 
-	return runner.TerraformProxy(cmd.Context(), cfg, paths, &runner.ProxyOptions{
+	return runner.TerraformProxy(cmd.Context(), dg, &runner.ProxyOptions{
 		Site:    generateFlags.siteName,
 		Command: args,
 	})