Terragrunt performance improvements (#3307)

* Performance testing

* Usage of common cache

* Added partial parse cache

* HCL cache fetching

* Add cache deep copy

* Fixed run_cmd cache

* Run cmd cache

* Add context cache

* Add test for git cache

* Mod update

* Add tracking of cache hit/miss

* Updated parse key

* Dependency outputs

* Add depednency caching

* Dependency cleanup

* Cleanup

* Deps update

* Code simplification

* Performance improvements for dependency fetching

* Cleanup

* Updated cache names

cli/app.go

@@ -127,9 +127,11 @@ func (app *App) RunContext(ctx context.Context, args []string) error {
 		}
 	}(ctx)
 
+	ctx = config.WithConfigValues(ctx)
+
 	// init engine if required
 	if engine.IsEngineEnabled() {
-		ctx = engine.ContextWithEngine(ctx)
+		ctx = engine.WithEngineValues(ctx)
 	}
 	defer func(ctx context.Context) {
 		if err := engine.Shutdown(ctx); err != nil {

cli/commands/terraform/creds/providers/amazonsts/provider.go

@@ -35,7 +35,7 @@ func (provider *Provider) GetCredentials(ctx context.Context) (*providers.Creden
 		return nil, nil
 	}
 
-	if cached, hit := credentialsCache.Get(iamRoleOpts.RoleARN); hit {
+	if cached, hit := credentialsCache.Get(ctx, iamRoleOpts.RoleARN); hit {
 		provider.terragruntOptions.Logger.Debugf("Using cached credentials for IAM role %s.", iamRoleOpts.RoleARN)
 		return cached, nil
 	}
@@ -56,10 +56,10 @@ func (provider *Provider) GetCredentials(ctx context.Context) (*providers.Creden
 		},
 	}
 
-	credentialsCache.Put(iamRoleOpts.RoleARN, creds, time.Now().Add(time.Duration(iamRoleOpts.AssumeRoleDuration)*time.Second))
+	credentialsCache.Put(ctx, iamRoleOpts.RoleARN, creds, time.Now().Add(time.Duration(iamRoleOpts.AssumeRoleDuration)*time.Second))
 
 	return creds, nil
 }
 
 // credentialsCache is a cache of credentials.
-var credentialsCache = cache.NewExpiringCache[*providers.Credentials]()
+var credentialsCache = cache.NewExpiringCache[*providers.Credentials]("credentialsCache")

cli/commands/terraform/version_check.go

@@ -34,7 +34,7 @@ const versionParts = 3
 // - TerraformVersion
 // TODO: Look into a way to refactor this function to avoid the side effect.
 func checkVersionConstraints(ctx context.Context, terragruntOptions *options.TerragruntOptions) error {
-	configContext := config.NewParsingContext(context.Background(), terragruntOptions).WithDecodeList(config.TerragruntVersionConstraints)
+	configContext := config.NewParsingContext(ctx, terragruntOptions).WithDecodeList(config.TerragruntVersionConstraints)
 
 	partialTerragruntConfig, err := config.PartialParseConfigFile(
 		configContext,

config/cache_test.go

@@ -1,16 +1,19 @@
 package config
 
 import (
+	"context"
 	"testing"
 
 	"github.com/gruntwork-io/terragrunt/internal/cache"
 	"github.com/stretchr/testify/assert"
 )
 
+const testCacheName = "TerragruntConfig"
+
 func TestTerragruntConfigCacheCreation(t *testing.T) {
 	t.Parallel()
 
-	cache := cache.NewCache[TerragruntConfig]()
+	cache := cache.NewCache[TerragruntConfig](testCacheName)
 
 	assert.NotNil(t, cache.Mutex)
 	assert.NotNil(t, cache.Cache)
@@ -23,9 +26,10 @@ func TestTerragruntConfigCacheOperation(t *testing.T) {
 
 	testCacheKey := "super-safe-cache-key"
 
-	cache := cache.NewCache[TerragruntConfig]()
+	ctx := context.Background()
+	cache := cache.NewCache[TerragruntConfig](testCacheName)
 
-	actualResult, found := cache.Get(testCacheKey)
+	actualResult, found := cache.Get(ctx, testCacheKey)
 
 	assert.False(t, found)
 	assert.Empty(t, actualResult)
@@ -34,8 +38,8 @@ func TestTerragruntConfigCacheOperation(t *testing.T) {
 		IsPartial: true, // Any random property will be sufficient
 	}
 
-	cache.Put(testCacheKey, stubTerragruntConfig)
-	actualResult, found = cache.Get(testCacheKey)
+	cache.Put(ctx, testCacheKey, stubTerragruntConfig)
+	actualResult, found = cache.Get(ctx, testCacheKey)
 
 	assert.True(t, found)
 	assert.NotEmpty(t, actualResult)

config/config.go

@@ -33,13 +33,11 @@ import (
 const (
 	DefaultTerragruntConfigPath     = "terragrunt.hcl"
 	DefaultTerragruntJsonConfigPath = "terragrunt.hcl.json"
+	FoundInFile                     = "found_in_file"
 
-	DefaultEngineType = "rpc"
-)
-
-const FoundInFile = "found_in_file"
+	iamRoleCacheName = "iamRoleCache"
 
-const (
+	DefaultEngineType                   = "rpc"
 	MetadataTerraform                   = "terraform"
 	MetadataTerraformBinary             = "terraform_binary"
 	MetadataTerraformVersionConstraint  = "terraform_version_constraint"
@@ -696,12 +694,11 @@ func ReadTerragruntConfig(ctx context.Context, terragruntOptions *options.Terrag
 	return ParseConfigFile(parcingCtx, terragruntOptions.TerragruntConfigPath, nil)
 }
 
-var hclCache = cache.NewCache[*hclparse.File]()
-
 // Parse the Terragrunt config file at the given path. If the include parameter is not nil, then treat this as a config
 // included in some other config file when resolving relative paths.
 func ParseConfigFile(ctx *ParsingContext, configPath string, includeFromChild *IncludeConfig) (*TerragruntConfig, error) {
 	var config *TerragruntConfig
+	hclCache := cache.ContextCache[*hclparse.File](ctx, HclCacheContextKey)
 	err := telemetry.Telemetry(ctx, ctx.TerragruntOptions, "parse_config_file", map[string]interface{}{
 		"config_path": configPath,
 		"working_dir": ctx.TerragruntOptions.WorkingDir,
@@ -725,15 +722,15 @@ func ParseConfigFile(ctx *ParsingContext, configPath string, includeFromChild *I
 		}
 		var file *hclparse.File
 		var cacheKey = fmt.Sprintf("parse-config-%v-%v-%v-%v-%v-%v", configPath, childKey, decodeListKey, ctx.TerragruntOptions.WorkingDir, dir, fileInfo.ModTime().UnixMicro())
-		if cacheConfig, found := hclCache.Get(cacheKey); found {
+		if cacheConfig, found := hclCache.Get(ctx, cacheKey); found {
 			file = cacheConfig
 		} else {
 			// Parse the HCL file into an AST body that can be decoded multiple times later without having to re-parse
 			file, err = hclparse.NewParser().WithOptions(ctx.ParserOptions...).ParseFromFile(configPath)
 			if err != nil {
 				return err
 			}
-			hclCache.Put(cacheKey, file)
+			hclCache.Put(ctx, cacheKey, file)
 		}
 		config, err = ParseConfig(ctx, file, includeFromChild)
 		if err != nil {
@@ -856,7 +853,7 @@ func ParseConfig(ctx *ParsingContext, file *hclparse.File, includeFromChild *Inc
 }
 
 // iamRoleCache - store for cached values of IAM roles
-var iamRoleCache = cache.NewCache[options.IAMRoleOptions]()
+var iamRoleCache = cache.NewCache[options.IAMRoleOptions](iamRoleCacheName)
 
 // setIAMRole - extract IAM role details from Terragrunt flags block
 func setIAMRole(ctx *ParsingContext, file *hclparse.File, includeFromChild *IncludeConfig) error {
@@ -866,14 +863,14 @@ func setIAMRole(ctx *ParsingContext, file *hclparse.File, includeFromChild *Incl
 	} else {
 		// as key is considered HCL code and include configuration
 		var key = fmt.Sprintf("%v-%v", file.Content(), includeFromChild)
-		var config, found = iamRoleCache.Get(key)
+		var config, found = iamRoleCache.Get(ctx, key)
 		if !found {
 			iamConfig, err := TerragruntConfigFromPartialConfig(ctx.WithDecodeList(TerragruntFlags), file, includeFromChild)
 			if err != nil {
 				return err
 			}
 			config = iamConfig.GetIAMRoleOptions()
-			iamRoleCache.Put(key, config)
+			iamRoleCache.Put(ctx, key, config)
 		}
 		// We merge the OriginalIAMRoleOptions into the one from the config, because the CLI passed IAMRoleOptions has
 		// precedence.

config/config_helpers.go

@@ -69,6 +69,8 @@ const (
 	FuncNameEndsWith                                = "endswith"
 	FuncNameStrContains                             = "strcontains"
 	FuncNameTimeCmp                                 = "timecmp"
+
+	sopsCacheName = "sopsCache"
 )
 
 // List of terraform commands that accept -lock-timeout
@@ -308,14 +310,14 @@ func parseGetEnvParameters(parameters []string) (EnvVar, error) {
 	return envVariable, nil
 }
 
-// runCommandCache - cache of evaluated `run_cmd` invocations
-// see: https://github.com/gruntwork-io/terragrunt/issues/1427
-var runCommandCache = cache.NewCache[string]()
-
 // runCommand is a helper function that runs a command and returns the stdout as the interporation
 // for each `run_cmd` in locals section, function is called twice
 // result
 func runCommand(ctx *ParsingContext, args []string) (string, error) {
+	// runCommandCache - cache of evaluated `run_cmd` invocations
+	// see: https://github.com/gruntwork-io/terragrunt/issues/1427
+	runCommandCache := cache.ContextCache[string](ctx, RunCmdCacheContextKey)
+
 	if len(args) == 0 {
 		return "", errors.WithStackTrace(EmptyStringNotAllowedError("parameter to the run_cmd function"))
 	}
@@ -341,7 +343,7 @@ func runCommand(ctx *ParsingContext, args []string) (string, error) {
 	// To avoid re-run of the same run_cmd command, is used in memory cache for command results, with caching key path + arguments
 	// see: https://github.com/gruntwork-io/terragrunt/issues/1427
 	cacheKey := fmt.Sprintf("%v-%v", cachePath, args)
-	cachedValue, foundInCache := runCommandCache.Get(cacheKey)
+	cachedValue, foundInCache := runCommandCache.Get(ctx, cacheKey)
 	if foundInCache {
 		if suppressOutput {
 			ctx.TerragruntOptions.Logger.Debugf("run_cmd, cached output: [REDACTED]")
@@ -366,7 +368,7 @@ func runCommand(ctx *ParsingContext, args []string) (string, error) {
 
 	// Persisting result in cache to avoid future re-evaluation
 	// see: https://github.com/gruntwork-io/terragrunt/issues/1427
-	runCommandCache.Put(cacheKey, value)
+	runCommandCache.Put(ctx, cacheKey, value)
 	return value, nil
 }
 
@@ -728,7 +730,7 @@ func getModulePathFromSourceUrl(sourceUrl string) (string, error) {
 //
 // The cache keys are the canonical paths to the encrypted files, and the values are the
 // plain-text result of the decrypt operation.
-var sopsCache = cache.NewCache[string]()
+var sopsCache = cache.NewCache[string](sopsCacheName)
 
 // decrypts and returns sops encrypted utf-8 yaml or json data as a string
 func sopsDecryptFile(ctx *ParsingContext, params []string) (string, error) {
@@ -751,7 +753,7 @@ func sopsDecryptFile(ctx *ParsingContext, params []string) (string, error) {
 		return "", errors.WithStackTrace(err)
 	}
 
-	if val, ok := sopsCache.Get(canonicalSourceFile); ok {
+	if val, ok := sopsCache.Get(ctx, canonicalSourceFile); ok {
 		return val, nil
 	}
 
@@ -762,7 +764,7 @@ func sopsDecryptFile(ctx *ParsingContext, params []string) (string, error) {
 
 	if utf8.Valid(rawData) {
 		value := string(rawData)
-		sopsCache.Put(canonicalSourceFile, value)
+		sopsCache.Put(ctx, canonicalSourceFile, value)
 		return value, nil
 	}
 

config/config_partial.go

@@ -2,14 +2,18 @@ package config
 
 import (
 	"fmt"
+	"os"
 	"path/filepath"
 
+	clone "github.com/huandu/go-clone"
+
+	"github.com/gruntwork-io/terragrunt/internal/cache"
+
 	"github.com/hashicorp/hcl/v2"
 	"github.com/zclconf/go-cty/cty"
 
 	"github.com/gruntwork-io/go-commons/errors"
 	"github.com/gruntwork-io/terragrunt/config/hclparse"
-	"github.com/gruntwork-io/terragrunt/internal/cache"
 	"github.com/gruntwork-io/terragrunt/util"
 )
 
@@ -136,26 +140,39 @@ func DecodeBaseBlocks(ctx *ParsingContext, file *hclparse.File, includeFromChild
 }
 
 func PartialParseConfigFile(ctx *ParsingContext, configPath string, include *IncludeConfig) (*TerragruntConfig, error) {
-	file, err := hclparse.NewParser().WithOptions(ctx.ParserOptions...).ParseFromFile(configPath)
+	hclCache := cache.ContextCache[*hclparse.File](ctx, HclCacheContextKey)
+
+	fileInfo, err := os.Stat(configPath)
 	if err != nil {
 		return nil, err
 	}
+	var file *hclparse.File
+	var cacheKey = fmt.Sprintf("configPath-%v-modTime-%v", configPath, fileInfo.ModTime().UnixMicro())
+
+	if cacheConfig, found := hclCache.Get(ctx, cacheKey); found {
+		file = cacheConfig
+	} else {
+		file, err = hclparse.NewParser().WithOptions(ctx.ParserOptions...).ParseFromFile(configPath)
+		if err != nil {
+			return nil, err
+		}
+	}
 
 	return TerragruntConfigFromPartialConfig(ctx, file, include)
 }
 
-var terragruntConfigCache = cache.NewCache[TerragruntConfig]()
-
 // Wrapper of PartialParseConfigString which checks for cached configs.
 // filename, configString, includeFromChild and decodeList are used for the cache key,
 // by getting the default value (%#v) through fmt.
 func TerragruntConfigFromPartialConfig(ctx *ParsingContext, file *hclparse.File, includeFromChild *IncludeConfig) (*TerragruntConfig, error) {
 	var cacheKey = fmt.Sprintf("%#v-%#v-%#v-%#v", file.ConfigPath, file.Content(), includeFromChild, ctx.PartialParseDecodeList)
 
+	terragruntConfigCache := cache.ContextCache[*TerragruntConfig](ctx, RunCmdCacheContextKey)
 	if ctx.TerragruntOptions.UsePartialParseConfigCache {
-		if config, found := terragruntConfigCache.Get(cacheKey); found {
+		if config, found := terragruntConfigCache.Get(ctx, cacheKey); found {
 			ctx.TerragruntOptions.Logger.Debugf("Cache hit for '%s' (partial parsing), decodeList: '%v'.", file.ConfigPath, ctx.PartialParseDecodeList)
-			return &config, nil
+			deepCopy := clone.Clone(config).(*TerragruntConfig)
+			return deepCopy, nil
 		}
 
 		ctx.TerragruntOptions.Logger.Debugf("Cache miss for '%s' (partial parsing), decodeList: '%v'.", file.ConfigPath, ctx.PartialParseDecodeList)
@@ -167,7 +184,8 @@ func TerragruntConfigFromPartialConfig(ctx *ParsingContext, file *hclparse.File,
 	}
 
 	if ctx.TerragruntOptions.UsePartialParseConfigCache {
-		terragruntConfigCache.Put(cacheKey, *config)
+		putConfig := clone.Clone(config).(*TerragruntConfig)
+		terragruntConfigCache.Put(ctx, cacheKey, putConfig)
 	}
 
 	return config, nil

config/context.go

@@ -0,0 +1,31 @@
+package config
+
+import (
+	"context"
+
+	"github.com/gruntwork-io/terragrunt/config/hclparse"
+	"github.com/gruntwork-io/terragrunt/internal/cache"
+)
+
+type configKey byte
+
+const (
+	HclCacheContextKey              configKey = iota
+	TerragruntConfigCacheContextKey configKey = iota
+	RunCmdCacheContextKey           configKey = iota
+	DependencyOutputCacheContextKey configKey = iota
+
+	hclCacheName              = "hclCache"
+	configCacheName           = "configCache"
+	runCmdCacheName           = "runCmdCache"
+	dependencyOutputCacheName = "dependencyOutputCache"
+)
+
+// WithConfigValues add to context default values for configuration.
+func WithConfigValues(ctx context.Context) context.Context {
+	ctx = context.WithValue(ctx, HclCacheContextKey, cache.NewCache[*hclparse.File](hclCacheName))
+	ctx = context.WithValue(ctx, TerragruntConfigCacheContextKey, cache.NewCache[*TerragruntConfig](configCacheName))
+	ctx = context.WithValue(ctx, RunCmdCacheContextKey, cache.NewCache[string](runCmdCacheName))
+	ctx = context.WithValue(ctx, DependencyOutputCacheContextKey, cache.NewCache[*dependencyOutputCache](dependencyOutputCacheName))
+	return ctx
+}