Add the --exit-on-error flag

This changes the behavior of kubeval such that it no longer halts upon
encountering the first error. Instead, it will log the error, then
continue to process other files/documents.

The previous behavior can be attained by passing the --exit-on-error
flag.

fixtures/multi_invalid_resources.yaml

@@ -0,0 +1,9 @@
+kind:
+---
+kind:
+---
+kind:
+---
+kind:
+---
+kind:

kubeval/kubeval.go

@@ -42,6 +42,10 @@ var Strict bool
 // for resource definitions without an available schema
 var IgnoreMissingSchemas bool
 
+// ExitOnError tells kubeval whether to halt processing upon the
+// first error encountered or to continue, aggregating all errors
+var ExitOnError bool
+
 // ValidFormat is a type for quickly forcing
 // new formats on the gojsonschema loader
 type ValidFormat struct{}
@@ -258,10 +262,13 @@ func ValidateWithCache(config []byte, fileName string, schemaCache map[string]*g
 	for _, element := range bits {
 		if len(element) > 0 {
 			result, err := validateResource(element, fileName, schemaCache)
-			results = append(results, result)
 			if err != nil {
 				errors = multierror.Append(errors, err)
+				if ExitOnError {
+					return results, errors
+				}
 			}
+			results = append(results, result)
 		} else {
 			result := ValidationResult{}
 			result.FileName = fileName

kubeval/kubeval_test.go

@@ -5,6 +5,7 @@ import (
 	"path/filepath"
 	"testing"
 
+	multierror "github.com/hashicorp/go-multierror"
 	"github.com/xeipuuv/gojsonschema"
 )
 
@@ -89,7 +90,6 @@ func TestStrictCatchesAdditionalErrors(t *testing.T) {
 	}
 }
 
-
 func TestValidateMultipleVersions(t *testing.T) {
 	Strict = true
 	Version = "1.14.0"
@@ -118,6 +118,36 @@ func TestValidateInputsWithErrors(t *testing.T) {
 	}
 }
 
+func TestValidateMultipleResourcesWithErrors(t *testing.T) {
+	var tests = []string{
+		"multi_invalid_resources.yaml",
+	}
+	for _, test := range tests {
+		filePath, _ := filepath.Abs("../fixtures/" + test)
+		fileContents, _ := ioutil.ReadFile(filePath)
+		ExitOnError = true
+		_, err := Validate(fileContents, test)
+		if err == nil {
+			t.Errorf("Validate should not pass when testing invalid configuration in " + test)
+		} else if merr, ok := err.(*multierror.Error); ok {
+			if len(merr.Errors) != 1 {
+				t.Errorf("Validate should encounter exactly 1 error when testing invalid configuration in " + test + " with ExitOnError=true")
+			}
+		}
+		ExitOnError = false
+		_, err = Validate(fileContents, test)
+		if err == nil {
+			t.Errorf("Validate should not pass when testing invalid configuration in " + test)
+		} else if merr, ok := err.(*multierror.Error); ok {
+			if len(merr.Errors) != 5 {
+				t.Errorf("Validate should encounter exactly 5 errors when testing invalid configuration in " + test)
+			}
+		} else if !ok {
+			t.Errorf("Validate should encounter exactly 5 errors when testing invalid configuration in " + test)
+		}
+	}
+}
+
 func TestDetermineSchema(t *testing.T) {
 	Strict = false
 	schema := determineSchema("sample", "v1")
@@ -171,4 +201,3 @@ func TestSkipCrdSchemaMiss(t *testing.T) {
 		t.Errorf("For custom CRD's with schema missing we should skip with SkipCrdSchemaMiss flag")
 	}
 }
-

main.go

@@ -69,12 +69,16 @@ var RootCmd = &cobra.Command{
 				fileContents, err := ioutil.ReadFile(filePath)
 				if err != nil {
 					log.Error("Could not open file", fileName)
-					os.Exit(1)
+					earlyExit()
+					success = false
+					continue
 				}
 				results, err := kubeval.ValidateWithCache(fileContents, fileName, schemaCache)
 				if err != nil {
 					log.Error(err)
-					os.Exit(1)
+					earlyExit()
+					success = false
+					continue
 				}
 				success = logResults(results, success)
 			}
@@ -102,6 +106,12 @@ func logResults(results []kubeval.ValidationResult, success bool) bool {
 	return success
 }
 
+func earlyExit() {
+	if kubeval.ExitOnError {
+		os.Exit(1)
+	}
+}
+
 // Execute adds all child commands to the root command sets flags appropriately.
 // This is called by main.main(). It only needs to happen once to the rootCmd.
 func Execute() {
@@ -119,6 +129,7 @@ func init() {
 	RootCmd.Flags().BoolVarP(&kubeval.OpenShift, "openshift", "", false, "Use OpenShift schemas instead of upstream Kubernetes")
 	RootCmd.Flags().BoolVarP(&kubeval.Strict, "strict", "", false, "Disallow additional properties not in schema")
 	RootCmd.Flags().BoolVarP(&kubeval.IgnoreMissingSchemas, "ignore-missing-schemas", "", false, "Skip validation for resource definitions without a schema")
+	RootCmd.Flags().BoolVarP(&kubeval.ExitOnError, "exit-on-error", "", false, "Immediately stop execution when the first error is encountered")
 	RootCmd.SetVersionTemplate(`{{.Version}}`)
 	viper.BindPFlag("schema_location", RootCmd.Flags().Lookup("schema-location"))
 	RootCmd.PersistentFlags().StringP("filename", "f", "stdin", "filename to be displayed when testing manifests read from stdin")