Add support for deletions (fixes #28)

cmd/formatter.go

@@ -60,6 +60,7 @@ func (f *Formatter) legacyJSON(diff digest.Differences) error {
 	type jsonDifference struct {
 		Additions     []string
 		Modifications []string
+		Deletions     []string
 	}
 
 	includes := config.GetIncludeColumnPositions()
@@ -74,7 +75,12 @@ func (f *Formatter) legacyJSON(diff digest.Differences) error {
 		modifications = append(modifications, includes.String(modification.Current))
 	}
 
-	jsonDiff := jsonDifference{Additions: additions, Modifications: modifications}
+	deletions := make([]string, 0, len(diff.Deletions))
+	for _, deletion := range diff.Deletions {
+		deletions = append(deletions, includes.String(deletion))
+	}
+
+	jsonDiff := jsonDifference{Additions: additions, Modifications: modifications, Deletions: deletions}
 	data, err := json.MarshalIndent(jsonDiff, "", "  ")
 
 	if err != nil {
@@ -100,6 +106,11 @@ func (f *Formatter) json(diff digest.Differences) error {
 		additions = append(additions, includes.String(addition))
 	}
 
+	deletions := make([]string, 0, len(diff.Deletions))
+	for _, deletion := range diff.Deletions {
+		deletions = append(deletions, includes.String(deletion))
+	}
+
 	type modification struct {
 		Original string
 		Current  string
@@ -108,6 +119,7 @@ func (f *Formatter) json(diff digest.Differences) error {
 	type jsonDifference struct {
 		Additions     []string
 		Modifications []modification
+		Deletions     []string
 	}
 
 	modifications := make([]modification, 0, len(diff.Modifications))
@@ -116,7 +128,7 @@ func (f *Formatter) json(diff digest.Differences) error {
 		modifications = append(modifications, m)
 	}
 
-	data, err := json.MarshalIndent(jsonDifference{Additions: additions, Modifications: modifications}, "", "  ")
+	data, err := json.MarshalIndent(jsonDifference{Additions: additions, Modifications: modifications, Deletions: deletions}, "", "  ")
 
 	if err != nil {
 		return fmt.Errorf("error when serializing with JSON formatter: %v", err)
@@ -136,6 +148,7 @@ func (f *Formatter) json(diff digest.Differences) error {
 func (f *Formatter) rowMark(diff digest.Differences) error {
 	_, _ = fmt.Fprintf(f.stderr, "Additions %d\n", len(diff.Additions))
 	_, _ = fmt.Fprintf(f.stderr, "Modifications %d\n", len(diff.Modifications))
+	_, _ = fmt.Fprintf(f.stderr, "Deletions %d\n", len(diff.Deletions))
 	_, _ = fmt.Fprintf(f.stderr, "Rows:\n")
 
 	includes := config.GetIncludeColumnPositions()
@@ -150,6 +163,11 @@ func (f *Formatter) rowMark(diff digest.Differences) error {
 		modifications = append(modifications, includes.String(modification.Current))
 	}
 
+	deletions := make([]string, 0, len(diff.Deletions))
+	for _, deletion := range diff.Deletions {
+		deletions = append(deletions, includes.String(deletion))
+	}
+
 	for _, added := range additions {
 		_, _ = fmt.Fprintf(f.stdout, "%s,%s\n", added, "ADDED")
 	}
@@ -158,6 +176,10 @@ func (f *Formatter) rowMark(diff digest.Differences) error {
 		_, _ = fmt.Fprintf(f.stdout, "%s,%s\n", modified, "MODIFIED")
 	}
 
+	for _, deleted := range deletions {
+		_, _ = fmt.Fprintf(f.stdout, "%s,%s\n", deleted, "DELETED")
+	}
+
 	return nil
 }
 
@@ -178,6 +200,10 @@ func (f *Formatter) lineDiff(diff digest.Differences) error {
 		red(f.stdout, "- %s\n", includes.String(modification.Original))
 		green(f.stdout, "+ %s\n", includes.String(modification.Current))
 	}
+	blue(f.stderr, "# Deletions (%d)\n", len(diff.Deletions))
+	for _, deletion := range diff.Deletions {
+		red(f.stdout, "- %s\n", includes.String(deletion))
+	}
 
 	return nil
 }
@@ -221,6 +247,11 @@ func (f *Formatter) wordLevelDiffs(diff digest.Differences, deletionFormat, addi
 		_, _ = fmt.Fprintln(f.stdout, digest.Positions{}.String(result))
 	}
 
+	_, _ = fmt.Fprintln(f.stderr, blue("# Deletions (%d)", len(diff.Deletions)))
+	for _, deletion := range diff.Deletions {
+		_, _ = fmt.Fprintln(f.stdout, red(deletionFormat, includes.String(deletion)))
+	}
+
 	return nil
 
 }

cmd/formatter_test.go

@@ -13,14 +13,18 @@ import (
 func TestLegacyJSONFormat(t *testing.T) {
 	diff := digest.Differences{
 		Additions:     []digest.Addition{[]string{"additions"}},
-		Modifications: []digest.Modification{digest.Modification{Current: []string{"modification"}}},
+		Modifications: []digest.Modification{{Current: []string{"modification"}}},
+		Deletions:     []digest.Deletion{[]string{"deletions"}},
 	}
 	expected := `{
   "Additions": [
     "additions"
   ],
   "Modifications": [
     "modification"
+  ],
+  "Deletions": [
+    "deletions"
   ]
 }`
 
@@ -37,7 +41,8 @@ func TestLegacyJSONFormat(t *testing.T) {
 func TestJSONFormat(t *testing.T) {
 	diff := digest.Differences{
 		Additions:     []digest.Addition{[]string{"additions"}},
-		Modifications: []digest.Modification{digest.Modification{Original: []string{"original"}, Current: []string{"modification"}}},
+		Modifications: []digest.Modification{{Original: []string{"original"}, Current: []string{"modification"}}},
+		Deletions:     []digest.Deletion{[]string{"deletions"}},
 	}
 	expected := `{
   "Additions": [
@@ -48,6 +53,9 @@ func TestJSONFormat(t *testing.T) {
       "Original": "original",
       "Current": "modification"
     }
+  ],
+  "Deletions": [
+    "deletions"
   ]
 }`
 
@@ -63,13 +71,16 @@ func TestJSONFormat(t *testing.T) {
 func TestRowMarkFormatter(t *testing.T) {
 	diff := digest.Differences{
 		Additions:     []digest.Addition{[]string{"additions"}},
-		Modifications: []digest.Modification{digest.Modification{Current: []string{"modification"}}},
+		Modifications: []digest.Modification{{Current: []string{"modification"}}},
+		Deletions:     []digest.Deletion{[]string{"deletions"}},
 	}
 	expectedStdout := `additions,ADDED
 modification,MODIFIED
+deletions,DELETED
 `
 	expectedStderr := `Additions 1
 Modifications 1
+Deletions 1
 Rows:
 `
 
@@ -89,18 +100,21 @@ func TestLineDiff(t *testing.T) {
 	diff := digest.Differences{
 		Additions: []digest.Addition{[]string{"additions"}},
 		Modifications: []digest.Modification{
-			digest.Modification{
+			{
 				Original: []string{"original", "comma,separated,value"},
 				Current:  []string{"modification", "comma,separated,value-2"},
 			},
 		},
+		Deletions: []digest.Deletion{{"deletion", "this-row-was-deleted"}},
 	}
 	expectedStdout := `+ additions
 - original,"comma,separated,value"
 + modification,"comma,separated,value-2"
+- deletion,this-row-was-deleted
 `
 	expectedStderr := `# Additions (1)
 # Modifications (1)
+# Deletions (1)
 `
 
 	var stdout bytes.Buffer
@@ -118,13 +132,16 @@ func TestLineDiff(t *testing.T) {
 func TestWordDiff(t *testing.T) {
 	diff := digest.Differences{
 		Additions:     []digest.Addition{[]string{"additions"}},
-		Modifications: []digest.Modification{digest.Modification{Original: []string{"original"}, Current: []string{"modification"}}},
+		Modifications: []digest.Modification{{Original: []string{"original"}, Current: []string{"modification"}}},
+		Deletions:     []digest.Deletion{{"deletions"}},
 	}
 	expectedStdout := `{+additions+}
 [-original-]{+modification+}
+[-deletions-]
 `
 	expectedStderr := `# Additions (1)
 # Modifications (1)
+# Deletions (1)
 `
 
 	var stdout bytes.Buffer
@@ -142,13 +159,16 @@ func TestWordDiff(t *testing.T) {
 func TestColorWords(t *testing.T) {
 	diff := digest.Differences{
 		Additions:     []digest.Addition{[]string{"additions"}},
-		Modifications: []digest.Modification{digest.Modification{Original: []string{"original"}, Current: []string{"modification"}}},
+		Modifications: []digest.Modification{{Original: []string{"original"}, Current: []string{"modification"}}},
+		Deletions:     []digest.Deletion{{"deletions"}},
 	}
 	expectedStdout := `additions
 originalmodification
+deletions
 `
 	expectedStderr := `# Additions (1)
 # Modifications (1)
+# Deletions (1)
 `
 
 	var stdout bytes.Buffer
@@ -162,6 +182,7 @@ originalmodification
 	assert.Equal(t, expectedStdout, stdout.String())
 	assert.Equal(t, expectedStderr, stderr.String())
 }
+
 func TestWrongFormatter(t *testing.T) {
 	diff := digest.Differences{}
 	formatter := cmd.NewFormatter(nil, nil, cmd.Config{Format: "random-str"})

cmd/root.go

@@ -32,7 +32,7 @@ import (
 )
 
 var (
-	timed   bool
+	timed bool
 )
 
 // rootCmd represents the base command when called without any subcommands
@@ -77,14 +77,12 @@ Most suitable for csv files created from database tables`,
 			config.GetPrimaryKeys(),
 			config.GetValueColumns(),
 			config.GetIncludeColumnPositions(),
-			true,
 		)
 		deltaConfig := digest.NewConfig(
 			deltaFile,
 			config.GetPrimaryKeys(),
 			config.GetValueColumns(),
 			config.GetIncludeColumnPositions(),
-			false,
 		)
 
 		diff, err := digest.Diff(*baseConfig, *deltaConfig)
@@ -124,7 +122,7 @@ func isValidFile(path string) error {
 func Execute() {
 	rootCmd.Version = Version()
 	if err := rootCmd.Execute(); err != nil {
-		fmt.Fprintf(os.Stderr, "%v", err)
+		_, _ = fmt.Fprintf(os.Stderr, "%v\n", err)
 		os.Exit(1)
 	}
 }

go.mod

@@ -10,3 +10,5 @@ require (
 	github.com/stretchr/testify v1.3.0
 	golang.org/x/sys v0.0.0-20190804053845-51ab0e2deafa // indirect
 )
+
+go 1.13

pkg/digest/config.go

@@ -10,11 +10,10 @@ import "io"
 // Include: Include these positions in output. It is Value positions by default.
 // KeepSource: return the source and target string if diff is computed
 type Config struct {
-	Key        Positions
-	Value      Positions
-	Include    Positions
-	Reader     io.Reader
-	KeepSource bool
+	Key     Positions
+	Value   Positions
+	Include Positions
+	Reader  io.Reader
 }
 
 // NewConfig creates an instance of Config struct.
@@ -23,17 +22,15 @@ func NewConfig(
 	primaryKey Positions,
 	valueColumns Positions,
 	includeColumns Positions,
-	keepSource bool,
 ) *Config {
 	if len(includeColumns) == 0 {
 		includeColumns = valueColumns
 	}
 
 	return &Config{
-		Reader:     r,
-		Key:        primaryKey,
-		Value:      valueColumns,
-		Include:    includeColumns,
-		KeepSource: keepSource,
+		Reader:  r,
+		Key:     primaryKey,
+		Value:   valueColumns,
+		Include: includeColumns,
 	}
 }

pkg/digest/diff.go

@@ -10,18 +10,23 @@ type messageType int
 const (
 	addition     messageType = iota
 	modification messageType = iota
+	deletion     messageType = iota
 )
 
 // Differences represents the differences
 // between 2 csv content
 type Differences struct {
 	Additions     []Addition
 	Modifications []Modification
+	Deletions     []Deletion
 }
 
 // Addition is a row appearing in delta but missing in base
 type Addition []string
 
+// Deletion is a row appearing in base but missing in delta
+type Deletion []string
+
 // Modification is a row present in both delta and base
 // with the values column changed in delta
 type Modification struct {
@@ -51,12 +56,12 @@ func Diff(baseConfig, deltaConfig Config) (Differences, error) {
 		return Differences{}, fmt.Errorf("error processing base file: %v", err)
 	}
 
-	deltaConfig.KeepSource = true
 	deltaEngine := NewEngine(deltaConfig)
 	deltaDigestChannel, deltaErrorChannel := deltaEngine.StreamDigests()
 
 	additions := make([]Addition, 0)
 	modifications := make([]Modification, 0)
+	deletions := make([]Deletion, 0)
 
 	msgChannel := streamDifferences(baseFileDigest, deltaDigestChannel)
 	for msg := range msgChannel {
@@ -65,14 +70,18 @@ func Diff(baseConfig, deltaConfig Config) (Differences, error) {
 			additions = append(additions, msg.current)
 		case modification:
 			modifications = append(modifications, Modification{Original: msg.original, Current: msg.current})
+		case deletion:
+			deletions = append(deletions, msg.current)
+		default:
+			continue
 		}
 	}
 
 	if err := <-deltaErrorChannel; err != nil {
 		return Differences{}, fmt.Errorf("error processing delta file: %v", err)
 	}
 
-	return Differences{Additions: additions, Modifications: modifications}, nil
+	return Differences{Additions: additions, Modifications: modifications, Deletions: deletions}, nil
 }
 
 func streamDifferences(baseFileDigest *FileDigest, digestChannel chan []Digest) chan message {
@@ -89,13 +98,19 @@ func streamDifferences(baseFileDigest *FileDigest, digestChannel chan []Digest)
 						// Modification
 						msgChannel <- message{_type: modification, current: d.Source, original: base.SourceMap[d.Key]}
 					}
+					// delete from sourceMap so that at the end only deletions are left in base
+					delete(base.SourceMap, d.Key)
 				} else {
 					// Addition
 					msgChannel <- message{_type: addition, current: d.Source}
 				}
 			}
 		}
 
+		for _, value := range base.SourceMap {
+			msgChannel <- message{_type: deletion, current: value}
+		}
+
 	}(baseFileDigest, digestChannel, msgChannel)
 
 	return msgChannel