fix(tests): make proto/textproto tests not rely on jq (#3967)

unfortunately `jq` is not available in google3, so I changed the tests
back to awk for skipping nondeterministic fields and python's json.tool
for formatting. Because python's json output is different than jq's, all
our golden files had to be updated too.

kythe/cxx/extractor/proto/testdata/custom_corpus.UNIT

@@ -1,20 +1,20 @@
 {
-  "required_input": [
-    {
-      "v_name": {
-        "corpus": "my_custom_corpus",
-        "path": "kythe/cxx/extractor/proto/testdata/simple1.proto"
-      },
-      "info": {
-        "path": "kythe/cxx/extractor/proto/testdata/simple1.proto",
-        "digest": "36aa2295772636e8e940333faa06f0838953101f353ab52d72279389e16117be"
-      }
-    }
-  ],
-  "argument": [
-    "kythe/cxx/extractor/proto/testdata/simple1.proto"
-  ],
-  "source_file": [
-    "kythe/cxx/extractor/proto/testdata/simple1.proto"
-  ]
+    "argument": [
+        "kythe/cxx/extractor/proto/testdata/simple1.proto"
+    ],
+    "required_input": [
+        {
+            "info": {
+                "digest": "36aa2295772636e8e940333faa06f0838953101f353ab52d72279389e16117be",
+                "path": "kythe/cxx/extractor/proto/testdata/simple1.proto"
+            },
+            "v_name": {
+                "corpus": "my_custom_corpus",
+                "path": "kythe/cxx/extractor/proto/testdata/simple1.proto"
+            }
+        }
+    ],
+    "source_file": [
+        "kythe/cxx/extractor/proto/testdata/simple1.proto"
+    ],
 }

kythe/cxx/extractor/proto/testdata/custom_root_directory.UNIT

@@ -1,31 +1,31 @@
 {
-  "required_input": [
-    {
-      "v_name": {
-        "path": "relative_imports.proto"
-      },
-      "info": {
-        "path": "relative_imports.proto",
-        "digest": "c91db27db87695c42ab03479f5393d83fd881c5c4982579b0115a20c36342d2c"
-      }
-    },
-    {
-      "v_name": {
-        "path": "subdir/other.proto"
-      },
-      "info": {
-        "path": "subdir/other.proto",
-        "digest": "c9d3571c48d70781dc7d4984b1b8974f44cb5953c3be17cacacc15f35f5981eb"
-      }
-    }
-  ],
-  "argument": [
-    "kythe/cxx/extractor/proto/testdata/relative_imports.proto",
-    "--",
-    "--proto_path",
-    "kythe/cxx/extractor/proto/testdata/subdir"
-  ],
-  "source_file": [
-    "relative_imports.proto"
-  ]
+    "argument": [
+        "kythe/cxx/extractor/proto/testdata/relative_imports.proto",
+        "--",
+        "--proto_path",
+        "kythe/cxx/extractor/proto/testdata/subdir"
+    ],
+    "required_input": [
+        {
+            "info": {
+                "digest": "c91db27db87695c42ab03479f5393d83fd881c5c4982579b0115a20c36342d2c",
+                "path": "relative_imports.proto"
+            },
+            "v_name": {
+                "path": "relative_imports.proto"
+            }
+        },
+        {
+            "info": {
+                "digest": "c9d3571c48d70781dc7d4984b1b8974f44cb5953c3be17cacacc15f35f5981eb",
+                "path": "subdir/other.proto"
+            },
+            "v_name": {
+                "path": "subdir/other.proto"
+            }
+        }
+    ],
+    "source_file": [
+        "relative_imports.proto"
+    ],
 }

kythe/cxx/extractor/proto/testdata/custom_vname_config.UNIT

@@ -1,33 +1,33 @@
 {
-  "required_input": [
-    {
-      "v_name": {
-        "corpus": "corpus1",
-        "path": "kythe/cxx/extractor/proto/testdata/simple1.proto"
-      },
-      "info": {
-        "path": "kythe/cxx/extractor/proto/testdata/simple1.proto",
-        "digest": "36aa2295772636e8e940333faa06f0838953101f353ab52d72279389e16117be"
-      }
-    },
-    {
-      "v_name": {
-        "corpus": "corpus2",
-        "root": "kythe/cxx/extractor/proto/testdata",
-        "path": "simple2.proto"
-      },
-      "info": {
-        "path": "kythe/cxx/extractor/proto/testdata/simple2.proto",
-        "digest": "0db133854842fd4a2112176264bb7e596671db36d1d91dcb9910c662278680ae"
-      }
-    }
-  ],
-  "argument": [
-    "kythe/cxx/extractor/proto/testdata/simple1.proto",
-    "kythe/cxx/extractor/proto/testdata/simple2.proto"
-  ],
-  "source_file": [
-    "kythe/cxx/extractor/proto/testdata/simple1.proto",
-    "kythe/cxx/extractor/proto/testdata/simple2.proto"
-  ]
+    "argument": [
+        "kythe/cxx/extractor/proto/testdata/simple1.proto",
+        "kythe/cxx/extractor/proto/testdata/simple2.proto"
+    ],
+    "required_input": [
+        {
+            "info": {
+                "digest": "36aa2295772636e8e940333faa06f0838953101f353ab52d72279389e16117be",
+                "path": "kythe/cxx/extractor/proto/testdata/simple1.proto"
+            },
+            "v_name": {
+                "corpus": "corpus1",
+                "path": "kythe/cxx/extractor/proto/testdata/simple1.proto"
+            }
+        },
+        {
+            "info": {
+                "digest": "0db133854842fd4a2112176264bb7e596671db36d1d91dcb9910c662278680ae",
+                "path": "kythe/cxx/extractor/proto/testdata/simple2.proto"
+            },
+            "v_name": {
+                "corpus": "corpus2",
+                "path": "simple2.proto",
+                "root": "kythe/cxx/extractor/proto/testdata"
+            }
+        }
+    ],
+    "source_file": [
+        "kythe/cxx/extractor/proto/testdata/simple1.proto",
+        "kythe/cxx/extractor/proto/testdata/simple2.proto"
+    ],
 }

kythe/cxx/extractor/proto/testdata/duplicate_imports.UNIT

@@ -1,33 +1,33 @@
 {
-  "required_input": [
-    {
-      "v_name": {
-        "path": "kythe/cxx/extractor/proto/testdata/relative_imports.proto"
-      },
-      "info": {
-        "path": "kythe/cxx/extractor/proto/testdata/relative_imports.proto",
-        "digest": "c91db27db87695c42ab03479f5393d83fd881c5c4982579b0115a20c36342d2c"
-      }
-    },
-    {
-      "v_name": {
-        "path": "kythe/cxx/extractor/proto/testdata/subdir/other.proto"
-      },
-      "info": {
-        "path": "kythe/cxx/extractor/proto/testdata/subdir/other.proto",
-        "digest": "c9d3571c48d70781dc7d4984b1b8974f44cb5953c3be17cacacc15f35f5981eb"
-      }
-    }
-  ],
-  "argument": [
-    "kythe/cxx/extractor/proto/testdata/relative_imports.proto",
-    "kythe/cxx/extractor/proto/testdata/subdir/other.proto",
-    "--",
-    "--proto_path",
-    "kythe/cxx/extractor/proto/testdata/subdir"
-  ],
-  "source_file": [
-    "kythe/cxx/extractor/proto/testdata/relative_imports.proto",
-    "kythe/cxx/extractor/proto/testdata/subdir/other.proto"
-  ]
+    "argument": [
+        "kythe/cxx/extractor/proto/testdata/relative_imports.proto",
+        "kythe/cxx/extractor/proto/testdata/subdir/other.proto",
+        "--",
+        "--proto_path",
+        "kythe/cxx/extractor/proto/testdata/subdir"
+    ],
+    "required_input": [
+        {
+            "info": {
+                "digest": "c91db27db87695c42ab03479f5393d83fd881c5c4982579b0115a20c36342d2c",
+                "path": "kythe/cxx/extractor/proto/testdata/relative_imports.proto"
+            },
+            "v_name": {
+                "path": "kythe/cxx/extractor/proto/testdata/relative_imports.proto"
+            }
+        },
+        {
+            "info": {
+                "digest": "c9d3571c48d70781dc7d4984b1b8974f44cb5953c3be17cacacc15f35f5981eb",
+                "path": "kythe/cxx/extractor/proto/testdata/subdir/other.proto"
+            },
+            "v_name": {
+                "path": "kythe/cxx/extractor/proto/testdata/subdir/other.proto"
+            }
+        }
+    ],
+    "source_file": [
+        "kythe/cxx/extractor/proto/testdata/relative_imports.proto",
+        "kythe/cxx/extractor/proto/testdata/subdir/other.proto"
+    ],
 }

kythe/cxx/extractor/proto/testdata/duplicate_imports2.UNIT

@@ -1,33 +1,33 @@
 {
-  "required_input": [
-    {
-      "v_name": {
-        "path": "kythe/cxx/extractor/proto/testdata/relative_imports.proto"
-      },
-      "info": {
-        "path": "kythe/cxx/extractor/proto/testdata/relative_imports.proto",
-        "digest": "c91db27db87695c42ab03479f5393d83fd881c5c4982579b0115a20c36342d2c"
-      }
-    },
-    {
-      "v_name": {
-        "path": "kythe/cxx/extractor/proto/testdata/subdir/other.proto"
-      },
-      "info": {
-        "path": "kythe/cxx/extractor/proto/testdata/subdir/other.proto",
-        "digest": "c9d3571c48d70781dc7d4984b1b8974f44cb5953c3be17cacacc15f35f5981eb"
-      }
-    }
-  ],
-  "argument": [
-    "kythe/cxx/extractor/proto/testdata/subdir/other.proto",
-    "kythe/cxx/extractor/proto/testdata/relative_imports.proto",
-    "--",
-    "--proto_path",
-    "kythe/cxx/extractor/proto/testdata/subdir"
-  ],
-  "source_file": [
-    "kythe/cxx/extractor/proto/testdata/subdir/other.proto",
-    "kythe/cxx/extractor/proto/testdata/relative_imports.proto"
-  ]
+    "argument": [
+        "kythe/cxx/extractor/proto/testdata/subdir/other.proto",
+        "kythe/cxx/extractor/proto/testdata/relative_imports.proto",
+        "--",
+        "--proto_path",
+        "kythe/cxx/extractor/proto/testdata/subdir"
+    ],
+    "required_input": [
+        {
+            "info": {
+                "digest": "c91db27db87695c42ab03479f5393d83fd881c5c4982579b0115a20c36342d2c",
+                "path": "kythe/cxx/extractor/proto/testdata/relative_imports.proto"
+            },
+            "v_name": {
+                "path": "kythe/cxx/extractor/proto/testdata/relative_imports.proto"
+            }
+        },
+        {
+            "info": {
+                "digest": "c9d3571c48d70781dc7d4984b1b8974f44cb5953c3be17cacacc15f35f5981eb",
+                "path": "kythe/cxx/extractor/proto/testdata/subdir/other.proto"
+            },
+            "v_name": {
+                "path": "kythe/cxx/extractor/proto/testdata/subdir/other.proto"
+            }
+        }
+    ],
+    "source_file": [
+        "kythe/cxx/extractor/proto/testdata/subdir/other.proto",
+        "kythe/cxx/extractor/proto/testdata/relative_imports.proto"
+    ],
 }

kythe/cxx/extractor/proto/testdata/kzip_diff_test.sh

@@ -13,21 +13,57 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+
+# Output every line except ones that contains match_string. Also, do not output
+# extra_lines_to_skip lines that follow the line that contained match_string.
+function skip() {
+  if [ $# -ne 3 ] && [ $# -ne 2 ]; then
+    echo "Usage: $0 match_string extra_lines_to_skip [input_file]"
+    return 1
+  fi
+  local -r match_string="$1"
+  local -r lines_to_skip="$2"
+
+   if [ $# -eq 2 ]; then
+    awk "/$match_string/{skip=$lines_to_skip;next} skip>0{--skip;next} {print}"
+  else
+    awk "/$match_string/{skip=$lines_to_skip;next} skip>0{--skip;next} {print}" "$3"
+  fi
+  return 0
+}
+
+# Modify input_file to remove lines that contains match_string. Also remove
+# extra_lines_to_skip lines that follow the line that contained match_string.
+function skip_inplace() {
+  if [ $# -ne 3 ]; then
+    echo "Usage: $0 match_string lines_to_skip input_file"
+    return 1
+  fi
+  local -r match_string="$1"
+  local -r lines_to_skip="$2"
+  local -r input_file="$3"
+
+   local -r temp_file=$(mktemp)
+  skip "$match_string" "$lines_to_skip" "$input_file" > "$temp_file"
+  mv "$temp_file" "$input_file"
+}
+
+
 set -e
 
 KZIP_TOOL=$1; shift
-JQ=$1; shift
+FORMATJSON=$1; shift
 KZIP=$1; shift
 GOLDEN=$1; shift
 
 # Work in tmpdir because the runfiles dir is not writable
 cp $KZIP "${TEST_TMPDIR}/file.kzip"
-# Extract textproto version of compilation unit from kzip.
-UNIT=$("${KZIP_TOOL}" view "${TEST_TMPDIR}/file.kzip")
+# Extract json version of compilation unit from kzip.
 UNIT_FILE="${TEST_TMPDIR}/$(basename $GOLDEN)"
+"${KZIP_TOOL}" view "${TEST_TMPDIR}/file.kzip" | "$FORMATJSON" > "$UNIT_FILE"
 # Remove working_directory, which will change depending on the machine the test
 # is run on.
-echo "$UNIT" | $JQ 'del(.working_directory)' > "$UNIT_FILE"
+skip_inplace "working_directory" 0 $UNIT_FILE
 
 echo
 echo "Diffing generated unit against golden"