Restore 3 Gnarly tests

.dockstore.yml

@@ -6,7 +6,7 @@ workflows:
      testParameterFiles:
           - /scripts/cnv_cromwell_tests/germline/cnv_germline_case_scattered_workflow.json
      filters:
-         branches: 
+         branches:
              - master
          tags:
              - /.*/
@@ -33,7 +33,7 @@ workflows:
    - name: cnv_somatic_pair_workflow
      subclass: WDL
      primaryDescriptorPath: /scripts/cnv_wdl/somatic/cnv_somatic_pair_workflow.wdl
-     testParameterFiles: 
+     testParameterFiles:
           -  /scripts/cnv_cromwell_tests/somatic/cnv_somatic_pair_wes_do-gc_workflow.json
      filters:
          branches:
@@ -88,6 +88,292 @@ workflows:
              - master
          tags:
              - /.*/
+   - name: GvsAssignIds
+     subclass: WDL
+     primaryDescriptorPath: /scripts/variantstore/wdl/GvsAssignIds.wdl
+     testParameterFiles:
+         - /scripts/variantstore/wdl/GvsAssignIds.example.inputs.json
+     filters:
+         branches:
+             - master
+             - ah_var_store
+         tags:
+             - /.*/
+   - name: GvsBenchmarkExtractTask
+     subclass: WDL
+     primaryDescriptorPath: /scripts/variantstore/wdl/GvsBenchmarkExtractTask.wdl
+     filters:
+         branches:
+             - master
+             - ah_var_store
+         tags:
+             - /.*/
+   - name: GvsRescatterCallsetInterval
+     subclass: WDL
+     primaryDescriptorPath: /scripts/variantstore/wdl/GvsRescatterCallsetInterval.wdl
+     filters:
+         branches:
+             - master
+             - ah_var_store
+         tags:
+             - /.*/
+   - name: GvsCreateFilterSet
+     subclass: WDL
+     primaryDescriptorPath: /scripts/variantstore/wdl/GvsCreateFilterSet.wdl
+     testParameterFiles:
+         - /scripts/variantstore/wdl/GvsCreateFilterSet.example.inputs.json
+     filters:
+         branches:
+             - master
+             - ah_var_store
+         tags:
+             - /.*/
+   - name: GvsPopulateAltAllele
+     subclass: WDL
+     primaryDescriptorPath: /scripts/variantstore/wdl/GvsPopulateAltAllele.wdl
+     filters:
+         branches:
+             - master
+             - ah_var_store
+         tags:
+             - /.*/
+   - name: GvsCreateTables
+     subclass: WDL
+     primaryDescriptorPath: /scripts/variantstore/wdl/GvsCreateTables.wdl
+     testParameterFiles:
+         - /scripts/variantstore/wdl/GvsCreateTables.example.inputs.json
+     filters:
+         branches:
+             - master
+             - ah_var_store
+         tags:
+             - /.*/
+   - name: GvsCreateVDS
+     subclass: WDL
+     primaryDescriptorPath: /scripts/variantstore/wdl/GvsCreateVDS.wdl
+     filters:
+       branches:
+         - master
+         - ah_var_store
+       tags:
+         - /.*/
+   - name: GvsExtractCallset
+     subclass: WDL
+     primaryDescriptorPath: /scripts/variantstore/wdl/GvsExtractCallset.wdl
+     filters:
+         branches:
+             - master
+             - ah_var_store
+             - VS-1379-ploidy
+         tags:
+             - /.*/
+   - name: GvsImportGenomes
+     subclass: WDL
+     primaryDescriptorPath: /scripts/variantstore/wdl/GvsImportGenomes.wdl
+     filters:
+         branches:
+             - master
+             - ah_var_store
+         tags:
+             - /.*/
+   - name: GvsBulkIngestGenomes
+     subclass: WDL
+     primaryDescriptorPath: /scripts/variantstore/wdl/GvsBulkIngestGenomes.wdl
+     filters:
+         branches:
+             - master
+             - ah_var_store
+         tags:
+             - /.*/
+   - name: GvsPrepareRangesCallset
+     subclass: WDL
+     primaryDescriptorPath: /scripts/variantstore/wdl/GvsPrepareRangesCallset.wdl
+     testParameterFiles:
+         - /scripts/variantstore/wdl/GvsPrepareRangesCallset.example.inputs.json
+     filters:
+         branches:
+             - master
+             - ah_var_store
+         tags:
+             - /.*/
+   - name: GvsCreateVATfromVDS
+     subclass: WDL
+     primaryDescriptorPath: /scripts/variantstore/wdl/GvsCreateVATfromVDS.wdl
+     filters:
+         branches:
+             - master
+             - ah_var_store
+             - rsa_vs_1355
+             - gg_VS-1395_Have_CreateVATFromVDS_MoreGoodness
+         tags:
+             - /.*/
+   - name: GvsCreateVATFilesFromBigQuery
+     subclass: WDL
+     primaryDescriptorPath: /scripts/variantstore/variant_annotations_table/GvsCreateVATFilesFromBigQuery.wdl
+     filters:
+         branches:
+             - master
+             - ah_var_store
+         tags:
+             - /.*/
+   - name: GvsValidateVat
+     subclass: WDL
+     primaryDescriptorPath: /scripts/variantstore/variant_annotations_table/GvsValidateVAT.wdl
+     testParameterFiles:
+         - /scripts/variantstore/variant_annotations_table/GvsValidateVat.example.inputs.json
+     filters:
+         branches:
+             - master
+             - ah_var_store
+         tags:
+             - /.*/
+   - name: GvsExtractCohortFromSampleNames
+     subclass: WDL
+     primaryDescriptorPath: /scripts/variantstore/wdl/GvsExtractCohortFromSampleNames.wdl
+     filters:
+         branches:
+             - master
+             - ah_var_store
+         tags:
+             - /.*/
+   - name: GvsWithdrawSamples
+     subclass: WDL
+     primaryDescriptorPath: /scripts/variantstore/wdl/GvsWithdrawSamples.wdl
+     filters:
+         branches:
+             - master
+             - ah_var_store
+         tags:
+             - /.*/
+   - name: GvsJointVariantCalling
+     subclass: WDL
+     primaryDescriptorPath: /scripts/variantstore/wdl/GvsJointVariantCalling.wdl
+     filters:
+         branches:
+             - master
+             - ah_var_store
+             - rsa_vs_1300
+         tags:
+             - /.*/
+   - name: GvsBeta
+     subclass: WDL
+     primaryDescriptorPath: /scripts/variantstore/wdl/GvsJointVariantCalling.wdl
+     filters:
+         branches:
+             # `master` is here so there will be *something* under the branches filter, but GVS WDLs are not on `master`
+             # so this shouldn't show up in the list of available versions in Dockstore or Terra.
+             - master
+         tags:
+             - /.*/
+   - name: GvsJointVariantCallsetCost
+     subclass: WDL
+     primaryDescriptorPath: /scripts/variantstore/wdl/GvsJointVariantCallsetCost.wdl
+     filters:
+         branches:
+             - master
+             - ah_var_store
+         tags:
+             - /.*/
+   - name: GvsCalculatePrecisionAndSensitivity
+     subclass: WDL
+     primaryDescriptorPath: /scripts/variantstore/wdl/GvsCalculatePrecisionAndSensitivity.wdl
+     filters:
+         branches:
+             - master
+             - ah_var_store
+             - rsa_vs_1168_bge
+         tags:
+             - /.*/
+   - name: GvsQuickstartVcfIntegration
+     subclass: WDL
+     primaryDescriptorPath: /scripts/variantstore/wdl/GvsQuickstartVcfIntegration.wdl
+     filters:
+         branches:
+             - master
+             - ah_var_store
+         tags:
+             - /.*/
+   - name: GvsQuickstartHailIntegration
+     subclass: WDL
+     primaryDescriptorPath: /scripts/variantstore/wdl/GvsQuickstartHailIntegration.wdl
+     filters:
+         branches:
+             - master
+             - ah_var_store
+         tags:
+             - /.*/
+   - name: GvsQuickstartIntegration
+     subclass: WDL
+     primaryDescriptorPath: /scripts/variantstore/wdl/GvsQuickstartIntegration.wdl
+     filters:
+         branches:
+             - master
+             - ah_var_store
+             - vs_1178_merge_master_to_ah_var_store_again
+         tags:
+             - /.*/
+   - name: GvsIngestTieout
+     subclass: WDL
+     primaryDescriptorPath: /scripts/variantstore/wdl/GvsIngestTieout.wdl
+     filters:
+         branches:
+             - master
+             - ah_var_store
+         tags:
+             - /.*/
+   - name: GvsCallsetCost
+     subclass: WDL
+     primaryDescriptorPath: /scripts/variantstore/wdl/GvsCallsetCost.wdl
+     filters:
+         branches:
+             - master
+             - ah_var_store
+         tags:
+             - /.*/
+   - name: GvsExtractAvroFilesForHail
+     subclass: WDL
+     primaryDescriptorPath: /scripts/variantstore/wdl/GvsExtractAvroFilesForHail.wdl
+     filters:
+         branches:
+             - master
+             - ah_var_store
+         tags:
+             - /.*/
+   - name: GvsCallsetStatistics
+     subclass: WDL
+     primaryDescriptorPath: /scripts/variantstore/wdl/GvsCallsetStatistics.wdl
+     filters:
+         branches:
+             - master
+             - ah_var_store
+         tags:
+             - /.*/
+   - name: GvsFindHailCruft
+     subclass: WDL
+     primaryDescriptorPath: /scripts/variantstore/wdl/GvsFindHailCruft.wdl
+     filters:
+         branches:
+             - master
+             - ah_var_store
+         tags:
+             - /.*/
+   - name: GvsTieoutVcfMaxAltAlleles
+     subclass: WDL
+     primaryDescriptorPath: /scripts/variantstore/wdl/GvsTieoutVcfMaxAltAlleles.wdl
+     filters:
+       branches:
+         - ah_var_store
+         - master
+       tags:
+         - /.*/
+   - name: HailFromWdl
+     subclass: WDL
+     primaryDescriptorPath: /scripts/variantstore/wdl/HailFromWdl.wdl
+     filters:
+         branches:
+             - master
+         tags:
+             - /.*/
    - name: MitochondriaPipeline
      subclass: WDL
      primaryDescriptorPath: /scripts/mitochondria_m2_wdl/MitochondriaPipeline.wdl
@@ -136,3 +422,35 @@ workflows:
              - master
          tags:
              - /.*/
+   - name: GvsExtractCallsetPgen
+     subclass: WDL
+     primaryDescriptorPath: /scripts/variantstore/wdl/GvsExtractCallsetPgen.wdl
+     filters:
+       branches:
+         - master
+         - ah_var_store
+         - EchoCallset
+   - name: GvsExtractCallsetPgenMerged
+     subclass: WDL
+     primaryDescriptorPath: /scripts/variantstore/wdl/GvsExtractCallsetPgenMerged.wdl
+     filters:
+       branches:
+         - master
+         - ah_var_store
+         - EchoCallset
+         - vs_1178_merge_master_to_ah_var_store_again
+   - name: MergePgenHierarchicalWdl
+     subclass: WDL
+     primaryDescriptorPath: /scripts/variantstore/wdl/MergePgenHierarchical.wdl
+     filters:
+       branches:
+         - master
+         - ah_var_store
+         - EchoCallset
+   - name: GvsTieoutPgenToVcf
+     subclass: WDL
+     primaryDescriptorPath: /scripts/variantstore/wdl/GvsTieoutPgenToVcf.wdl
+     filters:
+       branches:
+         - ah_var_store
+         - EchoCallset

build.gradle

@@ -345,6 +345,9 @@ dependencies {
     implementation('net.grey-panther:natural-comparator:1.1')
     implementation('com.fasterxml.jackson.module:jackson-module-scala_2.12:2.9.8')
 
+    // pgen jni
+    implementation('org.broadinstitute:pgenjni:1.0.1')
+
     testUtilsImplementation sourceSets.main.output
     testUtilsImplementation 'org.testng:testng:' + testNGVersion
     testUtilsImplementation 'org.apache.hadoop:hadoop-minicluster:' + hadoopVersion
@@ -353,6 +356,8 @@ dependencies {
 
     testImplementation "org.mockito:mockito-core:2.28.2"
     testImplementation "com.google.jimfs:jimfs:1.1"
+
+    testImplementation "com.github.luben:zstd-jni:1.5.5-11"
 }
 
 // This list needs to be kept in sync with the corresponding list in scripts/dockertest.gradle.
@@ -605,6 +610,10 @@ task collectBundleIntoDir(type: Copy) {
     from("scripts/sv", { into("scripts/sv") })
     from("scripts/cnv_wdl/", { into("scripts/cnv_wdl") })
     from("scripts/mutect2_wdl/", { into("scripts/mutect2_wdl") })
+    from("scripts/variantstore/") {
+        include 'wdl/extract/curate_input_array_files.py'
+        into("scripts/variantstore")
+    }
     from("scripts/dataproc-cluster-ui", { into("scripts/")})
     into "$buildDir/bundle-files-collected"
 }

build_docker.sh

@@ -124,9 +124,9 @@ fi
 
 echo "Building image to tag ${REPO_PRJ}:${GITHUB_TAG}..."
 if [ -n "${IS_PUSH}" ]; then
-    docker build -t ${REPO_PRJ}:${GITHUB_TAG} --build-arg RELEASE=${RELEASE}  --squash .
+    docker build -t ${REPO_PRJ}:${GITHUB_TAG} --build-arg RELEASE=${RELEASE}  --squash . --iidfile /tmp/idfile.txt
 else
-    docker build -t ${REPO_PRJ}:${GITHUB_TAG} --build-arg RELEASE=${RELEASE} .
+    docker build -t ${REPO_PRJ}:${GITHUB_TAG} --build-arg RELEASE=${RELEASE} . --iidfile /tmp/idfile.txt
 fi
 # Since we build the docker image with stages, the first build stage for GATK will be leftover in
 # the local docker context after executing the above commands. This step reclaims that space automatically