diff --git a/.github/actions/upload-gatk-test-results/action.yml b/.github/actions/upload-gatk-test-results/action.yml
index 62166d4c64f..a1645913cf1 100644
--- a/.github/actions/upload-gatk-test-results/action.yml
+++ b/.github/actions/upload-gatk-test-results/action.yml
@@ -11,10 +11,10 @@ inputs:
description: 'The job-unique identifier to use for uploading the results of this run into a google bucket'
required: true
is-docker:
- description: 'Whether to label the uplaod artifact as coming from a docker test'
+ description: 'Whether to label the upload artifact as coming from a docker test'
required: false
identifier:
- description: 'Identifier to use for finding the unique name for jobs in order to determine html logs location (eg. "Java 8 build and test integration")'
+ description: 'Identifier to use for finding the unique name for jobs in order to determine html logs location (eg. "Java 17 build and test integration")'
required: true
## Secrets and token inputs
repo-token:
@@ -23,7 +23,7 @@ inputs:
bot-comment-key:
description: 'Key corresponding to the user account to be used for making comments on github about test failures'
required: false
-## option to skip all but the artifact uplaod
+## option to skip all but the artifact upload
only-artifact:
description: 'if "true" this will skip any uploading steps that require permissions and only upload the artifact file'
required: false
diff --git a/.github/workflows/gatk-tests.yml b/.github/workflows/gatk-tests.yml
index a908b98cd15..20471b29890 100644
--- a/.github/workflows/gatk-tests.yml
+++ b/.github/workflows/gatk-tests.yml
@@ -7,7 +7,7 @@ on:
workflow_dispatch:
env:
- CROMWELL_VERSION: 51
+ CROMWELL_VERSION: 84
TERM: dumb
GRADLE_OPTS: "-Xmx2048m -Dorg.gradle.daemon=false"
HELLBENDER_TEST_INPUTS: gs://hellbender/test/resources/
@@ -73,19 +73,10 @@ jobs:
needs: check-secrets
strategy:
matrix:
- java: [ 8, 11 ]
+ java: [ 17 ]
experimental: [ false ]
- scalaVersion: [ 2.11, 2.12 ]
+ scalaVersion: [ 2.12 ]
testType: [ cloud, integration, unit ]
- exclude:
- - java: 11
- scalaVersion: 2.11
- - java: 8
- scalaVersion: 2.12
- - java: 8
- testType: integration
- - java: 8
- testType: unit
fail-fast: false
continue-on-error: ${{ matrix.experimental }}
env:
@@ -100,7 +91,7 @@ jobs:
uses: actions/setup-java@v2
with:
java-version: ${{ matrix.Java }}
- distribution: 'adopt'
+ distribution: 'temurin'
cache: gradle
- name: 'Compile with Gradle'
@@ -120,6 +111,10 @@ jobs:
if: needs.check-secrets.outputs.google-credentials == 'true'
run: echo "HELLBENDER_JSON_SERVICE_ACCOUNT_KEY=${{ steps.auth.outputs.credentials_file_path }}" >> $GITHUB_ENV
+ # See https://cloud.google.com/compute/docs/troubleshooting/known-issues#ubuntu-systems
+ - name: "get the updated signature key for google cloud"
+ run: curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key --keyring /usr/share/keyrings/cloud.google.gpg add -
+
- name: 'Set up Cloud SDK'
if: needs.check-secrets.outputs.google-credentials == 'true'
uses: google-github-actions/setup-gcloud@v0
@@ -134,7 +129,7 @@ jobs:
if: ${{ needs.check-secrets.outputs.google-credentials == 'true' || matrix.testType != 'cloud'}}
id: jacoco-tests
run: |
- ./gradlew -Dscala.version=${{ env.SCALA_VERSION }} jacocoTestReport
+ ./gradlew --daemon -Dscala.version=${{ env.SCALA_VERSION }} jacocoTestReport
- uses: ./.github/actions/upload-gatk-test-results
if: always()
@@ -154,9 +149,9 @@ jobs:
runs-on: ubuntu-latest
strategy:
matrix:
- java: [ 8 ]
+ java: [ 17 ]
experimental: [ false ]
- scalaVersion: [ 2.11 ]
+ scalaVersion: [ 2.12 ]
testType: [ integration, unit, variantcalling, conda ]
fail-fast: false
continue-on-error: ${{ matrix.experimental }}
@@ -174,7 +169,7 @@ jobs:
uses: actions/setup-java@v2
with:
java-version: ${{ matrix.Java }}
- distribution: 'adopt'
+ distribution: 'temurin'
cache: gradle
#Google Cloud stuff
@@ -190,6 +185,10 @@ jobs:
if: needs.check-secrets.outputs.google-credentials == 'true'
run: echo "HELLBENDER_JSON_SERVICE_ACCOUNT_KEY=${{ steps.auth.outputs.credentials_file_path }}" >> $GITHUB_ENV
+ # See https://cloud.google.com/compute/docs/troubleshooting/known-issues#ubuntu-systems
+ - name: "get the updated signature key for google cloud"
+ run: curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key --keyring /usr/share/keyrings/cloud.google.gpg add -
+
- name: 'Set up Cloud SDK'
if: needs.check-secrets.outputs.google-credentials == 'true'
uses: google-github-actions/setup-gcloud@v0
@@ -259,6 +258,12 @@ jobs:
- uses: actions/checkout@v2
with:
fetch: 0
+ - name: Set up java 17
+ uses: actions/setup-java@v2
+ with:
+ java-version: '17'
+ distribution: 'temurin'
+ cache: gradle
- uses: ./.github/actions/install-cromwell
with:
CROMWELL_VERSION: ${{ env.CROMWELL_VERSION }}
@@ -273,11 +278,11 @@ jobs:
- uses: actions/checkout@v2
with:
fetch: 0
- - name: Set up java 8
+ - name: Set up java 17
uses: actions/setup-java@v2
with:
- java-version: '8'
- distribution: 'adopt'
+ java-version: '17'
+ distribution: 'temurin'
cache: gradle
- uses: ./.github/actions/install-cromwell
with:
@@ -300,11 +305,11 @@ jobs:
fetch-depth: 0
- name: pull lfs files
run: git lfs pull
- - name: Set up java 8
+ - name: Set up java 17
uses: actions/setup-java@v2
with:
- java-version: '8'
- distribution: 'adopt'
+ java-version: '17'
+ distribution: 'temurin'
cache: gradle
- uses: ./.github/actions/install-cromwell
with:
@@ -335,6 +340,8 @@ jobs:
run: |
echo "Running CNV somatic workflows";
bash scripts/cnv_cromwell_tests/somatic/run_cnv_somatic_workflows.sh;
+ echo "Gather CNV somatic cromwell workflows logs";
+ bash -c "find /home/runner/work/gatk/gatk/scripts/cnv_cromwell_tests/somatic/cromwell-executions -exec cat {} \;";
- name: "M2_WDL_TEST"
if: ${{ matrix.wdlTest == 'RUN_M2_WDL' }}
diff --git a/Dockerfile b/Dockerfile
index 3c551f0ff24..57f7f5ee5fb 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -8,11 +8,27 @@ ADD . /gatk
WORKDIR /gatk
# Get an updated gcloud signing key, in case the one in the base image has expired
-RUN curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add -
+RUN rm /etc/apt/sources.list.d/google-cloud-sdk.list
+RUN apt update
+RUN apt-key list
+RUN curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key --keyring /usr/share/keyrings/cloud.google.gpg add -
RUN add-apt-repository universe && apt update
RUN apt-get --assume-yes install git-lfs
RUN git lfs install --force
+##Get Java 17 temurin JDK
+#RUN apt update && apt upgrade
+RUN apt install wget
+RUN wget https://github.com/adoptium/temurin17-binaries/releases/download/jdk-17.0.1%2B12/OpenJDK17U-jdk_x64_linux_hotspot_17.0.1_12.tar.gz
+RUN tar -xvf OpenJDK17U-jdk_x64_linux_hotspot_17.*.tar.gz
+RUN mv jdk-17.0.1+12 /opt/
+#
+ENV JAVA_HOME /opt/jdk-17.0.1+12
+ENV PATH $JAVA_HOME/bin:$PATH
+RUN echo $JAVA_HOME
+RUN update-alternatives --install /usr/bin/java java /opt/jdk-17.0.1+12/bin/java 1
+RUN java -version
+
#Download only resources required for the build, not for testing
RUN git lfs pull --include src/main/resources/large
@@ -23,6 +39,23 @@ RUN unzip -o -j $( find /gatk/unzippedJar -name "gatkPython*.zip" ) -d /gatk/unz
# Using OpenJDK 8
FROM broadinstitute/gatk:gatkbase-2.3.0
+RUN rm /etc/apt/sources.list.d/google-cloud-sdk.list
+RUN apt update
+RUN apt-key list
+
+#Get Java 17 temurin JDK
+#RUN apt update && apt upgrade
+RUN apt install wget
+RUN wget https://github.com/adoptium/temurin17-binaries/releases/download/jdk-17.0.1%2B12/OpenJDK17U-jdk_x64_linux_hotspot_17.0.1_12.tar.gz
+RUN tar -xvf OpenJDK17U-jdk_x64_linux_hotspot_17.*.tar.gz
+RUN mv jdk-17.0.1+12 /opt/
+
+ENV JAVA_HOME /opt/jdk-17.0.1+12
+ENV PATH $JAVA_HOME/bin:$PATH
+RUN echo $JAVA_HOME
+RUN update-alternatives --install /usr/bin/java java /opt/jdk-17.0.1+12/bin/java 1
+RUN java -version
+
WORKDIR /gatk
# Location of the unzipped gatk bundle files
@@ -43,6 +76,9 @@ RUN mkdir .gradle
WORKDIR /gatk
+# use UTF-8 encoding to get around (??) scala class name shapeless/$tilde$qmark$greater$?.class
+ENV LANG="en_US.UTF-8" LANGUAGE="en_US:en" LC_ALL="en_US.UTF-8"
+
# Create a simple unit test runner
ENV CI true
RUN echo "source activate gatk" > /root/run_unit_tests.sh && \
@@ -54,11 +90,11 @@ RUN echo "source activate gatk" > /root/run_unit_tests.sh && \
echo "mkdir /gatk/srcdir" >> /root/run_unit_tests.sh && \
echo "cp -rp /gatkCloneMountPoint/src/main/java/* /gatk/srcdir" >> /root/run_unit_tests.sh && \
echo "export SOURCE_DIR=/gatk/srcdir" >> /root/run_unit_tests.sh && \
- echo "export GRADLE_OPTS=\"-Xmx1024m -Dorg.gradle.daemon=false\"" /root/run_unit_tests.sh && \
- echo "export CP_DIR=/gatk/testClasses" /root/run_unit_tests.sh && \
+ echo "export GRADLE_OPTS=\"-Xmx1024m -Dorg.gradle.daemon=false --add-opens java.prefs/java.util.prefs=ALL-UNNAMED\"" >> /root/run_unit_tests.sh && \
+ echo "export CP_DIR=/gatk/testClasses" >> /root/run_unit_tests.sh && \
echo "ln -s /gatkCloneMountPoint/src/ /gatkCloneMountPoint/scripts/docker/src" >> /root/run_unit_tests.sh && \
echo "ln -s /gatkCloneMountPoint/build/ /gatkCloneMountPoint/scripts/docker/build" >> /root/run_unit_tests.sh && \
- echo "cd /gatk/ && /gatkCloneMountPoint/gradlew -b /gatkCloneMountPoint/dockertest.gradle testOnPackagedReleaseJar jacocoTestReportOnPackagedReleaseJar -a -p /gatkCloneMountPoint" >> /root/run_unit_tests.sh
+ echo "cd /gatk/ && /gatkCloneMountPoint/gradlew -Dfile.encoding=UTF-8 -b /gatkCloneMountPoint/dockertest.gradle testOnPackagedReleaseJar jacocoTestReportOnPackagedReleaseJar -a -p /gatkCloneMountPoint" >> /root/run_unit_tests.sh
WORKDIR /root
RUN cp -r /root/run_unit_tests.sh /gatk
diff --git a/README.md b/README.md
index c90e7a78802..30c15f47797 100644
--- a/README.md
+++ b/README.md
@@ -51,7 +51,7 @@ releases of the toolkit.
## Requirements
* To run GATK:
- * Java 8 is needed to run or build GATK.
+ * Java 17 is needed to run or build GATK.
We recommend either of the following:
* OpenJDK 8 with Hotspot from [AdoptOpenJdk](https://adoptopenjdk.net/)
* [OracleJDK 8](https://www.oracle.com/technetwork/java/javase/downloads/jdk8-downloads-2133151.html)
@@ -61,7 +61,7 @@ releases of the toolkit.
See [Python Dependencies](#python) for more information.
* R 3.2.5 (needed for producing plots in certain tools)
* To build GATK:
- * A Java 8 JDK
+ * A Java 17 JDK
* Git 2.5 or greater
* [git-lfs](https://git-lfs.github.com/) 1.1.0 or greater. Required to download the large files used to build GATK, and
test files required to run the test suite. Run `git lfs install` after downloading, followed by `git lfs pull` from
@@ -454,7 +454,7 @@ We use [git-lfs](https://git-lfs.github.com/) to version and distribute test dat
#### Creating a GATK project in the IntelliJ IDE (last tested with version 2016.2.4):
-* Ensure that you have `gradle` and the Java 8 JDK installed
+* Ensure that you have `gradle` and the Java 17 JDK installed
* You may need to install the TestNG and Gradle plugins (in preferences)
@@ -476,7 +476,7 @@ We use [git-lfs](https://git-lfs.github.com/) to version and distribute test dat
* After downloading project dependencies, IntelliJ should open a new window with your GATK project
-* Make sure that the Java version is set correctly by going to File -> "Project Structure" -> "Project". Check that the "Project SDK" is set to your Java 1.8 JDK, and "Project language level" to 8 (you may need to add your Java 8 JDK under "Platform Settings" -> SDKs if it isn't there already). Then click "Apply"/"Ok".
+* Make sure that the Java version is set correctly by going to File -> "Project Structure" -> "Project". Check that the "Project SDK" is set to your Java 1.17 JDK, and "Project language level" to 17 (you may need to add your Java 17 JDK under "Platform Settings" -> SDKs if it isn't there already). Then click "Apply"/"Ok".
#### Setting up debugging in IntelliJ
diff --git a/build.gradle b/build.gradle
index e2754b5a489..a22bac75e66 100644
--- a/build.gradle
+++ b/build.gradle
@@ -23,8 +23,6 @@ import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar
import de.undercouch.gradle.tasks.download.Download
import javax.tools.ToolProvider
-import java.time.format.DateTimeFormatter
-import java.time.ZonedDateTime
mainClassName = "org.broadinstitute.hellbender.Main"
@@ -63,9 +61,9 @@ repositories {
final htsjdkVersion = System.getProperty('htsjdk.version','2.24.1')
final picardVersion = System.getProperty('picard.version','2.27.1')
final barclayVersion = System.getProperty('barclay.version','4.0.2')
-final sparkVersion = System.getProperty('spark.version', '2.4.5')
-final scalaVersion = System.getProperty('scala.version', '2.11')
-final hadoopVersion = System.getProperty('hadoop.version', '3.2.1')
+final sparkVersion = System.getProperty('spark.version', '3.3.0')
+final scalaVersion = System.getProperty('scala.version', '2.12')
+final hadoopVersion = System.getProperty('hadoop.version', '3.3.1')
final disqVersion = System.getProperty('disq.version','0.3.6')
final genomicsdbVersion = System.getProperty('genomicsdb.version','1.4.3')
final bigQueryVersion = System.getProperty('bigQuery.version', '2.9.0')
@@ -138,19 +136,13 @@ def looksLikeWereInAGitRepository(){
// Ensure that we have a clone of the git repository, and resolve any required git-lfs
// resource files that are needed to run the build but are still lfs stub files.
def ensureBuildPrerequisites(largeResourcesFolder, buildPrerequisitesMessage, skipGitCheck) {
- if (!JavaVersion.current().isJava8Compatible()) {
- throw new GradleException(
- "Java 8 or later is required to build GATK, but ${JavaVersion.current()} was found. "
- + "$buildPrerequisitesMessage")
+ if (!JavaVersion.current().equals(JavaVersion.VERSION_17)) {
+ println("Warning: using Java ${JavaVersion.current()} but only Java 17 has been tested.")
}
- // Make sure we can get a ToolProvider class loader (for Java 8). If not we may have just a JRE.
- if (JavaVersion.current().isJava8() && ToolProvider.getSystemToolClassLoader() == null) {
+ if (!JavaVersion.current().isCompatibleWith(JavaVersion.VERSION_17)) {
throw new GradleException(
- "The ClassLoader obtained from the Java ToolProvider is null. "
- + "A full Java 8 or 11 JDK must be installed, check that you are not using a JRE. $buildPrerequisitesMessage")
- }
- if (!JavaVersion.current().isJava8() && !JavaVersion.current().isJava11()) {
- println("Warning: using Java ${JavaVersion.current()} but only Java 8 and Java 11 have been tested.")
+ "A Java 17 compatible (Java 17 or later) version is required to build GATK, but ${JavaVersion.current()} was found. "
+ + "$buildPrerequisitesMessage")
}
if (!skipGitCheck && !looksLikeWereInAGitRepository() ) {
throw new GradleException("This doesn't appear to be a git folder. " +
@@ -182,11 +174,18 @@ configurations.all {
// force testng dependency so we don't pick up a different version via GenomicsDB
force 'org.testng:testng:' + testNGVersion
force 'org.broadinstitute:barclay:' + barclayVersion
- force 'com.twitter:chill_2.11:0.8.1'
+ force 'com.twitter:chill_2.12:0.10.0'
+ force 'org.apache.commons:commons-math3:3.5'
// make sure we don't pick up an incorrect version of the GATK variant of the google-nio library
// via Picard, etc.
force googleCloudNioDependency
+
+ force 'com.esotericsoftware:kryo:4.0.0'
+
+ // hadoop uses an old jetty that doesn't like the new java version numbering scheme but 11.0.1 is
+ //missing methods used by org.apache.hadoop.http.HttpServer2
+ //force 'org.eclipse.jetty:jetty-servlet:9.4.40.v20210413'
}
all*.exclude group: 'org.slf4j', module: 'slf4j-jdk14' //exclude this to prevent slf4j complaining about to many slf4j bindings
all*.exclude group: 'com.google.guava', module: 'guava-jdk5'
@@ -231,7 +230,7 @@ configurations {
exclude module: 'javax.servlet'
exclude module: 'servlet-api'
exclude group: 'com.esotericsoftware.kryo'
- exclude module: 'spark-mllib_2.11'
+ exclude module: 'spark-mllib_2.12'
exclude group: 'org.scala-lang'
exclude module: 'kryo'
}
@@ -327,7 +326,7 @@ dependencies {
implementation('org.apache.hadoop:hadoop-client:' + hadoopVersion) // should be a 'provided' dependency
implementation('com.github.jsr203hadoop:jsr203hadoop:1.0.3')
- implementation('de.javakaffee:kryo-serializers:0.41') {
+ implementation('de.javakaffee:kryo-serializers:0.45') {
exclude module: 'kryo' // use Spark's version
}
@@ -342,15 +341,15 @@ dependencies {
implementation('com.github.fommil.netlib:netlib-native_system-osx-x86_64:1.1:natives')
// Dependency change for including MLLib
- implementation('com.esotericsoftware:kryo:3.0.3'){
- exclude group: 'com.esotericsoftware', module: 'reflectasm'
+ implementation('com.esotericsoftware:kryo:5.3.0'){
+ //exclude group: 'com.esotericsoftware', module: 'reflectasm'
exclude group: 'org.ow2.asm', module: 'asm'
}
- // Dependency change for including MLLib
- implementation('com.esotericsoftware:reflectasm:1.10.0:shaded') {
- transitive = false
- }
+// // Dependency change for including MLLib
+// implementation('com.esotericsoftware:reflectasm:1.10.0:shaded') {
+// transitive = false
+// }
implementation('com.intel.gkl:gkl:0.8.8') {
exclude module: 'htsjdk'
@@ -378,6 +377,125 @@ dependencies {
testImplementation "com.google.jimfs:jimfs:1.1"
}
+run {
+ // taken from the union of everything encountered by tests, plus everything defined here:
+ // https://github.com/apache/spark/blob/v3.3.0/launcher/src/main/java/org/apache/spark/launcher/JavaModuleOptions.java
+ jvmArgs = [
+ '--add-opens', 'java.base/java.lang=ALL-UNNAMED',
+ '--add-opens', 'java.base/java.lang.invoke=ALL-UNNAMED',
+ '--add-opens', 'java.base/java.lang.reflect=ALL-UNNAMED',
+ '--add-opens', 'java.base/java.io=ALL-UNNAMED',
+ '--add-opens', 'java.base/java.net=ALL-UNNAMED',
+ '--add-opens', 'java.base/java.nio=ALL-UNNAMED',
+ '--add-opens', 'java.base/java.util=ALL-UNNAMED',
+ '--add-opens', 'java.base/java.util.concurrent=ALL-UNNAMED',
+ '--add-opens', 'java.base/java.util.concurrent.atomic=ALL-UNNAMED',
+ '--add-opens', 'java.base/sun.nio.ch=ALL-UNNAMED',
+ '--add-opens', 'java.base/sun.nio.cs=ALL-UNNAMED',
+ '--add-opens', 'java.base/sun.security.action=ALL-UNNAMED',
+ '--add-opens', 'java.base/sun.util.calendar=ALL-UNNAMED',
+ '--add-opens', 'java.base/sun.nio.fs=ALL-UNNAMED',
+ '--add-opens', 'java.base/java.nio.channels.spi=ALL-UNNAMED',
+ '--add-opens', 'java.base/jdk.internal.ref=ALL-UNNAMED',
+ '--add-opens', 'java.base/java.lang.ref=ALL-UNNAMED',
+ '--add-opens', 'java.base/java.util.zip=ALL-UNNAMED',
+ '--add-opens', 'java.base/java.util.jar=ALL-UNNAMED',
+ '--add-opens', 'java.base/java.nio.file.attribute=ALL-UNNAMED',
+ '--add-opens', 'java.base/jdk.internal.loader=ALL-UNNAMED',
+ '--add-opens', 'java.base/sun.net.www.protocol.jar=ALL-UNNAMED',
+ '--add-opens', 'java.base/sun.invoke.util=ALL-UNNAMED',
+ '--add-opens', 'java.base/java.util.concurrent.locks=ALL-UNNAMED',
+ '--add-opens', 'java.base/java.security=ALL-UNNAMED',
+ '--add-opens', 'java.base/sun.reflect.annotation=ALL-UNNAMED',
+ '--add-opens', 'java.base/java.text=ALL-UNNAMED',
+ '--add-opens', 'java.base/java.nio.charset=ALL-UNNAMED',
+ '--add-opens', 'java.base/sun.reflect.generics.reflectiveObjects=ALL-UNNAMED',
+ '--add-opens', 'java.management/com.sun.jmx.mbeanserver=ALL-UNNAMED',
+ '--add-opens', 'java.management/javax.management=ALL-UNNAMED',
+ '--add-opens', 'java.base/java.util.regex=ALL-UNNAMED',
+ '--add-opens', 'java.base/sun.util.locale=ALL-UNNAMED',
+ '--add-opens', 'java.base/jdk.internal.math=ALL-UNNAMED',
+ '--add-opens', 'java.xml/com.sun.xml.internal.stream.util=ALL-UNNAMED',
+ '--add-opens', 'java.base/java.time=ALL-UNNAMED',
+ '--add-opens', 'java.base/sun.reflect.generics.factory=ALL-UNNAMED',
+ '--add-opens', 'java.base/java.nio.channels=ALL-UNNAMED',
+ '--add-opens', 'java.base/sun.security.util=ALL-UNNAMED',
+ '--add-opens', 'java.base/java.time.zone=ALL-UNNAMED',
+ '--add-opens', 'java.base/sun.reflect.generics.scope=ALL-UNNAMED',
+ '--add-opens', 'java.base/sun.reflect.generics.tree=ALL-UNNAMED',
+ '--add-opens', 'java.management/com.sun.jmx.interceptor=ALL-UNNAMED',
+ '--add-opens', 'java.management/javax.management.openmbean=ALL-UNNAMED',
+ '--add-opens', 'java.management/sun.management=ALL-UNNAMED',
+ '--add-opens', 'jdk.management/com.sun.management.internal=ALL-UNNAMED',
+ '--add-opens', 'jdk.management.jfr/jdk.management.jfr=ALL-UNNAMED',
+ '--add-opens', 'jdk.jfr/jdk.jfr.internal.management=ALL-UNNAMED',
+ '--add-opens', 'java.base/jdk.internal.module=ALL-UNNAMED',
+ '--add-opens', 'java.base/java.lang.module=ALL-UNNAMED',
+ '--add-opens', 'java.security.jgss/sun.security.krb5=ALL-UNNAMED',
+ '--add-opens', 'java.prefs/java.util.prefs=ALL-UNNAMED', // jacoco
+
+ '-Dio.netty.tryReflectionSetAccessible=true'
+ ]
+ }
+test {
+ jvmArgs = [
+ '--add-opens', 'java.base/java.lang=ALL-UNNAMED',
+ '--add-opens', 'java.base/java.lang.invoke=ALL-UNNAMED',
+ '--add-opens', 'java.base/java.lang.reflect=ALL-UNNAMED',
+ '--add-opens', 'java.base/java.io=ALL-UNNAMED',
+ '--add-opens', 'java.base/java.net=ALL-UNNAMED',
+ '--add-opens', 'java.base/java.nio=ALL-UNNAMED',
+ '--add-opens', 'java.base/java.util=ALL-UNNAMED',
+ '--add-opens', 'java.base/java.util.concurrent=ALL-UNNAMED',
+ '--add-opens', 'java.base/java.util.concurrent.atomic=ALL-UNNAMED',
+ '--add-opens', 'java.base/sun.nio.ch=ALL-UNNAMED',
+ '--add-opens', 'java.base/sun.nio.cs=ALL-UNNAMED',
+ '--add-opens', 'java.base/sun.security.action=ALL-UNNAMED',
+ '--add-opens', 'java.base/sun.util.calendar=ALL-UNNAMED',
+ '--add-opens', 'java.base/sun.nio.fs=ALL-UNNAMED',
+ '--add-opens', 'java.base/java.nio.channels.spi=ALL-UNNAMED',
+ '--add-opens', 'java.base/jdk.internal.ref=ALL-UNNAMED',
+ '--add-opens', 'java.base/java.lang.ref=ALL-UNNAMED',
+ '--add-opens', 'java.base/java.util.zip=ALL-UNNAMED',
+ '--add-opens', 'java.base/java.util.jar=ALL-UNNAMED',
+ '--add-opens', 'java.base/java.nio.file.attribute=ALL-UNNAMED',
+ '--add-opens', 'java.base/jdk.internal.loader=ALL-UNNAMED',
+ '--add-opens', 'java.base/sun.net.www.protocol.jar=ALL-UNNAMED',
+ '--add-opens', 'java.base/sun.invoke.util=ALL-UNNAMED',
+ '--add-opens', 'java.base/java.util.concurrent.locks=ALL-UNNAMED',
+ '--add-opens', 'java.base/java.security=ALL-UNNAMED',
+ '--add-opens', 'java.base/sun.reflect.annotation=ALL-UNNAMED',
+ '--add-opens', 'java.base/java.text=ALL-UNNAMED',
+ '--add-opens', 'java.base/java.nio.charset=ALL-UNNAMED',
+ '--add-opens', 'java.base/sun.reflect.generics.reflectiveObjects=ALL-UNNAMED',
+ '--add-opens', 'java.management/com.sun.jmx.mbeanserver=ALL-UNNAMED',
+ '--add-opens', 'java.management/javax.management=ALL-UNNAMED',
+ '--add-opens', 'java.base/java.util.regex=ALL-UNNAMED',
+ '--add-opens', 'java.base/sun.util.locale=ALL-UNNAMED',
+ '--add-opens', 'java.base/jdk.internal.math=ALL-UNNAMED',
+ '--add-opens', 'java.xml/com.sun.xml.internal.stream.util=ALL-UNNAMED',
+ '--add-opens', 'java.base/java.time=ALL-UNNAMED',
+ '--add-opens', 'java.base/sun.reflect.generics.factory=ALL-UNNAMED',
+ '--add-opens', 'java.base/java.nio.channels=ALL-UNNAMED',
+ '--add-opens', 'java.base/sun.security.util=ALL-UNNAMED',
+ '--add-opens', 'java.base/java.time.zone=ALL-UNNAMED',
+ '--add-opens', 'java.base/sun.reflect.generics.scope=ALL-UNNAMED',
+ '--add-opens', 'java.base/sun.reflect.generics.tree=ALL-UNNAMED',
+ '--add-opens', 'java.management/com.sun.jmx.interceptor=ALL-UNNAMED',
+ '--add-opens', 'java.management/javax.management.openmbean=ALL-UNNAMED',
+ '--add-opens', 'java.management/sun.management=ALL-UNNAMED',
+ '--add-opens', 'jdk.management/com.sun.management.internal=ALL-UNNAMED',
+ '--add-opens', 'jdk.management.jfr/jdk.management.jfr=ALL-UNNAMED',
+ '--add-opens', 'jdk.jfr/jdk.jfr.internal.management=ALL-UNNAMED',
+ '--add-opens', 'java.base/jdk.internal.module=ALL-UNNAMED',
+ '--add-opens', 'java.base/java.lang.module=ALL-UNNAMED',
+ '--add-opens', 'java.security.jgss/sun.security.krb5=ALL-UNNAMED',
+ '--add-opens', 'java.prefs/java.util.prefs=ALL-UNNAMED', // jacoco
+
+ '-Dio.netty.tryReflectionSetAccessible=true'
+ ]
+}
+
//add gatk launcher script to the jar as a resource
processResources {
from("gatk")
@@ -390,8 +508,8 @@ processTestResources {
include "org/broadinstitute/hellbender/utils/io/*"
}
-sourceCompatibility = 1.8
-targetCompatibility = 1.8
+sourceCompatibility = 1.17
+targetCompatibility = 1.17
def createSymlinks(archivePath, symlinkLocation) {
exec {
@@ -428,7 +546,7 @@ tasks.withType(Jar) {
}
wrapper {
- gradleVersion = '7.3.2'
+ gradleVersion = '7.5.1'
}
tasks.withType(ShadowJar) {
@@ -487,6 +605,7 @@ task shadowTestClassJar(type: ShadowJar){
// A minimal jar that only contains the extra dependencies needed for running the tests
task shadowTestJar(type: ShadowJar){
+ dependsOn 'compileTestUtilsJava', 'processTestUtilsResources'
group = "Shadow"
description = " A minimal jar that only contains the extra dependencies needed for running the tests that arent packaged in the main shadow jar"
from {
@@ -498,12 +617,13 @@ task shadowTestJar(type: ShadowJar){
}
task collectBundleIntoDir(type: Copy) {
- dependsOn shadowJar, sparkJar, 'condaEnvironmentDefinition', 'gatkTabComplete', 'gatkDoc'
+ //dependsOn shadowJar, sparkJar, 'condaEnvironmentDefinition', 'gatkTabComplete', 'gatkDoc'
+ dependsOn shadowJar, sparkJar, 'condaEnvironmentDefinition'
doFirst {
assert file("gatk").exists()
assert file("README.md").exists()
- assert file("$docBuildDir/tabCompletion/gatk-completion.sh").exists()
+ //assert file("$docBuildDir/tabCompletion/gatk-completion.sh").exists()
assert file("src/main/resources/org/broadinstitute/hellbender/utils/config/GATKConfig.properties").exists()
}
@@ -511,8 +631,8 @@ task collectBundleIntoDir(type: Copy) {
from(sparkJar.archivePath)
from("gatk")
from("README.md")
- from("$docBuildDir/tabCompletion/gatk-completion.sh")
- from("$docBuildDir/gatkDoc", { into("gatkdoc") })
+ //from("$docBuildDir/tabCompletion/gatk-completion.sh")
+ //from("$docBuildDir/gatkDoc", { into("gatkdoc") })
from("src/main/resources/org/broadinstitute/hellbender/utils/config/GATKConfig.properties") {
rename 'GATKConfig.properties', 'GATKConfig.EXAMPLE.properties'
@@ -619,36 +739,36 @@ task testUtilsJar(type: Jar){
from sourceSets.testUtils.output
}
-tasks.withType(Javadoc) {
- // do this for all javadoc tasks, including gatkDoc
- options.addStringOption('Xdoclint:none')
- options.addStringOption('encoding', 'UTF-8')
-}
-
-javadoc {
- // This is a hack to disable the java 8 default javadoc lint until we fix the html formatting
- // We only want to do this for the javadoc task, not gatkDoc
- options.addStringOption('Xdoclint:none', '-quiet')
- source = sourceSets.main.allJava + files(configurations.externalSourceConfiguration.collect { zipTree(it) })
- include '**/*.java'
-}
-
-
-task testUtilsJavadoc(type: Javadoc) {
- // This is a hack to disable the java 8 default javadoc lint until we fix the html formatting
- // We only want to do this for the javadoc task, not gatkDoc
- options.addStringOption('Xdoclint:none', '-quiet')
- source = sourceSets.testUtils.allJava
- classpath = sourceSets.testUtils.runtimeClasspath
- destinationDir = file("$docBuildDir/testUtilsJavadoc")
- include '**/*.java'
-}
-
-task testUtilsJavadocJar(type: Jar, dependsOn: testUtilsJavadoc){
- archiveBaseName = "$project.name-test-utils"
- archiveClassifier = 'javadoc'
- from "$docBuildDir/testUtilsJavadoc"
-}
+//tasks.withType(Javadoc) {
+// // do this for all javadoc tasks, including gatkDoc
+// options.addStringOption('Xdoclint:none')
+// options.addStringOption('encoding', 'UTF-8')
+//}
+//
+//javadoc {
+// // This is a hack to disable the java 8 default javadoc lint until we fix the html formatting
+// // We only want to do this for the javadoc task, not gatkDoc
+// options.addStringOption('Xdoclint:none', '-quiet')
+// source = sourceSets.main.allJava + files(configurations.externalSourceConfiguration.collect { zipTree(it) })
+// include '**/*.java'
+//}
+//
+//
+//task testUtilsJavadoc(type: Javadoc) {
+// // This is a hack to disable the java 8 default javadoc lint until we fix the html formatting
+// // We only want to do this for the javadoc task, not gatkDoc
+// options.addStringOption('Xdoclint:none', '-quiet')
+// source = sourceSets.testUtils.allJava
+// classpath = sourceSets.testUtils.runtimeClasspath
+// destinationDir = file("$docBuildDir/testUtilsJavadoc")
+// include '**/*.java'
+//}
+//
+//task testUtilsJavadocJar(type: Jar, dependsOn: testUtilsJavadoc){
+// archiveBaseName = "$project.name-test-utils"
+// archiveClassifier = 'javadoc'
+// from "$docBuildDir/testUtilsJavadoc"
+//}
task testUtilsSourcesJar(type: Jar){
archiveBaseName = "$project.name-test-utils"
@@ -657,244 +777,244 @@ task testUtilsSourcesJar(type: Jar){
}
// Generate GATK Online Doc
-task gatkDoc(type: Javadoc, dependsOn: classes) {
- final File gatkDocDir = new File("$docBuildDir/gatkdoc")
- doFirst {
- // make sure the output folder exists or we can create it
- if (!gatkDocDir.exists() && !gatkDocDir.mkdirs()) {
- throw new GradleException(String.format("Failure creating folder (%s) for GATK doc output in task (%s)",
- gatkDocDir.getAbsolutePath(),
- it.name));
- }
- copy {
- from('src/main/resources/org/broadinstitute/hellbender/utils/helpTemplates')
- include 'gatkDoc.css'
- into gatkDocDir
- }
- }
- // Include the Picard source jar, which contains various .R, .sh, .css, .html, .xml and .MF files and
- // other resources, but we only want the files that javadoc can handle, so just take the .java files.
- source = sourceSets.main.allJava + files(configurations.externalSourceConfiguration.collect { zipTree(it) })
- include '**/*.java'
-
- // The gatkDoc process instantiates any documented feature classes, so to run it we need the entire
- // runtime classpath, as well as jdk javadoc files such as tools.jar, where com.sun.javadoc lives.
- classpath = sourceSets.main.runtimeClasspath + javadocJDKFiles
- options.docletpath = classpath.asType(List)
- options.doclet = "org.broadinstitute.hellbender.utils.help.GATKHelpDoclet"
-
- //gradle 6.x+ defaults to setting this true which breaks the barclay doclet
- options.noTimestamp(false)
-
- outputs.dir(gatkDocDir)
- options.destinationDirectory(gatkDocDir)
-
- options.addStringOption("settings-dir", "src/main/resources/org/broadinstitute/hellbender/utils/helpTemplates");
- if (project.hasProperty('phpDoc')) {
- // use -PphpDoc to generate .php file extensions, otherwise rely on default of .html
- final String phpExtension = "php"
- options.addStringOption("output-file-extension", phpExtension)
- options.addStringOption("index-file-extension", phpExtension)
- }
- options.addStringOption("absolute-version", getVersion())
- options.addStringOption("build-timestamp", ZonedDateTime.now().format(DateTimeFormatter.RFC_1123_DATE_TIME))
-}
-
-// Generate GATK Bash Tab Completion File
-task gatkTabComplete(type: Javadoc, dependsOn: classes) {
- final File tabCompletionDir = new File("$docBuildDir/tabCompletion")
- doFirst {
- // make sure the output folder exists or we can create it
- if (!tabCompletionDir.exists() && !tabCompletionDir.mkdirs()) {
- throw new GradleException(String.format("Failure creating folder (%s) for GATK tab completion output in task (%s)",
- tabCompletionDir.getAbsolutePath(),
- it.name));
- }
- }
- // Include the Picard source jar, which contains various .R, .sh, .css, .html, .xml and .MF files and
- // other resources, but we only want the files that javadoc can handle, so just take the .java files.
- source = sourceSets.main.allJava + files(configurations.externalSourceConfiguration.collect { zipTree(it) })
- include '**/*.java'
-
- // The gatkDoc process instantiates any documented feature classes, so to run it we need the entire
- // runtime classpath, as well as jdk javadoc files such as tools.jar, where com.sun.javadoc lives, and Picard.
- classpath = sourceSets.main.runtimeClasspath + javadocJDKFiles
-
- options.docletpath = classpath.asType(List)
- options.doclet = "org.broadinstitute.barclay.help.BashTabCompletionDoclet"
-
- //gradle 6.x+ defaults to setting this true which breaks the barclay doclet
- options.noTimestamp(false)
-
- outputs.dir(tabCompletionDir)
- options.destinationDirectory(tabCompletionDir)
-
- // This is a hack to work around a gross Gradle bug:
- options.addStringOption('use-default-templates', '-use-default-templates')
-
- options.addStringOption("output-file-extension", "sh")
- options.addStringOption("index-file-extension", "sh")
- options.addStringOption("absolute-version", getVersion())
- options.addStringOption("build-timestamp", ZonedDateTime.now().format(DateTimeFormatter.RFC_1123_DATE_TIME))
-
- options.addStringOption("caller-script-name", "gatk")
-
- options.addStringOption("caller-pre-legal-args", "--help --list --dry-run --java-options")
- options.addStringOption("caller-pre-arg-val-types", "null null null String")
- options.addStringOption("caller-pre-mutex-args", "--help;list,dry-run,java-options --list;help,dry-run,java-options")
- options.addStringOption("caller-pre-alias-args", "--help;-h")
- options.addStringOption("caller-pre-arg-min-occurs", "0 0 0 0")
- options.addStringOption("caller-pre-arg-max-occurs", "1 1 1 1")
-
- options.addStringOption("caller-post-legal-args", "--spark-runner --spark-master --cluster --dry-run --java-options --conf --driver-memory --driver-cores --executor-memory --executor-cores --num-executors")
- options.addStringOption("caller-post-arg-val-types", "String String String null String file int int int int int")
- options.addStringOption("caller-post-mutex-args", "")
- options.addStringOption("caller-post-alias-args", "")
- options.addStringOption("caller-post-arg-min-occurs", "0 0 0 0 0 0 0 0 0 0")
- options.addStringOption("caller-post-arg-max-occurs", "1 1 1 1 1 1 1 1 1 1")
-}
-
-def getWDLInputJSONTestFileNameFromWDLName(File wdlName) {
- String fileWithoutExt = wdlName.name.take(wdlName.name.lastIndexOf('.'))
- return new File (wdlName.getParentFile(), fileWithoutExt + "Inputs.json").getAbsolutePath()
-}
+//task gatkDoc(type: Javadoc, dependsOn: classes) {
+// final File gatkDocDir = new File("$docBuildDir/gatkdoc")
+// doFirst {
+// // make sure the output folder exists or we can create it
+// if (!gatkDocDir.exists() && !gatkDocDir.mkdirs()) {
+// throw new GradleException(String.format("Failure creating folder (%s) for GATK doc output in task (%s)",
+// gatkDocDir.getAbsolutePath(),
+// it.name));
+// }
+// copy {
+// from('src/main/resources/org/broadinstitute/hellbender/utils/helpTemplates')
+// include 'gatkDoc.css'
+// into gatkDocDir
+// }
+// }
+// // Include the Picard source jar, which contains various .R, .sh, .css, .html, .xml and .MF files and
+// // other resources, but we only want the files that javadoc can handle, so just take the .java files.
+// source = sourceSets.main.allJava + files(configurations.externalSourceConfiguration.collect { zipTree(it) })
+// include '**/*.java'
+//
+// // The gatkDoc process instantiates any documented feature classes, so to run it we need the entire
+// // runtime classpath, as well as jdk javadoc files such as tools.jar, where com.sun.javadoc lives.
+// classpath = sourceSets.main.runtimeClasspath + javadocJDKFiles
+// options.docletpath = classpath.asType(List)
+// options.doclet = "org.broadinstitute.hellbender.utils.help.GATKHelpDoclet"
+//
+// //gradle 6.x+ defaults to setting this true which breaks the barclay doclet
+// options.noTimestamp(false)
+//
+// outputs.dir(gatkDocDir)
+// options.destinationDirectory(gatkDocDir)
+//
+// options.addStringOption("settings-dir", "src/main/resources/org/broadinstitute/hellbender/utils/helpTemplates");
+// if (project.hasProperty('phpDoc')) {
+// // use -PphpDoc to generate .php file extensions, otherwise rely on default of .html
+// final String phpExtension = "php"
+// options.addStringOption("output-file-extension", phpExtension)
+// options.addStringOption("index-file-extension", phpExtension)
+// }
+// options.addStringOption("absolute-version", getVersion())
+// options.addStringOption("build-timestamp", ZonedDateTime.now().format(DateTimeFormatter.RFC_1123_DATE_TIME))
+//}
+//
+//// Generate GATK Bash Tab Completion File
+//task gatkTabComplete(type: Javadoc, dependsOn: classes) {
+// final File tabCompletionDir = new File("$docBuildDir/tabCompletion")
+// doFirst {
+// // make sure the output folder exists or we can create it
+// if (!tabCompletionDir.exists() && !tabCompletionDir.mkdirs()) {
+// throw new GradleException(String.format("Failure creating folder (%s) for GATK tab completion output in task (%s)",
+// tabCompletionDir.getAbsolutePath(),
+// it.name));
+// }
+// }
+// // Include the Picard source jar, which contains various .R, .sh, .css, .html, .xml and .MF files and
+// // other resources, but we only want the files that javadoc can handle, so just take the .java files.
+// source = sourceSets.main.allJava + files(configurations.externalSourceConfiguration.collect { zipTree(it) })
+// include '**/*.java'
+//
+// // The gatkDoc process instantiates any documented feature classes, so to run it we need the entire
+// // runtime classpath, as well as jdk javadoc files such as tools.jar, where com.sun.javadoc lives, and Picard.
+// classpath = sourceSets.main.runtimeClasspath + javadocJDKFiles
+//
+// options.docletpath = classpath.asType(List)
+// options.doclet = "org.broadinstitute.barclay.help.BashTabCompletionDoclet"
+//
+// //gradle 6.x+ defaults to setting this true which breaks the barclay doclet
+// options.noTimestamp(false)
+//
+// outputs.dir(tabCompletionDir)
+// options.destinationDirectory(tabCompletionDir)
+//
+// // This is a hack to work around a gross Gradle bug:
+// options.addStringOption('use-default-templates', '-use-default-templates')
+//
+// options.addStringOption("output-file-extension", "sh")
+// options.addStringOption("index-file-extension", "sh")
+// options.addStringOption("absolute-version", getVersion())
+// options.addStringOption("build-timestamp", ZonedDateTime.now().format(DateTimeFormatter.RFC_1123_DATE_TIME))
+//
+// options.addStringOption("caller-script-name", "gatk")
+//
+// options.addStringOption("caller-pre-legal-args", "--help --list --dry-run --java-options")
+// options.addStringOption("caller-pre-arg-val-types", "null null null String")
+// options.addStringOption("caller-pre-mutex-args", "--help;list,dry-run,java-options --list;help,dry-run,java-options")
+// options.addStringOption("caller-pre-alias-args", "--help;-h")
+// options.addStringOption("caller-pre-arg-min-occurs", "0 0 0 0")
+// options.addStringOption("caller-pre-arg-max-occurs", "1 1 1 1")
+//
+// options.addStringOption("caller-post-legal-args", "--spark-runner --spark-master --cluster --dry-run --java-options --conf --driver-memory --driver-cores --executor-memory --executor-cores --num-executors")
+// options.addStringOption("caller-post-arg-val-types", "String String String null String file int int int int int")
+// options.addStringOption("caller-post-mutex-args", "")
+// options.addStringOption("caller-post-alias-args", "")
+// options.addStringOption("caller-post-arg-min-occurs", "0 0 0 0 0 0 0 0 0 0")
+// options.addStringOption("caller-post-arg-max-occurs", "1 1 1 1 1 1 1 1 1 1")
+//}
+//
+//def getWDLInputJSONTestFileNameFromWDLName(File wdlName) {
+// String fileWithoutExt = wdlName.name.take(wdlName.name.lastIndexOf('.'))
+// return new File (wdlName.getParentFile(), fileWithoutExt + "Inputs.json").getAbsolutePath()
+//}
// Generate GATK Tool WDL
-task gatkWDLGen(type: Javadoc, dependsOn: classes) {
- final File gatkWDLDir = new File("$docBuildDir/wdlGen")
- outputs.dir(gatkWDLDir)
- doFirst {
- // make sure the output folder exists or we can create it
- if (!gatkWDLDir.exists() && !gatkWDLDir.mkdirs()) {
- throw new GradleException(String.format("Failure creating folder (%s) for GATK WDL output in task (%s)",
- gatkWDLDir.getAbsolutePath(),
- it.name));
- }
- copy {
- from('src/main/resources/org/broadinstitute/hellbender/utils/wdlTemplates/common.html')
- into gatkWDLDir
- }
- }
- source = sourceSets.main.allJava + files(configurations.externalSourceConfiguration.collect { zipTree(it) })
- include '**/*.java'
-
- // The gatkWDLGen process instantiates any documented feature classes, so to run it we need the entire
- // runtime classpath, as well as jdk javadoc files such as tools.jar, where com.sun.javadoc lives.
- classpath = sourceSets.main.runtimeClasspath + javadocJDKFiles
- options.docletpath = classpath.asType(List)
- options.doclet = "org.broadinstitute.hellbender.utils.help.GATKWDLDoclet"
-
- //gradle 6.x+ defaults to setting this true which breaks the barclay doclet
- options.noTimestamp(false)
-
- outputs.dir(gatkWDLDir)
- options.destinationDirectory(gatkWDLDir)
-
- options.addStringOption("settings-dir", "src/main/resources/org/broadinstitute/hellbender/utils/wdlTemplates");
- options.addStringOption("output-file-extension", "wdl")
- options.addStringOption("index-file-extension", "html")
-
- options.addStringOption("absolute-version", getVersion())
- options.addStringOption("build-timestamp", ZonedDateTime.now().format(DateTimeFormatter.RFC_1123_DATE_TIME))
-
- // the wdl doclet will populate the test JSON input files with the name of a dummy
- // file in this location, in order to satisfy cromwell's attempts to localize inputs and outputs
- options.addStringOption("build-dir", System.getenv("TRAVIS_BUILD_DIR") ?: new File(".").getAbsolutePath())
-}
-
-def execWDLValidation = { validateWDL ->
- println "Executing: $validateWDL"
- try {
- def retCode = validateWDL.execute().waitFor()
- if (retCode.intValue() != 0) {
- throw new GradleException("Execution of \"$validateWDL\" failed with exit code: $retCode.")
- }
- return retCode
- } catch (IOException e) {
- throw new GradleException("An IOException occurred while attempting to execute the command $validateWDL.")
- }
-}
-
-task gatkValidateScriptsWdl() {
- doFirst {
- // running this task requires a local cromwell installation, with environment variables CROMWELL_JAR,
- // WOMTOOL_JAR set to the jar locations
- if (System.getenv('CROMWELL_JAR') == null || System.getenv('WOMTOOL_JAR') == null) {
- throw new GradleException("Running this task requires the CROMWELL_JAR and WOMTOOL_JAR environment variables to be set")
- }
- }
-
- doLast {
- // Run the womtool validator on all WDL files in the 'scripts' directory
- final File wdlFolder = new File("scripts")
- def wdlFiles = fileTree(dir: wdlFolder).filter {
- f -> f.getAbsolutePath().endsWith(".wdl")
- }
- final womtoolLocation = System.getenv('WOMTOOL_JAR')
- wdlFiles.any() { wdlFile ->
- final validateWDLCommand = "java -jar $womtoolLocation validate $wdlFile"
- execWDLValidation(validateWDLCommand)
- }
- }
-}
-
-task gatkValidateGeneratedWdl(dependsOn: [gatkWDLGen, shadowJar]) {
- doFirst {
- // running this task requires a local cromwell installation, with environment variables CROMWELL_JAR,
- // WOMTOOL_JAR set to the jar locations
- if (System.getenv('CROMWELL_JAR') == null || System.getenv('WOMTOOL_JAR') == null) {
- throw new GradleException("Running this task requires the CROMWELL_JAR and WOMTOOL_JAR environment variables to be set")
- }
- }
-
- doLast {
- // first, run the womtool validator on WDL files in the 'docs/wdlGen' directory
- final File wdlGenFolder = new File("$docBuildDir/wdlGen")
- def wdlFiles = fileTree(dir: wdlGenFolder).filter {
- f -> !f.getAbsolutePath().endsWith(".html") && !f.getAbsolutePath().endsWith(".json")
- }
- final womtoolLocation = System.getenv('WOMTOOL_JAR')
- wdlFiles.any() { wdlFile ->
- final validateWDLCommand = "java -jar $womtoolLocation validate $wdlFile"
- execWDLValidation(validateWDLCommand)
- }
-
- // now execute the *AllArgs test wdls using cromwell
- wdlFiles = fileTree(dir: wdlGenFolder).filter {
- f -> f.getAbsolutePath().endsWith("AllArgsTest.wdl")
- }
-
- // the test JSON input file is populated by the WDL gen process with the name of this dummy file
- // to satisfy cromwell's attempt to de/localize input/output files
- def buildDir = System.getenv("TRAVIS_BUILD_DIR") ?: new File(".").getAbsolutePath()
- final dummyWDLTestFileName = "$buildDir/dummyWDLTestFile"
- final File dummyWDLTestFile = file(dummyWDLTestFileName)
- final cromwellLocation = System.getenv('CROMWELL_JAR')
- try {
- wdlFiles.any() { wdlFile ->
- final testInputJSON = getWDLInputJSONTestFileNameFromWDLName(wdlFile)
- final runWDLCommand = "java -jar $cromwellLocation run --inputs $testInputJSON $wdlFile"
- execWDLValidation("touch $dummyWDLTestFileName")
- execWDLValidation(runWDLCommand)
- }
- } finally {
- // delete the dummy test file and the 'cromwell-executions' directory left behind by cromwell
- dummyWDLTestFile.delete()
- file("$buildDir/cromwell-executions").deleteDir()
- file("$buildDir/cromwell-workflow-logs").deleteDir()
- }
- }
-}
+//task gatkWDLGen(type: Javadoc, dependsOn: classes) {
+// final File gatkWDLDir = new File("$docBuildDir/wdlGen")
+// outputs.dir(gatkWDLDir)
+// doFirst {
+// // make sure the output folder exists or we can create it
+// if (!gatkWDLDir.exists() && !gatkWDLDir.mkdirs()) {
+// throw new GradleException(String.format("Failure creating folder (%s) for GATK WDL output in task (%s)",
+// gatkWDLDir.getAbsolutePath(),
+// it.name));
+// }
+// copy {
+// from('src/main/resources/org/broadinstitute/hellbender/utils/wdlTemplates/common.html')
+// into gatkWDLDir
+// }
+// }
+// source = sourceSets.main.allJava + files(configurations.externalSourceConfiguration.collect { zipTree(it) })
+// include '**/*.java'
+//
+// // The gatkWDLGen process instantiates any documented feature classes, so to run it we need the entire
+// // runtime classpath, as well as jdk javadoc files such as tools.jar, where com.sun.javadoc lives.
+// classpath = sourceSets.main.runtimeClasspath + javadocJDKFiles
+// options.docletpath = classpath.asType(List)
+// options.doclet = "org.broadinstitute.hellbender.utils.help.GATKWDLDoclet"
+//
+// //gradle 6.x+ defaults to setting this true which breaks the barclay doclet
+// options.noTimestamp(false)
+//
+// outputs.dir(gatkWDLDir)
+// options.destinationDirectory(gatkWDLDir)
+//
+// options.addStringOption("settings-dir", "src/main/resources/org/broadinstitute/hellbender/utils/wdlTemplates");
+// options.addStringOption("output-file-extension", "wdl")
+// options.addStringOption("index-file-extension", "html")
+//
+// options.addStringOption("absolute-version", getVersion())
+// options.addStringOption("build-timestamp", ZonedDateTime.now().format(DateTimeFormatter.RFC_1123_DATE_TIME))
+//
+// // the wdl doclet will populate the test JSON input files with the name of a dummy
+// // file in this location, in order to satisfy cromwell's attempts to localize inputs and outputs
+// options.addStringOption("build-dir", System.getenv("TRAVIS_BUILD_DIR") ?: new File(".").getAbsolutePath())
+//}
+//
+//def execWDLValidation = { validateWDL ->
+// println "Executing: $validateWDL"
+// try {
+// def retCode = validateWDL.execute().waitFor()
+// if (retCode.intValue() != 0) {
+// throw new GradleException("Execution of \"$validateWDL\" failed with exit code: $retCode.")
+// }
+// return retCode
+// } catch (IOException e) {
+// throw new GradleException("An IOException occurred while attempting to execute the command $validateWDL.")
+// }
+//}
+//
+//task gatkValidateScriptsWdl() {
+// doFirst {
+// // running this task requires a local cromwell installation, with environment variables CROMWELL_JAR,
+// // WOMTOOL_JAR set to the jar locations
+// if (System.getenv('CROMWELL_JAR') == null || System.getenv('WOMTOOL_JAR') == null) {
+// throw new GradleException("Running this task requires the CROMWELL_JAR and WOMTOOL_JAR environment variables to be set")
+// }
+// }
+//
+// doLast {
+// // Run the womtool validator on all WDL files in the 'scripts' directory
+// final File wdlFolder = new File("scripts")
+// def wdlFiles = fileTree(dir: wdlFolder).filter {
+// f -> f.getAbsolutePath().endsWith(".wdl")
+// }
+// final womtoolLocation = System.getenv('WOMTOOL_JAR')
+// wdlFiles.any() { wdlFile ->
+// final validateWDLCommand = "java -jar $womtoolLocation validate $wdlFile"
+// execWDLValidation(validateWDLCommand)
+// }
+// }
+//}
+//
+//task gatkValidateGeneratedWdl(dependsOn: [gatkWDLGen, shadowJar]) {
+// doFirst {
+// // running this task requires a local cromwell installation, with environment variables CROMWELL_JAR,
+// // WOMTOOL_JAR set to the jar locations
+// if (System.getenv('CROMWELL_JAR') == null || System.getenv('WOMTOOL_JAR') == null) {
+// throw new GradleException("Running this task requires the CROMWELL_JAR and WOMTOOL_JAR environment variables to be set")
+// }
+// }
+//
+// doLast {
+// // first, run the womtool validator on WDL files in the 'docs/wdlGen' directory
+// final File wdlGenFolder = new File("$docBuildDir/wdlGen")
+// def wdlFiles = fileTree(dir: wdlGenFolder).filter {
+// f -> !f.getAbsolutePath().endsWith(".html") && !f.getAbsolutePath().endsWith(".json")
+// }
+// final womtoolLocation = System.getenv('WOMTOOL_JAR')
+// wdlFiles.any() { wdlFile ->
+// final validateWDLCommand = "java -jar $womtoolLocation validate $wdlFile"
+// execWDLValidation(validateWDLCommand)
+// }
+//
+// // now execute the *AllArgs test wdls using cromwell
+// wdlFiles = fileTree(dir: wdlGenFolder).filter {
+// f -> f.getAbsolutePath().endsWith("AllArgsTest.wdl")
+// }
+//
+// // the test JSON input file is populated by the WDL gen process with the name of this dummy file
+// // to satisfy cromwell's attempt to de/localize input/output files
+// def buildDir = System.getenv("TRAVIS_BUILD_DIR") ?: new File(".").getAbsolutePath()
+// final dummyWDLTestFileName = "$buildDir/dummyWDLTestFile"
+// final File dummyWDLTestFile = file(dummyWDLTestFileName)
+// final cromwellLocation = System.getenv('CROMWELL_JAR')
+// try {
+// wdlFiles.any() { wdlFile ->
+// final testInputJSON = getWDLInputJSONTestFileNameFromWDLName(wdlFile)
+// final runWDLCommand = "java -jar $cromwellLocation run --inputs $testInputJSON $wdlFile"
+// execWDLValidation("touch $dummyWDLTestFileName")
+// execWDLValidation(runWDLCommand)
+// }
+// } finally {
+// // delete the dummy test file and the 'cromwell-executions' directory left behind by cromwell
+// dummyWDLTestFile.delete()
+// file("$buildDir/cromwell-executions").deleteDir()
+// file("$buildDir/cromwell-workflow-logs").deleteDir()
+// }
+// }
+//}
/**
*This specifies what artifacts will be built and uploaded when performing a maven upload.
*/
artifacts {
- archives javadocJar
+ //archives javadocJar
archives sourcesJar
archives testUtilsJar
- archives testUtilsJavadocJar
+ //archives testUtilsJavadocJar
archives testUtilsSourcesJar
}
@@ -963,7 +1083,7 @@ publishing {
artifact testUtilsJar
artifact testUtilsSourcesJar
- artifact testUtilsJavadocJar
+ //artifact testUtilsJavadocJar
}
}
diff --git a/gatk b/gatk
index 45559912047..6d515be8acb 100755
--- a/gatk
+++ b/gatk
@@ -37,12 +37,120 @@ EXTRA_JAVA_OPTIONS_SPARK= "-DGATK_STACKTRACE_ON_USER_EXCEPTION=true " \
"-Dsamjdk.use_async_io_read_samtools=false " \
"-Dsamjdk.use_async_io_write_samtools=false " \
"-Dsamjdk.use_async_io_write_tribble=false " \
- "-Dsamjdk.compression_level=2 "
-
-PACKAGED_LOCAL_JAR_OPTIONS= ["-Dsamjdk.use_async_io_read_samtools=false",
+ "-Dsamjdk.compression_level=2 " \
+ "--add-opens", "java.base/java.lang=ALL-UNNAMED " \
+ "--add-opens", "java.base/java.lang.invoke=ALL-UNNAMED " \
+ "--add-opens", "java.base/java.lang.reflect=ALL-UNNAMED " \
+ "--add-opens", "java.base/java.io=ALL-UNNAMED " \
+ "--add-opens", "java.base/java.net=ALL-UNNAMED " \
+ "--add-opens", "java.base/java.nio=ALL-UNNAMED " \
+ "--add-opens", "java.base/java.util=ALL-UNNAMED " \
+ "--add-opens", "java.base/java.util.concurrent=ALL-UNNAMED " \
+ "--add-opens", "java.base/java.util.concurrent.atomic=ALL-UNNAMED " \
+ "--add-opens", "java.base/sun.nio.ch=ALL-UNNAMED " \
+ "--add-opens", "java.base/sun.nio.cs=ALL-UNNAMED " \
+ "--add-opens", "java.base/sun.security.action=ALL-UNNAMED " \
+ "--add-opens", "java.base/sun.util.calendar=ALL-UNNAMED " \
+ "--add-opens", "java.base/sun.nio.fs=ALL-UNNAMED " \
+ "--add-opens", "java.base/java.nio.channels.spi=ALL-UNNAMED " \
+ "--add-opens", "java.base/jdk.internal.ref=ALL-UNNAMED " \
+ "--add-opens", "java.base/java.lang.ref=ALL-UNNAMED " \
+ "--add-opens", "java.base/java.util.zip=ALL-UNNAMED " \
+ "--add-opens", "java.base/java.util.jar=ALL-UNNAMED " \
+ "--add-opens", "java.base/java.nio.file.attribute=ALL-UNNAMED " \
+ "--add-opens", "java.base/jdk.internal.loader=ALL-UNNAMED " \
+ "--add-opens", "java.base/sun.net.www.protocol.jar=ALL-UNNAMED " \
+ "--add-opens", "java.base/sun.invoke.util=ALL-UNNAMED " \
+ "--add-opens", "java.base/java.util.concurrent.locks=ALL-UNNAMED " \
+ "--add-opens", "java.base/java.security=ALL-UNNAMED " \
+ "--add-opens", "java.base/sun.reflect.annotation=ALL-UNNAMED " \
+ "--add-opens", "java.base/java.text=ALL-UNNAMED " \
+ "--add-opens", "java.base/java.nio.charset=ALL-UNNAMED " \
+ "--add-opens", "java.base/sun.reflect.generics.reflectiveObjects=ALL-UNNAMED " \
+ "--add-opens", "java.management/com.sun.jmx.mbeanserver=ALL-UNNAMED " \
+ "--add-opens", "java.management/javax.management=ALL-UNNAMED " \
+ "--add-opens", "java.base/java.util.regex=ALL-UNNAMED " \
+ "--add-opens", "java.base/sun.util.locale=ALL-UNNAMED " \
+ "--add-opens", "java.base/jdk.internal.math=ALL-UNNAMED " \
+ "--add-opens", "java.xml/com.sun.xml.internal.stream.util=ALL-UNNAMED " \
+ "--add-opens", "java.base/java.time=ALL-UNNAMED " \
+ "--add-opens", "java.base/sun.reflect.generics.factory=ALL-UNNAMED' " \
+ "--add-opens", "java.base/java.nio.channels=ALL-UNNAMED " \
+ "--add-opens", "java.base/sun.security.util=ALL-UNNAMED " \
+ "--add-opens", "java.base/java.time.zone=ALL-UNNAMED " \
+ "--add-opens", "java.base/sun.reflect.generics.scope=ALL-UNNAMED " \
+ "--add-opens", "java.base/sun.reflect.generics.tree=ALL-UNNAMED " \
+ "--add-opens", "java.management/com.sun.jmx.interceptor=ALL-UNNAMED " \
+ "--add-opens", "java.management/javax.management.openmbean=ALL-UNNAMED " \
+ "--add-opens", "java.management/sun.management=ALL-UNNAMED " \
+ "--add-opens", "jdk.management/com.sun.management.internal=ALL-UNNAMED " \
+ "--add-opens", "jdk.management.jfr/jdk.management.jfr=ALL-UNNAMED " \
+ "--add-opens", "jdk.jfr/jdk.jfr.internal.management=ALL-UNNAMED " \
+ "--add-opens", "java.base/jdk.internal.module=ALL-UNNAMED " \
+ "--add-opens", "java.base/java.lang.module=ALL-UNNAMED " \
+ "--add-opens", "java.security.jgss/sun.security.krb5=ALL-UNNAMED " \
+ "--add-opens", "java.prefs/java.util.prefs=ALL-UNNAMED " \
+ "-Dio.netty.tryReflectionSetAccessible=true"
+
+PACKAGED_LOCAL_JAR_OPTIONS= [
+ "-Dsamjdk.use_async_io_read_samtools=false",
"-Dsamjdk.use_async_io_write_samtools=true",
"-Dsamjdk.use_async_io_write_tribble=false",
- "-Dsamjdk.compression_level=2"]
+ "-Dsamjdk.compression_level=2",
+ "--add-opens", "java.base/java.lang=ALL-UNNAMED",
+ "--add-opens", "java.base/java.lang.invoke=ALL-UNNAMED",
+ "--add-opens", "java.base/java.lang.reflect=ALL-UNNAMED",
+ "--add-opens", "java.base/java.io=ALL-UNNAMED",
+ "--add-opens", "java.base/java.net=ALL-UNNAMED",
+ "--add-opens", "java.base/java.nio=ALL-UNNAMED",
+ "--add-opens", "java.base/java.util=ALL-UNNAMED",
+ "--add-opens", "java.base/java.util.concurrent=ALL-UNNAMED",
+ "--add-opens", "java.base/java.util.concurrent.atomic=ALL-UNNAMED ",
+ "--add-opens", "java.base/sun.nio.ch=ALL-UNNAMED",
+ "--add-opens", "java.base/sun.nio.cs=ALL-UNNAMED",
+ "--add-opens", "java.base/sun.security.action=ALL-UNNAMED",
+ "--add-opens", "java.base/sun.util.calendar=ALL-UNNAMED",
+ "--add-opens", "java.base/sun.nio.fs=ALL-UNNAMED",
+ "--add-opens", "java.base/java.nio.channels.spi=ALL-UNNAMED",
+ "--add-opens", "java.base/jdk.internal.ref=ALL-UNNAMED",
+ "--add-opens", "java.base/java.lang.ref=ALL-UNNAMED",
+ "--add-opens", "java.base/java.util.zip=ALL-UNNAMED",
+ "--add-opens", "java.base/java.util.jar=ALL-UNNAMED",
+ "--add-opens", "java.base/java.nio.file.attribute=ALL-UNNAMED",
+ "--add-opens", "java.base/jdk.internal.loader=ALL-UNNAMED",
+ "--add-opens", "java.base/sun.net.www.protocol.jar=ALL-UNNAMED",
+ "--add-opens", "java.base/sun.invoke.util=ALL-UNNAMED",
+ "--add-opens", "java.base/java.util.concurrent.locks=ALL-UNNAMED",
+ "--add-opens", "java.base/java.security=ALL-UNNAMED",
+ "--add-opens", "java.base/sun.reflect.annotation=ALL-UNNAMED",
+ "--add-opens", "java.base/java.text=ALL-UNNAMED",
+ "--add-opens", "java.base/java.nio.charset=ALL-UNNAMED",
+ "--add-opens", "java.base/sun.reflect.generics.reflectiveObjects=ALL-UNNAMED",
+ "--add-opens", "java.management/com.sun.jmx.mbeanserver=ALL-UNNAMED",
+ "--add-opens", "java.management/javax.management=ALL-UNNAMED",
+ "--add-opens", "java.base/java.util.regex=ALL-UNNAMED",
+ "--add-opens", "java.base/sun.util.locale=ALL-UNNAMED",
+ "--add-opens", "java.base/jdk.internal.math=ALL-UNNAMED",
+ "--add-opens", "java.xml/com.sun.xml.internal.stream.util=ALL-UNNAMED",
+ "--add-opens", "java.base/java.time=ALL-UNNAMED",
+ "--add-opens", "java.base/sun.reflect.generics.factory=ALL-UNNAMED",
+ "--add-opens", "java.base/java.nio.channels=ALL-UNNAMED",
+ "--add-opens", "java.base/sun.security.util=ALL-UNNAMED",
+ "--add-opens", "java.base/java.time.zone=ALL-UNNAMED",
+ "--add-opens", "java.base/sun.reflect.generics.scope=ALL-UNNAMED",
+ "--add-opens", "java.base/sun.reflect.generics.tree=ALL-UNNAMED",
+ "--add-opens", "java.management/com.sun.jmx.interceptor=ALL-UNNAMED",
+ "--add-opens", "java.management/javax.management.openmbean=ALL-UNNAMED",
+ "--add-opens", "java.management/sun.management=ALL-UNNAMED",
+ "--add-opens", "jdk.management/com.sun.management.internal=ALL-UNNAMED",
+ "--add-opens", "jdk.management.jfr/jdk.management.jfr=ALL-UNNAMED",
+ "--add-opens", "jdk.jfr/jdk.jfr.internal.management=ALL-UNNAMED",
+ "--add-opens", "java.base/jdk.internal.module=ALL-UNNAMED",
+ "--add-opens", "java.base/java.lang.module=ALL-UNNAMED",
+ "--add-opens", "java.security.jgss/sun.security.krb5=ALL-UNNAMED",
+ "--add-opens", "java.prefs/java.util.prefs=ALL-UNNAMED",
+ "-Dio.netty.tryReflectionSetAccessible=true"
+]
DEFAULT_SPARK_ARGS_PREFIX = '--conf'
DEFAULT_SPARK_ARGS = {
diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar
index e708b1c023e..41d9927a4d4 100644
Binary files a/gradle/wrapper/gradle-wrapper.jar and b/gradle/wrapper/gradle-wrapper.jar differ
diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties
index d2880ba8000..ae04661ee73 100644
--- a/gradle/wrapper/gradle-wrapper.properties
+++ b/gradle/wrapper/gradle-wrapper.properties
@@ -1,5 +1,5 @@
distributionBase=GRADLE_USER_HOME
distributionPath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-7.3.2-bin.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-7.5.1-bin.zip
zipStoreBase=GRADLE_USER_HOME
zipStorePath=wrapper/dists
diff --git a/gradlew b/gradlew
index 4f906e0c811..1b6c787337f 100755
--- a/gradlew
+++ b/gradlew
@@ -1,7 +1,7 @@
-#!/usr/bin/env sh
+#!/bin/sh
#
-# Copyright 2015 the original author or authors.
+# Copyright © 2015-2021 the original authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -17,67 +17,101 @@
#
##############################################################################
-##
-## Gradle start up script for UN*X
-##
+#
+# Gradle start up script for POSIX generated by Gradle.
+#
+# Important for running:
+#
+# (1) You need a POSIX-compliant shell to run this script. If your /bin/sh is
+# noncompliant, but you have some other compliant shell such as ksh or
+# bash, then to run this script, type that shell name before the whole
+# command line, like:
+#
+# ksh Gradle
+#
+# Busybox and similar reduced shells will NOT work, because this script
+# requires all of these POSIX shell features:
+# * functions;
+# * expansions «$var», «${var}», «${var:-default}», «${var+SET}»,
+# «${var#prefix}», «${var%suffix}», and «$( cmd )»;
+# * compound commands having a testable exit status, especially «case»;
+# * various built-in commands including «command», «set», and «ulimit».
+#
+# Important for patching:
+#
+# (2) This script targets any POSIX shell, so it avoids extensions provided
+# by Bash, Ksh, etc; in particular arrays are avoided.
+#
+# The "traditional" practice of packing multiple parameters into a
+# space-separated string is a well documented source of bugs and security
+# problems, so this is (mostly) avoided, by progressively accumulating
+# options in "$@", and eventually passing that to Java.
+#
+# Where the inherited environment variables (DEFAULT_JVM_OPTS, JAVA_OPTS,
+# and GRADLE_OPTS) rely on word-splitting, this is performed explicitly;
+# see the in-line comments for details.
+#
+# There are tweaks for specific operating systems such as AIX, CygWin,
+# Darwin, MinGW, and NonStop.
+#
+# (3) This script is generated from the Groovy template
+# https://github.com/gradle/gradle/blob/master/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt
+# within the Gradle project.
+#
+# You can find Gradle at https://github.com/gradle/gradle/.
+#
##############################################################################
# Attempt to set APP_HOME
+
# Resolve links: $0 may be a link
-PRG="$0"
-# Need this for relative symlinks.
-while [ -h "$PRG" ] ; do
- ls=`ls -ld "$PRG"`
- link=`expr "$ls" : '.*-> \(.*\)$'`
- if expr "$link" : '/.*' > /dev/null; then
- PRG="$link"
- else
- PRG=`dirname "$PRG"`"/$link"
- fi
+app_path=$0
+
+# Need this for daisy-chained symlinks.
+while
+ APP_HOME=${app_path%"${app_path##*/}"} # leaves a trailing /; empty if no leading path
+ [ -h "$app_path" ]
+do
+ ls=$( ls -ld "$app_path" )
+ link=${ls#*' -> '}
+ case $link in #(
+ /*) app_path=$link ;; #(
+ *) app_path=$APP_HOME$link ;;
+ esac
done
-SAVED="`pwd`"
-cd "`dirname \"$PRG\"`/" >/dev/null
-APP_HOME="`pwd -P`"
-cd "$SAVED" >/dev/null
+
+APP_HOME=$( cd "${APP_HOME:-./}" && pwd -P ) || exit
APP_NAME="Gradle"
-APP_BASE_NAME=`basename "$0"`
+APP_BASE_NAME=${0##*/}
# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"'
# Use the maximum available, or set MAX_FD != -1 to use that value.
-MAX_FD="maximum"
+MAX_FD=maximum
warn () {
echo "$*"
-}
+} >&2
die () {
echo
echo "$*"
echo
exit 1
-}
+} >&2
# OS specific support (must be 'true' or 'false').
cygwin=false
msys=false
darwin=false
nonstop=false
-case "`uname`" in
- CYGWIN* )
- cygwin=true
- ;;
- Darwin* )
- darwin=true
- ;;
- MINGW* )
- msys=true
- ;;
- NONSTOP* )
- nonstop=true
- ;;
+case "$( uname )" in #(
+ CYGWIN* ) cygwin=true ;; #(
+ Darwin* ) darwin=true ;; #(
+ MSYS* | MINGW* ) msys=true ;; #(
+ NONSTOP* ) nonstop=true ;;
esac
CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar
@@ -87,9 +121,9 @@ CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar
if [ -n "$JAVA_HOME" ] ; then
if [ -x "$JAVA_HOME/jre/sh/java" ] ; then
# IBM's JDK on AIX uses strange locations for the executables
- JAVACMD="$JAVA_HOME/jre/sh/java"
+ JAVACMD=$JAVA_HOME/jre/sh/java
else
- JAVACMD="$JAVA_HOME/bin/java"
+ JAVACMD=$JAVA_HOME/bin/java
fi
if [ ! -x "$JAVACMD" ] ; then
die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME
@@ -98,7 +132,7 @@ Please set the JAVA_HOME variable in your environment to match the
location of your Java installation."
fi
else
- JAVACMD="java"
+ JAVACMD=java
which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
Please set the JAVA_HOME variable in your environment to match the
@@ -106,80 +140,95 @@ location of your Java installation."
fi
# Increase the maximum file descriptors if we can.
-if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then
- MAX_FD_LIMIT=`ulimit -H -n`
- if [ $? -eq 0 ] ; then
- if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then
- MAX_FD="$MAX_FD_LIMIT"
- fi
- ulimit -n $MAX_FD
- if [ $? -ne 0 ] ; then
- warn "Could not set maximum file descriptor limit: $MAX_FD"
- fi
- else
- warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT"
- fi
+if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then
+ case $MAX_FD in #(
+ max*)
+ MAX_FD=$( ulimit -H -n ) ||
+ warn "Could not query maximum file descriptor limit"
+ esac
+ case $MAX_FD in #(
+ '' | soft) :;; #(
+ *)
+ ulimit -n "$MAX_FD" ||
+ warn "Could not set maximum file descriptor limit to $MAX_FD"
+ esac
fi
-# For Darwin, add options to specify how the application appears in the dock
-if $darwin; then
- GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\""
-fi
+# Collect all arguments for the java command, stacking in reverse order:
+# * args from the command line
+# * the main class name
+# * -classpath
+# * -D...appname settings
+# * --module-path (only if needed)
+# * DEFAULT_JVM_OPTS, JAVA_OPTS, and GRADLE_OPTS environment variables.
# For Cygwin or MSYS, switch paths to Windows format before running java
-if [ "$cygwin" = "true" -o "$msys" = "true" ] ; then
- APP_HOME=`cygpath --path --mixed "$APP_HOME"`
- CLASSPATH=`cygpath --path --mixed "$CLASSPATH"`
-
- JAVACMD=`cygpath --unix "$JAVACMD"`
-
- # We build the pattern for arguments to be converted via cygpath
- ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null`
- SEP=""
- for dir in $ROOTDIRSRAW ; do
- ROOTDIRS="$ROOTDIRS$SEP$dir"
- SEP="|"
- done
- OURCYGPATTERN="(^($ROOTDIRS))"
- # Add a user-defined pattern to the cygpath arguments
- if [ "$GRADLE_CYGPATTERN" != "" ] ; then
- OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)"
- fi
+if "$cygwin" || "$msys" ; then
+ APP_HOME=$( cygpath --path --mixed "$APP_HOME" )
+ CLASSPATH=$( cygpath --path --mixed "$CLASSPATH" )
+
+ JAVACMD=$( cygpath --unix "$JAVACMD" )
+
# Now convert the arguments - kludge to limit ourselves to /bin/sh
- i=0
- for arg in "$@" ; do
- CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -`
- CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option
-
- if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition
- eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"`
- else
- eval `echo args$i`="\"$arg\""
+ for arg do
+ if
+ case $arg in #(
+ -*) false ;; # don't mess with options #(
+ /?*) t=${arg#/} t=/${t%%/*} # looks like a POSIX filepath
+ [ -e "$t" ] ;; #(
+ *) false ;;
+ esac
+ then
+ arg=$( cygpath --path --ignore --mixed "$arg" )
fi
- i=`expr $i + 1`
+ # Roll the args list around exactly as many times as the number of
+ # args, so each arg winds up back in the position where it started, but
+ # possibly modified.
+ #
+ # NB: a `for` loop captures its iteration list before it begins, so
+ # changing the positional parameters here affects neither the number of
+ # iterations, nor the values presented in `arg`.
+ shift # remove old arg
+ set -- "$@" "$arg" # push replacement arg
done
- case $i in
- 0) set -- ;;
- 1) set -- "$args0" ;;
- 2) set -- "$args0" "$args1" ;;
- 3) set -- "$args0" "$args1" "$args2" ;;
- 4) set -- "$args0" "$args1" "$args2" "$args3" ;;
- 5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;;
- 6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;;
- 7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;;
- 8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;;
- 9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;;
- esac
fi
-# Escape application args
-save () {
- for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done
- echo " "
-}
-APP_ARGS=`save "$@"`
+# Collect all arguments for the java command;
+# * $DEFAULT_JVM_OPTS, $JAVA_OPTS, and $GRADLE_OPTS can contain fragments of
+# shell script including quotes and variable substitutions, so put them in
+# double quotes to make sure that they get re-expanded; and
+# * put everything else in single quotes, so that it's not re-expanded.
+
+set -- \
+ "-Dorg.gradle.appname=$APP_BASE_NAME" \
+ -classpath "$CLASSPATH" \
+ org.gradle.wrapper.GradleWrapperMain \
+ "$@"
+
+# Use "xargs" to parse quoted args.
+#
+# With -n1 it outputs one arg per line, with the quotes and backslashes removed.
+#
+# In Bash we could simply go:
+#
+# readarray ARGS < <( xargs -n1 <<<"$var" ) &&
+# set -- "${ARGS[@]}" "$@"
+#
+# but POSIX shell has neither arrays nor command substitution, so instead we
+# post-process each arg (as a line of input to sed) to backslash-escape any
+# character that might be a shell metacharacter, then use eval to reverse
+# that process (while maintaining the separation between arguments), and wrap
+# the whole thing up as a single "set" statement.
+#
+# This will of course break if any of these variables contains a newline or
+# an unmatched quote.
+#
-# Collect all arguments for the java command, following the shell quoting and substitution rules
-eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS"
+eval "set -- $(
+ printf '%s\n' "$DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS" |
+ xargs -n1 |
+ sed ' s~[^-[:alnum:]+,./:=@_]~\\&~g; ' |
+ tr '\n' ' '
+ )" '"$@"'
exec "$JAVACMD" "$@"
diff --git a/scripts/docker/dockertest.gradle b/scripts/docker/dockertest.gradle
index 3623dc18b52..bf21192493f 100644
--- a/scripts/docker/dockertest.gradle
+++ b/scripts/docker/dockertest.gradle
@@ -36,9 +36,130 @@ task unpackTestJar(type: Copy){
}
}
+test {
+ jvmArgs = [
+ '--add-opens', 'java.base/java.lang=ALL-UNNAMED',
+ '--add-opens', 'java.base/java.lang.invoke=ALL-UNNAMED',
+ '--add-opens', 'java.base/java.lang.reflect=ALL-UNNAMED',
+ '--add-opens', 'java.base/java.io=ALL-UNNAMED',
+ '--add-opens', 'java.base/java.net=ALL-UNNAMED',
+ '--add-opens', 'java.base/java.nio=ALL-UNNAMED',
+ '--add-opens', 'java.base/java.util=ALL-UNNAMED',
+ '--add-opens', 'java.base/java.util.concurrent=ALL-UNNAMED',
+ '--add-opens', 'java.base/java.util.concurrent.atomic=ALL-UNNAMED',
+ '--add-opens', 'java.base/sun.nio.ch=ALL-UNNAMED',
+ '--add-opens', 'java.base/sun.nio.cs=ALL-UNNAMED',
+ '--add-opens', 'java.base/sun.security.action=ALL-UNNAMED',
+ '--add-opens', 'java.base/sun.util.calendar=ALL-UNNAMED',
+ '--add-opens', 'java.base/sun.nio.fs=ALL-UNNAMED',
+ '--add-opens', 'java.base/java.nio.channels.spi=ALL-UNNAMED',
+
+ // required for tests
+ '--add-opens', 'java.base/jdk.internal.ref=ALL-UNNAMED',
+ '--add-opens', 'java.base/java.lang.ref=ALL-UNNAMED',
+ '--add-opens', 'java.base/java.util.zip=ALL-UNNAMED',
+ '--add-opens', 'java.base/java.util.jar=ALL-UNNAMED',
+ '--add-opens', 'java.base/java.nio.file.attribute=ALL-UNNAMED',
+ '--add-opens', 'java.base/jdk.internal.loader=ALL-UNNAMED',
+ '--add-opens', 'java.base/sun.net.www.protocol.jar=ALL-UNNAMED',
+ '--add-opens', 'java.base/sun.invoke.util=ALL-UNNAMED',
+ '--add-opens', 'java.base/java.util.concurrent.locks=ALL-UNNAMED',
+ '--add-opens', 'java.base/java.security=ALL-UNNAMED',
+ '--add-opens', 'java.base/sun.reflect.annotation=ALL-UNNAMED',
+ '--add-opens', 'java.base/java.text=ALL-UNNAMED',
+ '--add-opens', 'java.base/java.nio.charset=ALL-UNNAMED',
+ '--add-opens', 'java.base/sun.reflect.generics.reflectiveObjects=ALL-UNNAMED',
+ '--add-opens', 'java.management/com.sun.jmx.mbeanserver=ALL-UNNAMED',
+ '--add-opens', 'java.management/javax.management=ALL-UNNAMED',
+ '--add-opens', 'java.base/java.util.regex=ALL-UNNAMED',
+ '--add-opens', 'java.base/sun.util.locale=ALL-UNNAMED',
+ '--add-opens', 'java.base/jdk.internal.math=ALL-UNNAMED',
+ '--add-opens', 'java.xml/com.sun.xml.internal.stream.util=ALL-UNNAMED',
+ '--add-opens', 'java.base/java.time=ALL-UNNAMED',
+ '--add-opens', 'java.base/sun.reflect.generics.factory=ALL-UNNAMED',
+ '--add-opens', 'java.base/java.nio.channels=ALL-UNNAMED',
+ '--add-opens', 'java.base/sun.security.util=ALL-UNNAMED',
+ '--add-opens', 'java.base/java.time.zone=ALL-UNNAMED',
+ '--add-opens', 'java.base/sun.reflect.generics.scope=ALL-UNNAMED',
+ '--add-opens', 'java.base/sun.reflect.generics.tree=ALL-UNNAMED',
+ '--add-opens', 'java.management/com.sun.jmx.interceptor=ALL-UNNAMED',
+ '--add-opens', 'java.management/javax.management.openmbean=ALL-UNNAMED',
+ '--add-opens', 'java.management/sun.management=ALL-UNNAMED',
+ '--add-opens', 'jdk.management/com.sun.management.internal=ALL-UNNAMED',
+ '--add-opens', 'jdk.management.jfr/jdk.management.jfr=ALL-UNNAMED',
+ '--add-opens', 'jdk.jfr/jdk.jfr.internal.management=ALL-UNNAMED',
+ '--add-opens', 'java.base/jdk.internal.module=ALL-UNNAMED',
+ '--add-opens', 'java.base/java.lang.module=ALL-UNNAMED',
+ '--add-opens', 'java.prefs/java.util.prefs=ALL-UNNAMED', // jacoco
+
+ '-Dio.netty.tryReflectionSetAccessible=true'
+ ]
+ jacoco {
+ jvmArgs = [
+ '--add-opens', 'java.base/java.lang=ALL-UNNAMED',
+ '--add-opens', 'java.base/java.lang.invoke=ALL-UNNAMED',
+ '--add-opens', 'java.base/java.lang.reflect=ALL-UNNAMED',
+ '--add-opens', 'java.base/java.io=ALL-UNNAMED',
+ '--add-opens', 'java.base/java.net=ALL-UNNAMED',
+ '--add-opens', 'java.base/java.nio=ALL-UNNAMED',
+ '--add-opens', 'java.base/java.util=ALL-UNNAMED',
+ '--add-opens', 'java.base/java.util.concurrent=ALL-UNNAMED',
+ '--add-opens', 'java.base/java.util.concurrent.atomic=ALL-UNNAMED',
+ '--add-opens', 'java.base/sun.nio.ch=ALL-UNNAMED',
+ '--add-opens', 'java.base/sun.nio.cs=ALL-UNNAMED',
+ '--add-opens', 'java.base/sun.security.action=ALL-UNNAMED',
+ '--add-opens', 'java.base/sun.util.calendar=ALL-UNNAMED',
+ '--add-opens', 'java.base/sun.nio.fs=ALL-UNNAMED',
+ '--add-opens', 'java.base/java.nio.channels.spi=ALL-UNNAMED',
+
+ // required for tests
+ '--add-opens', 'java.base/jdk.internal.ref=ALL-UNNAMED',
+ '--add-opens', 'java.base/java.lang.ref=ALL-UNNAMED',
+ '--add-opens', 'java.base/java.util.zip=ALL-UNNAMED',
+ '--add-opens', 'java.base/java.util.jar=ALL-UNNAMED',
+ '--add-opens', 'java.base/java.nio.file.attribute=ALL-UNNAMED',
+ '--add-opens', 'java.base/jdk.internal.loader=ALL-UNNAMED',
+ '--add-opens', 'java.base/sun.net.www.protocol.jar=ALL-UNNAMED',
+ '--add-opens', 'java.base/sun.invoke.util=ALL-UNNAMED',
+ '--add-opens', 'java.base/java.util.concurrent.locks=ALL-UNNAMED',
+ '--add-opens', 'java.base/java.security=ALL-UNNAMED',
+ '--add-opens', 'java.base/sun.reflect.annotation=ALL-UNNAMED',
+ '--add-opens', 'java.base/java.text=ALL-UNNAMED',
+ '--add-opens', 'java.base/java.nio.charset=ALL-UNNAMED',
+ '--add-opens', 'java.base/sun.reflect.generics.reflectiveObjects=ALL-UNNAMED',
+ '--add-opens', 'java.management/com.sun.jmx.mbeanserver=ALL-UNNAMED',
+ '--add-opens', 'java.base/javax.management=ALL-UNNAMED',
+ '--add-opens', 'java.base/java.util.regex=ALL-UNNAMED',
+ '--add-opens', 'java.base/sun.util.locale=ALL-UNNAMED',
+ '--add-opens', 'java.base/jdk.internal.math=ALL-UNNAMED',
+ '--add-opens', 'java.xml/com.sun.xml.internal.stream.util=ALL-UNNAMED',
+ '--add-opens', 'java.base/java.time=ALL-UNNAMED',
+ '--add-opens', 'java.base/sun.reflect.generics.factory=ALL-UNNAMED',
+ '--add-opens', 'java.base/java.nio.channels=ALL-UNNAMED',
+ '--add-opens', 'java.base/sun.security.util=ALL-UNNAMED',
+ '--add-opens', 'java.base/java.time.zone=ALL-UNNAMED',
+ '--add-opens', 'java.base/sun.reflect.generics.scope=ALL-UNNAMED',
+ '--add-opens', 'java.base/sun.reflect.generics.tree=ALL-UNNAMED',
+ '--add-opens', 'java.management/com.sun.jmx.interceptor=ALL-UNNAMED',
+ '--add-opens', 'java.management/javax.management.openmbean=ALL-UNNAMED',
+ '--add-opens', 'java.management/sun.management=ALL-UNNAMED',
+ '--add-opens', 'jdk.management/com.sun.management.internal=ALL-UNNAMED',
+ '--add-opens', 'jdk.management.jfr/jdk.management.jfr=ALL-UNNAMED',
+ '--add-opens', 'jdk.jfr/jdk.jfr.internal.management=ALL-UNNAMED',
+ '--add-opens', 'java.base/jdk.internal.module=ALL-UNNAMED',
+ '--add-opens', 'java.base/java.lang.module=ALL-UNNAMED',
+ '--add-opens', 'java.prefs/java.util.prefs=ALL-UNNAMED', // jacoco
+
+ '-Dio.netty.tryReflectionSetAccessible=true'
+ ]
+ }
+}
+
task testOnPackagedReleaseJar(type: Test){
String testClassesJar = "$System.env.TEST_JAR"
+
String testDependencyJar = "$System.env.TEST_DEPENDENCY_JAR"
+
String gatkJar = "$System.env.GATK_JAR"
String testClassesUnpacked = "$System.env.CP_DIR"
@@ -60,7 +181,64 @@ task testOnPackagedReleaseJar(type: Test){
classpath.each {println it}
}
- classpath = files( gatkJar, testDependencyJar, testClassesJar)
+ jvmArgs = [
+ '--add-opens', 'java.base/java.lang=ALL-UNNAMED',
+ '--add-opens', 'java.base/java.lang.invoke=ALL-UNNAMED',
+ '--add-opens', 'java.base/java.lang.reflect=ALL-UNNAMED',
+ '--add-opens', 'java.base/java.io=ALL-UNNAMED',
+ '--add-opens', 'java.base/java.net=ALL-UNNAMED',
+ '--add-opens', 'java.base/java.nio=ALL-UNNAMED',
+ '--add-opens', 'java.base/java.util=ALL-UNNAMED',
+ '--add-opens', 'java.base/java.util.concurrent=ALL-UNNAMED',
+ '--add-opens', 'java.base/java.util.concurrent.atomic=ALL-UNNAMED',
+ '--add-opens', 'java.base/sun.nio.ch=ALL-UNNAMED',
+ '--add-opens', 'java.base/sun.nio.cs=ALL-UNNAMED',
+ '--add-opens', 'java.base/sun.security.action=ALL-UNNAMED',
+ '--add-opens', 'java.base/sun.util.calendar=ALL-UNNAMED',
+ '--add-opens', 'java.base/sun.nio.fs=ALL-UNNAMED',
+ '--add-opens', 'java.base/java.nio.channels.spi=ALL-UNNAMED',
+ '--add-opens', 'java.base/jdk.internal.ref=ALL-UNNAMED',
+ '--add-opens', 'java.base/java.lang.ref=ALL-UNNAMED',
+ '--add-opens', 'java.base/java.util.zip=ALL-UNNAMED',
+ '--add-opens', 'java.base/java.util.jar=ALL-UNNAMED',
+ '--add-opens', 'java.base/java.nio.file.attribute=ALL-UNNAMED',
+ '--add-opens', 'java.base/jdk.internal.loader=ALL-UNNAMED',
+ '--add-opens', 'java.base/sun.net.www.protocol.jar=ALL-UNNAMED',
+ '--add-opens', 'java.base/sun.invoke.util=ALL-UNNAMED',
+ '--add-opens', 'java.base/java.util.concurrent.locks=ALL-UNNAMED',
+ '--add-opens', 'java.base/java.security=ALL-UNNAMED',
+ '--add-opens', 'java.base/sun.reflect.annotation=ALL-UNNAMED',
+ '--add-opens', 'java.base/java.text=ALL-UNNAMED',
+ '--add-opens', 'java.base/java.nio.charset=ALL-UNNAMED',
+ '--add-opens', 'java.base/sun.reflect.generics.reflectiveObjects=ALL-UNNAMED',
+ '--add-opens', 'java.management/com.sun.jmx.mbeanserver=ALL-UNNAMED',
+ '--add-opens', 'java.management/javax.management=ALL-UNNAMED',
+ '--add-opens', 'java.base/java.util.regex=ALL-UNNAMED',
+ '--add-opens', 'java.base/sun.util.locale=ALL-UNNAMED',
+ '--add-opens', 'java.base/jdk.internal.math=ALL-UNNAMED',
+ '--add-opens', 'java.xml/com.sun.xml.internal.stream.util=ALL-UNNAMED',
+ '--add-opens', 'java.base/java.time=ALL-UNNAMED',
+ '--add-opens', 'java.base/sun.reflect.generics.factory=ALL-UNNAMED',
+ '--add-opens', 'java.base/java.nio.channels=ALL-UNNAMED',
+ '--add-opens', 'java.base/sun.security.util=ALL-UNNAMED',
+ '--add-opens', 'java.base/java.time.zone=ALL-UNNAMED',
+ '--add-opens', 'java.base/sun.reflect.generics.scope=ALL-UNNAMED',
+ '--add-opens', 'java.base/sun.reflect.generics.tree=ALL-UNNAMED',
+ '--add-opens', 'java.management/com.sun.jmx.interceptor=ALL-UNNAMED',
+ '--add-opens', 'java.management/javax.management.openmbean=ALL-UNNAMED',
+ '--add-opens', 'java.management/sun.management=ALL-UNNAMED',
+ '--add-opens', 'jdk.management/com.sun.management.internal=ALL-UNNAMED',
+ '--add-opens', 'jdk.management.jfr/jdk.management.jfr=ALL-UNNAMED',
+ '--add-opens', 'jdk.jfr/jdk.jfr.internal.management=ALL-UNNAMED',
+ '--add-opens', 'java.base/jdk.internal.module=ALL-UNNAMED',
+ '--add-opens', 'java.base/java.lang.module=ALL-UNNAMED',
+ '--add-opens', 'java.security.jgss/sun.security.krb5=ALL-UNNAMED',
+ '--add-opens', 'java.prefs/java.util.prefs=ALL-UNNAMED', // jacoco
+
+ '-Dio.netty.tryReflectionSetAccessible=true'
+ ]
+
+ classpath = files( gatkJar, testClassesJar, testDependencyJar)
testClassesDirs = files(testClassesUnpacked)
}
diff --git a/scripts/vcf_site_level_filtering_cromwell_tests/run_vcf_site_level_filtering_wdl.sh b/scripts/vcf_site_level_filtering_cromwell_tests/run_vcf_site_level_filtering_wdl.sh
index 1c19d18c3b6..d726678360b 100644
--- a/scripts/vcf_site_level_filtering_cromwell_tests/run_vcf_site_level_filtering_wdl.sh
+++ b/scripts/vcf_site_level_filtering_cromwell_tests/run_vcf_site_level_filtering_wdl.sh
@@ -35,4 +35,4 @@ echo "=================="
echo "Running Filtering WDL through cromwell"
ln -fs $WORKING_DIR/gatk/scripts/vcf_site_level_filtering_wdl/JointVcfFiltering.wdl
cd $WORKING_DIR/gatk/scripts/vcf_site_level_filtering_wdl/
-java -jar $CROMWELL_JAR run JointVcfFiltering.wdl -i $WORKING_DIR/vcf_site_level_filtering_travis.json
+java -DLOG_LEVEL=TRACE -jar $CROMWELL_JAR run JointVcfFiltering.wdl -i $WORKING_DIR/vcf_site_level_filtering_travis.json
diff --git a/src/main/java/org/broadinstitute/hellbender/cmdline/GATKPlugin/GATKAnnotationPluginDescriptor.java b/src/main/java/org/broadinstitute/hellbender/cmdline/GATKPlugin/GATKAnnotationPluginDescriptor.java
index c03be1fbc76..aa7e467b62e 100644
--- a/src/main/java/org/broadinstitute/hellbender/cmdline/GATKPlugin/GATKAnnotationPluginDescriptor.java
+++ b/src/main/java/org/broadinstitute/hellbender/cmdline/GATKPlugin/GATKAnnotationPluginDescriptor.java
@@ -11,12 +11,15 @@
import org.broadinstitute.hellbender.exceptions.GATKException;
import org.broadinstitute.hellbender.tools.walkers.annotator.Annotation;
import org.broadinstitute.hellbender.tools.walkers.annotator.PedigreeAnnotation;
+import org.broadinstitute.hellbender.utils.SerializableConsumer;
+import org.broadinstitute.hellbender.utils.SerializableFunction;
+import org.broadinstitute.hellbender.utils.SerializablePredicate;
import org.broadinstitute.hellbender.tools.walkers.annotator.flow.FlowAnnotatorBase;
import org.broadinstitute.hellbender.utils.Utils;
import org.broadinstitute.hellbender.utils.config.ConfigFactory;
import org.broadinstitute.hellbender.utils.config.GATKConfig;
-import java.io.File;
+import java.io.Serializable;
import java.lang.reflect.Modifier;
import java.util.*;
import java.util.stream.Collectors;
@@ -36,7 +39,9 @@
* NOTE: this class enforces that annotations with required arguments must see their arguments, yet this is not currently tested
* as no such annotations exist in the GATK.
*/
-public class GATKAnnotationPluginDescriptor extends CommandLinePluginDescriptor {
+public class GATKAnnotationPluginDescriptor extends CommandLinePluginDescriptor implements Serializable {
+ private static final long serialVersionUID = 1L;
+
/**
* At startup, set the plugin package name to the one(s) in the configuration file.
*/
@@ -116,7 +121,7 @@ public List getPackageNames() {
public GATKAnnotationPluginDescriptor(final GATKAnnotationArgumentCollection userArgs, final List toolDefaultAnnotations, final List> toolDefaultGroups) {
this.userArgs = userArgs;
if (null != toolDefaultAnnotations) {
- toolDefaultAnnotations.forEach(f -> {
+ toolDefaultAnnotations.forEach((SerializableConsumer) (f -> {
final Class extends Annotation> annotClass = f.getClass();
// anonymous classes have a 0-length simple name, and thus cannot be accessed or
// controlled by the user via the command line, but they should still be valid
@@ -128,15 +133,16 @@ public GATKAnnotationPluginDescriptor(final GATKAnnotationArgumentCollection use
}
populateAnnotationGroups(className, f);
this.toolDefaultAnnotations.put(className, f);
- });
+ }));
}
if (null != toolDefaultGroups) {
- toolDefaultGroups.forEach(a -> {if (a.isInterface() && a!=Annotation.class) {
- this.toolDefaultGroups.add(a.getSimpleName());
- } else {
- throw new GATKException(String.format("Tool specified annotation group %s is not a valid annotation group, must be an interface extending Annotation", a.getSimpleName()));
- }
- });
+ toolDefaultGroups.forEach((SerializableConsumer>) (a -> {
+ if (a.isInterface() && a!=Annotation.class) {
+ this.toolDefaultGroups.add(a.getSimpleName());
+ } else {
+ throw new GATKException(String.format("Tool specified annotation group %s is not a valid annotation group, must be an interface extending Annotation", a.getSimpleName()));
+ }
+ }));
}
}
/**
@@ -257,7 +263,9 @@ public Set getAllowedValuesForDescriptorHelp(String longArgName) {
return allDiscoveredAnnotations.keySet();
}
if (longArgName.equals(StandardArgumentDefinitions.ANNOTATIONS_TO_EXCLUDE_LONG_NAME)) {
- Set annotations = toolDefaultGroups.stream().map(k -> discoveredGroups.get(k).keySet()).flatMap(Collection::stream).collect(Collectors.toSet());
+ Set annotations = toolDefaultGroups.stream().map(
+ k -> discoveredGroups.get(k).keySet())
+ .flatMap(Collection::stream).collect(Collectors.toSet());
annotations.addAll(toolDefaultAnnotations.keySet());
return annotations;
}
@@ -281,9 +289,11 @@ public boolean isDependentArgumentAllowed(final Class> predecessorClass) {
if (!isAllowed) {
// Check whether any of the annotations have been added via groups (either tool default or user enabled)
isAllowed = Stream.of(userArgs.getUserEnabledAnnotationGroups(), toolDefaultGroups)
- .flatMap(Collection::stream)
- .anyMatch(group ->
- discoveredGroups.containsKey(group) && discoveredGroups.get(group).keySet().stream().anyMatch(s -> s.equals(predecessorName)));
+ .flatMap((SerializableFunction, Stream>) t -> t.stream())
+ .anyMatch((SerializablePredicate) (group ->
+ discoveredGroups.containsKey(group) &&
+ discoveredGroups.get(group).keySet().stream()
+ .anyMatch((SerializablePredicate) (s -> s.equals(predecessorName)))));
}
if (isAllowed) {
// Keep track of the ones we allow so we can validate later that they weren't subsequently disabled
@@ -326,29 +336,29 @@ public void validateAndResolvePlugins() throws CommandLineException {
}
// throw if a disabled annotation doesn't exist; warn if it wasn't enabled by the tool in the first place
- userArgs.getUserDisabledAnnotationNames().forEach(s -> {
+ userArgs.getUserDisabledAnnotationNames().forEach((SerializableConsumer) (s -> {
if (!allDiscoveredAnnotations.containsKey(s)) {
throw new CommandLineException.BadArgumentValue(String.format("Disabled annotation (%s) does not exist", s));
} else if (!toolDefaultAnnotations.containsKey(s)) {
logger.warn(String.format("Disabled annotation (%s) is not enabled by this tool", s));
}
- });
+ }));
// warn if an annotation is both default and enabled by the user
final Set redundantAnnots = new HashSet<>(toolDefaultAnnotations.keySet());
redundantAnnots.retainAll(userArgs.getUserEnabledAnnotationNames());
- redundantAnnots.forEach(
- s -> {
+ redundantAnnots.forEach((SerializableConsumer)
+ (s -> {
logger.warn(String.format("Redundant enabled annotation (%s) is enabled for this tool by default", s));
- });
+ }));
// warn if an annotation group is both default and enabled by the user
final Set redundantGroups = new HashSet<>(toolDefaultGroups);
redundantGroups.retainAll(userArgs.getUserEnabledAnnotationGroups());
- redundantGroups.forEach(
- s -> {
+ redundantGroups.forEach((SerializableConsumer)
+ (s -> {
logger.warn(String.format("Redundant enabled annotation group (%s) is enabled for this tool by default", s));
- });
+ }));
// Throw if args were specified for an annotation that was also disabled, or that was not enabled by the
// tool by default.
@@ -376,21 +386,21 @@ public void validateAndResolvePlugins() throws CommandLineException {
});
// throw if an annotation name was specified that has no corresponding instance
- userArgs.getUserEnabledAnnotationNames().forEach(s -> {
+ userArgs.getUserEnabledAnnotationNames().forEach((SerializableConsumer) (s -> {
Annotation ta = allDiscoveredAnnotations.get(s);
if (null == ta) {
if (!toolDefaultAnnotations.containsKey(s)) {
throw new CommandLineException("Unrecognized annotation name: " + s);
}
}
- });
+ }));
// throw if an annotation group was specified that has no corresponding instance
- userArgs.getUserEnabledAnnotationGroups().forEach(s -> {
+ userArgs.getUserEnabledAnnotationGroups().forEach((SerializableConsumer) (s -> {
if (!discoveredGroups.containsKey(s)) {
throw new CommandLineException("Unrecognized annotation group name: " + s);
}
- });
+ }));
// Populating the tool default annotations with the ones requested by groups
for (String group : toolDefaultGroups ) {
@@ -418,6 +428,7 @@ public void validateAndResolvePlugins() throws CommandLineException {
allDiscoveredAnnotations.values().stream().filter(PedigreeAnnotation.class::isInstance).map(a -> a.getClass().getSimpleName()).collect(Collectors.joining(", "))));
}
+ //TODO: fix these lambdas to have serializable types
// Populating any discovered flow annotations with the flowOrder arguments from the command line.
if (flowOrder!=null && !flowOrder.isEmpty() && getResolvedInstances().stream()
.filter(FlowAnnotatorBase.class::isInstance)
@@ -465,8 +476,8 @@ public List getDefaultInstances() {
@Override
public List getResolvedInstances() {
if (resolvedInstances == null) {
- final SortedSet annotations = new TreeSet<>(Comparator.comparing(t -> t.getClass().getSimpleName()));
-
+ final SortedSet annotations = new TreeSet<>(Comparator.comparing(
+ (SerializableFunction) t -> t.getClass().getSimpleName()));
if (!userArgs.getDisableToolDefaultAnnotations()) {
annotations.addAll(toolDefaultAnnotations.values());
}
diff --git a/src/main/java/org/broadinstitute/hellbender/tools/HaplotypeCallerSpark.java b/src/main/java/org/broadinstitute/hellbender/tools/HaplotypeCallerSpark.java
index 8ff76a1ae6c..52277de766f 100644
--- a/src/main/java/org/broadinstitute/hellbender/tools/HaplotypeCallerSpark.java
+++ b/src/main/java/org/broadinstitute/hellbender/tools/HaplotypeCallerSpark.java
@@ -3,7 +3,6 @@
import com.google.common.collect.Iterators;
import htsjdk.samtools.SAMFileHeader;
import htsjdk.samtools.SAMSequenceDictionary;
-import htsjdk.samtools.reference.ReferenceSequenceFile;
import htsjdk.samtools.util.FileExtensions;
import htsjdk.variant.variantcontext.VariantContext;
import org.apache.logging.log4j.Logger;
@@ -29,6 +28,11 @@
import org.broadinstitute.hellbender.exceptions.UserException;
import org.broadinstitute.hellbender.tools.walkers.annotator.Annotation;
import org.broadinstitute.hellbender.tools.walkers.annotator.VariantAnnotatorEngine;
+import org.broadinstitute.hellbender.tools.walkers.haplotypecaller.HaplotypeCaller;
+import org.broadinstitute.hellbender.tools.walkers.haplotypecaller.HaplotypeCallerArgumentCollection;
+import org.broadinstitute.hellbender.tools.walkers.haplotypecaller.HaplotypeCallerEngine;
+import org.broadinstitute.hellbender.tools.walkers.haplotypecaller.ReferenceConfidenceMode;
+import org.broadinstitute.hellbender.utils.SerializableSupplier;
import org.broadinstitute.hellbender.tools.walkers.haplotypecaller.*;
import org.broadinstitute.hellbender.utils.Utils;
import org.broadinstitute.hellbender.utils.fasta.CachingIndexedFastaSequenceFile;
@@ -142,7 +146,7 @@ private static void processAssemblyRegions(
final VariantAnnotatorEngine variantannotatorEngine = new VariantAnnotatorEngine(annotations, hcArgs.dbsnp.dbsnp, hcArgs.comps, hcArgs.emitReferenceConfidence != ReferenceConfidenceMode.NONE, false);
final Path referencePath = IOUtils.getPath(reference);
- final ReferenceSequenceFile driverReferenceSequenceFile = new CachingIndexedFastaSequenceFile(referencePath);
+ final CachingIndexedFastaSequenceFile driverReferenceSequenceFile = new CachingIndexedFastaSequenceFile(referencePath);
final HaplotypeCallerEngine hcEngine = new HaplotypeCallerEngine(hcArgs, assemblyRegionArgs, false, false, header, driverReferenceSequenceFile, variantannotatorEngine);
final String referenceFileName = referencePath.getFileName().toString();
final Broadcast hcArgsBroadcast = ctx.broadcast(hcArgs);
@@ -165,9 +169,9 @@ private static FlatMapFunction, VariantCon
final Broadcast hcArgsBroadcast,
final Broadcast assemblyRegionArgsBroadcast,
final Broadcast annotatorEngineBroadcast) {
- return (FlatMapFunction, VariantContext>) contexts -> {
+ return contexts -> {
// HaplotypeCallerEngine isn't serializable but is expensive to instantiate, so construct and reuse one for every partition
- final ReferenceSequenceFile taskReferenceSequenceFile = taskReferenceSequenceFile(referenceFileName);
+ final CachingIndexedFastaSequenceFile taskReferenceSequenceFile = taskReferenceSequenceFile(referenceFileName);
final HaplotypeCallerEngine hcEngine = new HaplotypeCallerEngine(hcArgsBroadcast.value(), assemblyRegionArgsBroadcast.value(), false, false, header, taskReferenceSequenceFile, annotatorEngineBroadcast.getValue());
Iterator> iterators = Utils.stream(contexts).map(context -> {
AssemblyRegion region = context.getAssemblyRegion();
@@ -194,7 +198,7 @@ protected Broadcast> assemblyRegionEvaluatorSu
final Path referencePath = IOUtils.getPath(referenceArguments.getReferenceFileName());
final String referenceFileName = referencePath.getFileName().toString();
final String pathOnExecutor = SparkFiles.get(referenceFileName);
- final ReferenceSequenceFile taskReferenceSequenceFile = new CachingIndexedFastaSequenceFile(IOUtils.getPath(pathOnExecutor));
+ final CachingIndexedFastaSequenceFile taskReferenceSequenceFile = new CachingIndexedFastaSequenceFile(IOUtils.getPath(pathOnExecutor));
final Collection annotations = makeVariantAnnotations();
final VariantAnnotatorEngine annotatorEngine = new VariantAnnotatorEngine(annotations, hcArgs.dbsnp.dbsnp, hcArgs.comps, hcArgs.emitReferenceConfidence != ReferenceConfidenceMode.NONE, false);
return assemblyRegionEvaluatorSupplierBroadcastFunction(ctx, hcArgs, assemblyRegionArgs, getHeaderForReads(), taskReferenceSequenceFile, annotatorEngine);
@@ -208,12 +212,12 @@ private static Broadcast> assemblyRegionEvalua
final Collection annotations) {
final Path referencePath = IOUtils.getPath(reference);
final String referenceFileName = referencePath.getFileName().toString();
- final ReferenceSequenceFile taskReferenceSequenceFile = taskReferenceSequenceFile(referenceFileName);
+ final CachingIndexedFastaSequenceFile taskReferenceSequenceFile = taskReferenceSequenceFile(referenceFileName);
final VariantAnnotatorEngine annotatorEngine = new VariantAnnotatorEngine(annotations, hcArgs.dbsnp.dbsnp, hcArgs.comps, hcArgs.emitReferenceConfidence != ReferenceConfidenceMode.NONE, false);
return assemblyRegionEvaluatorSupplierBroadcastFunction(ctx, hcArgs, assemblyRegionArgs, header, taskReferenceSequenceFile, annotatorEngine);
}
- private static ReferenceSequenceFile taskReferenceSequenceFile(final String referenceFileName) {
+ private static CachingIndexedFastaSequenceFile taskReferenceSequenceFile(final String referenceFileName) {
final String pathOnExecutor = SparkFiles.get(referenceFileName);
return new CachingIndexedFastaSequenceFile(IOUtils.getPath(pathOnExecutor));
}
@@ -222,9 +226,9 @@ private static Broadcast> assemblyRegionEvalua
final JavaSparkContext ctx,
final HaplotypeCallerArgumentCollection hcArgs,
AssemblyRegionArgumentCollection assemblyRegionArgs, final SAMFileHeader header,
- final ReferenceSequenceFile taskReferenceSequenceFile,
+ final CachingIndexedFastaSequenceFile taskReferenceSequenceFile,
final VariantAnnotatorEngine annotatorEngine) {
- Supplier supplier = new Supplier() {
+ SerializableSupplier supplier = new SerializableSupplier() {
@Override
public AssemblyRegionEvaluator get() {
return new HaplotypeCallerEngine(hcArgs, assemblyRegionArgs, false, false, header, taskReferenceSequenceFile, annotatorEngine);
diff --git a/src/main/java/org/broadinstitute/hellbender/tools/dragstr/CalibrateDragstrModel.java b/src/main/java/org/broadinstitute/hellbender/tools/dragstr/CalibrateDragstrModel.java
index 24bb5df696f..9b75290cb60 100644
--- a/src/main/java/org/broadinstitute/hellbender/tools/dragstr/CalibrateDragstrModel.java
+++ b/src/main/java/org/broadinstitute/hellbender/tools/dragstr/CalibrateDragstrModel.java
@@ -263,6 +263,7 @@ private void checkSequenceDictionaryCompatibility(final SAMSequenceDictionary re
}
}
+ @SuppressWarnings("deprecation")
private PrintWriter openSitesOutputWriter(final String sitesOutput) {
return sitesOutput == null ? new PrintWriter(new NullOutputStream())
: new PrintWriter(BucketUtils.createFile(sitesOutput));
diff --git a/src/main/java/org/broadinstitute/hellbender/tools/walkers/haplotypecaller/HaplotypeCallerEngine.java b/src/main/java/org/broadinstitute/hellbender/tools/walkers/haplotypecaller/HaplotypeCallerEngine.java
index 768264446de..00d22c7600c 100644
--- a/src/main/java/org/broadinstitute/hellbender/tools/walkers/haplotypecaller/HaplotypeCallerEngine.java
+++ b/src/main/java/org/broadinstitute/hellbender/tools/walkers/haplotypecaller/HaplotypeCallerEngine.java
@@ -25,6 +25,7 @@
import org.broadinstitute.hellbender.tools.walkers.genotyper.OutputMode;
import org.broadinstitute.hellbender.tools.walkers.genotyper.StandardCallerArgumentCollection;
import org.broadinstitute.hellbender.tools.walkers.haplotypecaller.readthreading.ReadThreadingAssembler;
+import org.broadinstitute.hellbender.utils.fasta.CachingIndexedFastaSequenceFile;
import org.broadinstitute.hellbender.utils.pileup.PileupBasedAlleles;
import org.broadinstitute.hellbender.transformers.IUPACReadTransformer;
import org.broadinstitute.hellbender.transformers.ReadTransformer;
@@ -169,7 +170,7 @@ public class HaplotypeCallerEngine implements AssemblyRegionEvaluator {
*/
public HaplotypeCallerEngine(final HaplotypeCallerArgumentCollection hcArgs, AssemblyRegionArgumentCollection assemblyRegionArgs, boolean createBamOutIndex,
boolean createBamOutMD5, final SAMFileHeader readsHeader,
- ReferenceSequenceFile referenceReader, VariantAnnotatorEngine annotationEngine) {
+ CachingIndexedFastaSequenceFile referenceReader, VariantAnnotatorEngine annotationEngine) {
this.dragstrParams = DragstrParamUtils.parse(hcArgs.likelihoodArgs.dragstrParams);
this.hcArgs = Utils.nonNull(hcArgs);
this.readsHeader = Utils.nonNull(readsHeader);
diff --git a/src/main/java/org/broadinstitute/hellbender/tools/walkers/haplotypecaller/RampedHaplotypeCallerEngine.java b/src/main/java/org/broadinstitute/hellbender/tools/walkers/haplotypecaller/RampedHaplotypeCallerEngine.java
index f0db1b52bac..b66bc6e7202 100644
--- a/src/main/java/org/broadinstitute/hellbender/tools/walkers/haplotypecaller/RampedHaplotypeCallerEngine.java
+++ b/src/main/java/org/broadinstitute/hellbender/tools/walkers/haplotypecaller/RampedHaplotypeCallerEngine.java
@@ -14,6 +14,7 @@
import org.broadinstitute.hellbender.tools.walkers.haplotypecaller.ramps.*;
import org.broadinstitute.hellbender.utils.SimpleInterval;
import org.broadinstitute.hellbender.utils.downsampling.ReservoirDownsampler;
+import org.broadinstitute.hellbender.utils.fasta.CachingIndexedFastaSequenceFile;
import org.broadinstitute.hellbender.utils.genotyper.AlleleLikelihoods;
import org.broadinstitute.hellbender.utils.haplotype.EventMap;
import org.broadinstitute.hellbender.utils.haplotype.Haplotype;
@@ -61,7 +62,7 @@ public class RampedHaplotypeCallerEngine extends HaplotypeCallerEngine {
public RampedHaplotypeCallerEngine(final HaplotypeCallerArgumentCollection hcArgs, AssemblyRegionArgumentCollection assemblyRegionArgs, boolean createBamOutIndex,
boolean createBamOutMD5, final SAMFileHeader readsHeader,
- ReferenceSequenceFile referenceReader, VariantAnnotatorEngine annotationEngine,
+ CachingIndexedFastaSequenceFile referenceReader, VariantAnnotatorEngine annotationEngine,
RampedHaplotypeCallerArgumentCollection rpArgs) {
super(hcArgs, assemblyRegionArgs, createBamOutIndex,
diff --git a/src/main/java/org/broadinstitute/hellbender/utils/SerializableConsumer.java b/src/main/java/org/broadinstitute/hellbender/utils/SerializableConsumer.java
new file mode 100644
index 00000000000..2f61106669e
--- /dev/null
+++ b/src/main/java/org/broadinstitute/hellbender/utils/SerializableConsumer.java
@@ -0,0 +1,6 @@
+package org.broadinstitute.hellbender.utils;
+
+import java.io.Serializable;
+import java.util.function.Consumer;
+
+public interface SerializableConsumer extends Consumer, Serializable {}
diff --git a/src/main/java/org/broadinstitute/hellbender/utils/SerializablePredicate.java b/src/main/java/org/broadinstitute/hellbender/utils/SerializablePredicate.java
new file mode 100644
index 00000000000..3da23d638bd
--- /dev/null
+++ b/src/main/java/org/broadinstitute/hellbender/utils/SerializablePredicate.java
@@ -0,0 +1,6 @@
+package org.broadinstitute.hellbender.utils;
+
+import java.io.Serializable;
+import java.util.function.Predicate;
+
+public interface SerializablePredicate extends Predicate, Serializable {}
diff --git a/src/main/java/org/broadinstitute/hellbender/utils/SerializableSupplier.java b/src/main/java/org/broadinstitute/hellbender/utils/SerializableSupplier.java
new file mode 100644
index 00000000000..b62df11bc71
--- /dev/null
+++ b/src/main/java/org/broadinstitute/hellbender/utils/SerializableSupplier.java
@@ -0,0 +1,6 @@
+package org.broadinstitute.hellbender.utils;
+
+import java.io.Serializable;
+import java.util.function.Supplier;
+
+public interface SerializableSupplier extends Supplier, Serializable {}
\ No newline at end of file
diff --git a/src/main/java/org/broadinstitute/hellbender/utils/fasta/CachingIndexedFastaSequenceFile.java b/src/main/java/org/broadinstitute/hellbender/utils/fasta/CachingIndexedFastaSequenceFile.java
index 1ead048346e..6cd2d678347 100644
--- a/src/main/java/org/broadinstitute/hellbender/utils/fasta/CachingIndexedFastaSequenceFile.java
+++ b/src/main/java/org/broadinstitute/hellbender/utils/fasta/CachingIndexedFastaSequenceFile.java
@@ -19,6 +19,7 @@
import org.broadinstitute.hellbender.utils.Utils;
import java.io.IOException;
+import java.io.Serializable;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Arrays;
@@ -31,10 +32,13 @@
*
* Instances of this class should be closed when they are no longer needed.
*/
-public final class CachingIndexedFastaSequenceFile implements ReferenceSequenceFile {
+public final class CachingIndexedFastaSequenceFile implements ReferenceSequenceFile, Serializable {
+ private static final long serialVersionUID = 1L;
+
protected static final Logger logger = LogManager.getLogger(CachingIndexedFastaSequenceFile.class);
- private final ReferenceSequenceFile sequenceFile;
+ // TODO: this gets around numerous serialization issues with non-serializable lambdas
+ private transient final ReferenceSequenceFile sequenceFile;
/** do we want to print debugging information about cache efficiency? */
private static final boolean PRINT_EFFICIENCY = false;
diff --git a/src/main/java/org/broadinstitute/hellbender/utils/help/GATKDocWorkUnit.java b/src/main/java/org/broadinstitute/hellbender/utils/help/GATKDocWorkUnit.java
index 1dce5a8422b..b94df3c80b8 100644
--- a/src/main/java/org/broadinstitute/hellbender/utils/help/GATKDocWorkUnit.java
+++ b/src/main/java/org/broadinstitute/hellbender/utils/help/GATKDocWorkUnit.java
@@ -1,41 +1,41 @@
-package org.broadinstitute.hellbender.utils.help;
-
-import org.broadinstitute.barclay.help.DocWorkUnit;
-import org.broadinstitute.barclay.help.DocWorkUnitHandler;
-import org.broadinstitute.barclay.help.DocumentedFeature;
-
-import org.broadinstitute.hellbender.utils.runtime.RuntimeUtils;
-
-/**
- * Custom DocWorkUnit used for generating GATK help/documentation. Overrides the defaults to provide tool
- * names that are annotated with a " (Picard)" suffix for Picard tools.
- *
- * NOTE: Methods in this class are intended to be called by Gradle/Javadoc only, and should not be called
- * by methods that are used by the GATK runtime. This class has a dependency on com.sun.javadoc classes,
- * which may not be present since they're not provided as part of the normal GATK runtime classpath.
- */
-@SuppressWarnings("removal")
-public class GATKDocWorkUnit extends DocWorkUnit {
-
- public GATKDocWorkUnit(
- final DocWorkUnitHandler workUnitHandler,
- final DocumentedFeature documentedFeatureAnnotation,
- final com.sun.javadoc.ClassDoc classDoc,
- final Class> clazz) {
- super(workUnitHandler, documentedFeatureAnnotation, classDoc, clazz);
- }
-
- @Override
- public String getName() {
- // Override getName to return a display name that annotates Picard tool names with " (Picard)"
- return RuntimeUtils.toolDisplayName(getClazz());
- }
-
- /**
- * Sort in order of the name of this WorkUnit
- */
- @Override
- public int compareTo(DocWorkUnit other) {
- return this.getName().compareTo(other.getName());
- }
-}
+//package org.broadinstitute.hellbender.utils.help;
+//
+//import org.broadinstitute.barclay.help.DocWorkUnit;
+//import org.broadinstitute.barclay.help.DocWorkUnitHandler;
+//import org.broadinstitute.barclay.help.DocumentedFeature;
+//
+//import org.broadinstitute.hellbender.utils.runtime.RuntimeUtils;
+//
+///**
+// * Custom DocWorkUnit used for generating GATK help/documentation. Overrides the defaults to provide tool
+// * names that are annotated with a " (Picard)" suffix for Picard tools.
+// *
+// * NOTE: Methods in this class are intended to be called by Gradle/Javadoc only, and should not be called
+// * by methods that are used by the GATK runtime. This class has a dependency on com.sun.javadoc classes,
+// * which may not be present since they're not provided as part of the normal GATK runtime classpath.
+// */
+//@SuppressWarnings("removal")
+//public class GATKDocWorkUnit extends DocWorkUnit {
+//
+// public GATKDocWorkUnit(
+// final DocWorkUnitHandler workUnitHandler,
+// final DocumentedFeature documentedFeatureAnnotation,
+// final com.sun.javadoc.ClassDoc classDoc,
+// final Class> clazz) {
+// super(workUnitHandler, documentedFeatureAnnotation, classDoc, clazz);
+// }
+//
+// @Override
+// public String getName() {
+// // Override getName to return a display name that annotates Picard tool names with " (Picard)"
+// return RuntimeUtils.toolDisplayName(getClazz());
+// }
+//
+// /**
+// * Sort in order of the name of this WorkUnit
+// */
+// @Override
+// public int compareTo(DocWorkUnit other) {
+// return this.getName().compareTo(other.getName());
+// }
+//}
diff --git a/src/main/java/org/broadinstitute/hellbender/utils/help/GATKGSONWorkUnit.java b/src/main/java/org/broadinstitute/hellbender/utils/help/GATKGSONWorkUnit.java
index 72eeb27aeb6..0216ab9b707 100644
--- a/src/main/java/org/broadinstitute/hellbender/utils/help/GATKGSONWorkUnit.java
+++ b/src/main/java/org/broadinstitute/hellbender/utils/help/GATKGSONWorkUnit.java
@@ -1,17 +1,17 @@
-package org.broadinstitute.hellbender.utils.help;
-
-import org.broadinstitute.barclay.help.GSONWorkUnit;
-
-/**
- * Class representing a GSONWorkUnit for GATK work units.
- *
- * Adds "walkertype" to the base gson object created by Barclay.
- */
-public class GATKGSONWorkUnit extends GSONWorkUnit {
-
- private String walkerType;
-
- public void setWalkerType(final String walkerType){
- this.walkerType = walkerType;
- }
-}
+//package org.broadinstitute.hellbender.utils.help;
+//
+//import org.broadinstitute.barclay.help.GSONWorkUnit;
+//
+///**
+// * Class representing a GSONWorkUnit for GATK work units.
+// *
+// * Adds "walkertype" to the base gson object created by Barclay.
+// */
+//public class GATKGSONWorkUnit extends GSONWorkUnit {
+//
+// private String walkerType;
+//
+// public void setWalkerType(final String walkerType){
+// this.walkerType = walkerType;
+// }
+//}
diff --git a/src/main/java/org/broadinstitute/hellbender/utils/help/GATKHelpDocWorkUnitHandler.java b/src/main/java/org/broadinstitute/hellbender/utils/help/GATKHelpDocWorkUnitHandler.java
index ab2cf1a9d06..b31bfcc640a 100644
--- a/src/main/java/org/broadinstitute/hellbender/utils/help/GATKHelpDocWorkUnitHandler.java
+++ b/src/main/java/org/broadinstitute/hellbender/utils/help/GATKHelpDocWorkUnitHandler.java
@@ -1,61 +1,61 @@
-package org.broadinstitute.hellbender.utils.help;
-
-import org.broadinstitute.barclay.argparser.CommandLineProgramProperties;
-import org.broadinstitute.barclay.help.DefaultDocWorkUnitHandler;
-import org.broadinstitute.barclay.help.DocWorkUnit;
-
-import org.broadinstitute.barclay.help.HelpDoclet;
-
-/**
- * The GATK Documentation work unit handler class that is the companion to GATKHelpDoclet.
- *
- * NOTE: Methods in this class are intended to be called by Gradle/Javadoc only, and should not be called
- * by methods that are used by the GATK runtime, as this class assumes a dependency on com.sun.javadoc classes
- * which may not be present.
- */
-public class GATKHelpDocWorkUnitHandler extends DefaultDocWorkUnitHandler {
-
- private final static String GATK_JAVADOC_TAG_PREFIX = "GATK"; // prefix for custom javadoc tags used by GATK
-
- private final static String GATK_FREEMARKER_TEMPLATE_NAME = "generic.template.html";
-
- public GATKHelpDocWorkUnitHandler(final HelpDoclet doclet) {
- super(doclet);
- }
- /**
- * @return Prefix for custom GATK tags that should be lifted from the javadoc and stored in the
- * FreeMarker map. These will be available in the template returned by {@link #getTemplateName}.
- */
- @Override
- protected String getTagFilterPrefix() { return GATK_JAVADOC_TAG_PREFIX; }
-
- /**
- * @param workUnit the classdoc object being processed
- * @return the name of a the freemarker template to be used for the class being documented.
- * Must reside in the folder passed to the Barclay Doclet via the "-settings-dir" parameter to
- * Javadoc.
- */
- @Override
- public String getTemplateName(final DocWorkUnit workUnit) { return GATK_FREEMARKER_TEMPLATE_NAME; }
-
-
- /**
- * Add any custom freemarker bindings discovered via custom javadoc tags. Subclasses can override this to
- * provide additional custom bindings.
- *
- * @param currentWorkUnit the work unit for the feature being documented
- */
- @Override
- protected void addCustomBindings(final DocWorkUnit currentWorkUnit) {
- super.addCustomBindings(currentWorkUnit);
-
- // Picard tools use the summary line for the long overview section, so extract that
- // from Picard tools only, and put it in the freemarker map.
- Class> toolClass = currentWorkUnit.getClazz();
- if (picard.cmdline.CommandLineProgram.class.isAssignableFrom(toolClass)) {
- final CommandLineProgramProperties clpProperties = currentWorkUnit.getCommandLineProperties();
- currentWorkUnit.setProperty("picardsummary", clpProperties.summary());
- }
- }
-
-}
+//package org.broadinstitute.hellbender.utils.help;
+//
+//import org.broadinstitute.barclay.argparser.CommandLineProgramProperties;
+//import org.broadinstitute.barclay.help.DefaultDocWorkUnitHandler;
+//import org.broadinstitute.barclay.help.DocWorkUnit;
+//
+//import org.broadinstitute.barclay.help.HelpDoclet;
+//
+///**
+// * The GATK Documentation work unit handler class that is the companion to GATKHelpDoclet.
+// *
+// * NOTE: Methods in this class are intended to be called by Gradle/Javadoc only, and should not be called
+// * by methods that are used by the GATK runtime, as this class assumes a dependency on com.sun.javadoc classes
+// * which may not be present.
+// */
+//public class GATKHelpDocWorkUnitHandler extends DefaultDocWorkUnitHandler {
+//
+// private final static String GATK_JAVADOC_TAG_PREFIX = "GATK"; // prefix for custom javadoc tags used by GATK
+//
+// private final static String GATK_FREEMARKER_TEMPLATE_NAME = "generic.template.html";
+//
+// public GATKHelpDocWorkUnitHandler(final HelpDoclet doclet) {
+// super(doclet);
+// }
+// /**
+// * @return Prefix for custom GATK tags that should be lifted from the javadoc and stored in the
+// * FreeMarker map. These will be available in the template returned by {@link #getTemplateName}.
+// */
+// @Override
+// protected String getTagFilterPrefix() { return GATK_JAVADOC_TAG_PREFIX; }
+//
+// /**
+// * @param workUnit the classdoc object being processed
+// * @return the name of a the freemarker template to be used for the class being documented.
+// * Must reside in the folder passed to the Barclay Doclet via the "-settings-dir" parameter to
+// * Javadoc.
+// */
+// @Override
+// public String getTemplateName(final DocWorkUnit workUnit) { return GATK_FREEMARKER_TEMPLATE_NAME; }
+//
+//
+// /**
+// * Add any custom freemarker bindings discovered via custom javadoc tags. Subclasses can override this to
+// * provide additional custom bindings.
+// *
+// * @param currentWorkUnit the work unit for the feature being documented
+// */
+// @Override
+// protected void addCustomBindings(final DocWorkUnit currentWorkUnit) {
+// super.addCustomBindings(currentWorkUnit);
+//
+// // Picard tools use the summary line for the long overview section, so extract that
+// // from Picard tools only, and put it in the freemarker map.
+// Class> toolClass = currentWorkUnit.getClazz();
+// if (picard.cmdline.CommandLineProgram.class.isAssignableFrom(toolClass)) {
+// final CommandLineProgramProperties clpProperties = currentWorkUnit.getCommandLineProperties();
+// currentWorkUnit.setProperty("picardsummary", clpProperties.summary());
+// }
+// }
+//
+//}
diff --git a/src/main/java/org/broadinstitute/hellbender/utils/help/GATKHelpDoclet.java b/src/main/java/org/broadinstitute/hellbender/utils/help/GATKHelpDoclet.java
index 69fd8588bf9..9175d6bd557 100644
--- a/src/main/java/org/broadinstitute/hellbender/utils/help/GATKHelpDoclet.java
+++ b/src/main/java/org/broadinstitute/hellbender/utils/help/GATKHelpDoclet.java
@@ -1,106 +1,106 @@
-package org.broadinstitute.hellbender.utils.help;
-
-import org.broadinstitute.barclay.help.DocumentedFeature;
-import org.broadinstitute.barclay.help.DocWorkUnit;
-import org.broadinstitute.barclay.help.GSONWorkUnit;
-import org.broadinstitute.barclay.help.HelpDoclet;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.Map;
-
-/**
- * Custom Barclay-based Javadoc Doclet used for generating GATK help/documentation.
- *
- * NOTE: Methods in this class are intended to be called by Gradle/Javadoc only, and should not be called
- * by methods that are used by the GATK runtime. This class has a dependency on com.sun.javadoc classes,
- * which may not be present since they're not provided as part of the normal GATK runtime classpath.
- */
-@SuppressWarnings("removal")
-public class GATKHelpDoclet extends HelpDoclet {
-
- private final static String GATK_FREEMARKER_INDEX_TEMPLATE_NAME = "generic.index.template.html";
- private final static String WALKER_TYPE_MAP_ENTRY = "walkertype"; // populated from javadoc custom tag
-
- /**
- * Create a doclet of the appropriate type and generate the FreeMarker templates properties.
- * @param rootDoc
- * @throws IOException
- */
- public static boolean start(final com.sun.javadoc.RootDoc rootDoc) throws IOException {
- return new GATKHelpDoclet().startProcessDocs(rootDoc);
- }
-
- /**
- * Return the name of the freemarker template to be used for the index generated by Barclay.
- * Must reside in the folder passed to the Barclay Javadc Doclet via the "-settings-dir" parameter.
- * @return name of freemarker index template
- */
- @Override
- public String getIndexTemplateName() {
- return GATK_FREEMARKER_INDEX_TEMPLATE_NAME;
- }
-
- /**
- * @return Create and return a DocWorkUnit-derived object to handle documentation
- * for the target feature(s) represented by documentedFeature.
- *
- * @param documentedFeature DocumentedFeature annotation for the target feature
- * @param classDoc javadoc classDoc for the target feature
- * @param clazz class of the target feature
- * @return DocWorkUnit to be used for this feature
- */
- @Override
- protected DocWorkUnit createWorkUnit(
- final DocumentedFeature documentedFeature,
- final com.sun.javadoc.ClassDoc classDoc,
- final Class> clazz)
- {
- return new GATKDocWorkUnit(
- new GATKHelpDocWorkUnitHandler(this),
- documentedFeature,
- classDoc,
- clazz);
- }
-
- /**
- * Create a GSONWorkUnit-derived object that holds our custom data. This method should create the object, and
- * propagate any custom javadoc tags from the template map to the newly created GSON object; specifically
- * "walkertype", which is pulled from a custom javadoc tag.
- *
- * @param workUnit work unit for which a GSON object is required
- * @param groupMaps
- * @param featureMaps
- * @return a GSONWorkUnit-derived object for this work unit, populated with any custom values
- */
- @Override
- protected GSONWorkUnit createGSONWorkUnit(
- final DocWorkUnit workUnit,
- final List