diff --git a/.github/actions/upload-gatk-test-results/action.yml b/.github/actions/upload-gatk-test-results/action.yml index efddf20019c..79c7e5b4417 100644 --- a/.github/actions/upload-gatk-test-results/action.yml +++ b/.github/actions/upload-gatk-test-results/action.yml @@ -11,10 +11,10 @@ inputs: description: 'The job-unique identifier to use for uploading the results of this run into a google bucket' required: true is-docker: - description: 'Whether to label the uplaod artifact as coming from a docker test' + description: 'Whether to label the upload artifact as coming from a docker test' required: false identifier: - description: 'Identifier to use for finding the unique name for jobs in order to determine html logs location (eg. "Java 8 build and test integration")' + description: 'Identifier to use for finding the unique name for jobs in order to determine html logs location (eg. "Java 17 build and test integration")' required: true ## Secrets and token inputs repo-token: @@ -23,7 +23,7 @@ inputs: bot-comment-key: description: 'Key corresponding to the user account to be used for making comments on github about test failures' required: false -## option to skip all but the artifact uplaod +## option to skip all but the artifact upload only-artifact: description: 'if "true" this will skip any uploading steps that require permissions and only upload the artifact file' required: false diff --git a/.github/workflows/gatk-tests.yml b/.github/workflows/gatk-tests.yml index d6db7c385ac..fde6626e9f3 100644 --- a/.github/workflows/gatk-tests.yml +++ b/.github/workflows/gatk-tests.yml @@ -7,7 +7,7 @@ on: workflow_dispatch: env: - CROMWELL_VERSION: 51 + CROMWELL_VERSION: 84 TERM: dumb GRADLE_OPTS: "-Xmx2048m -Dorg.gradle.daemon=false" HELLBENDER_TEST_INPUTS: gs://hellbender/test/resources/ @@ -73,19 +73,10 @@ jobs: needs: check-secrets strategy: matrix: - java: [ 8, 11.0.11+9 ] + java: [ 17.0.6+10 ] experimental: [ false ] - scalaVersion: [ 2.11, 2.12 ] + scalaVersion: [ 2.13 ] testType: [ cloud, integration, unit ] - exclude: - - java: 11.0.11+9 - scalaVersion: 2.11 - - java: 8 - scalaVersion: 2.12 - - java: 8 - testType: integration - - java: 8 - testType: unit fail-fast: false continue-on-error: ${{ matrix.experimental }} env: @@ -100,7 +91,7 @@ jobs: uses: actions/setup-java@v2 with: java-version: ${{ matrix.Java }} - distribution: 'adopt' + distribution: 'temurin' cache: gradle - name: 'Compile with Gradle' @@ -134,7 +125,7 @@ jobs: if: ${{ needs.check-secrets.outputs.google-credentials == 'true' || matrix.testType != 'cloud'}} id: jacoco-tests run: | - ./gradlew -Dscala.version=${{ env.SCALA_VERSION }} jacocoTestReport + ./gradlew --daemon -Dscala.version=${{ env.SCALA_VERSION }} jacocoTestReport - uses: ./.github/actions/upload-gatk-test-results if: always() @@ -154,9 +145,9 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - java: [ 8 ] + java: [ 17.0.6+10 ] experimental: [ false ] - scalaVersion: [ 2.11 ] + scalaVersion: [ 2.13 ] testType: [ integration, unit, variantcalling, conda ] fail-fast: false continue-on-error: ${{ matrix.experimental }} @@ -174,7 +165,7 @@ jobs: uses: actions/setup-java@v2 with: java-version: ${{ matrix.Java }} - distribution: 'adopt' + distribution: 'temurin' cache: gradle #Google Cloud stuff @@ -259,6 +250,12 @@ jobs: - uses: actions/checkout@v2 with: fetch: 0 + - name: Set up java 17 + uses: actions/setup-java@v2 + with: + java-version: '17' + distribution: 'temurin' + cache: gradle - uses: ./.github/actions/install-cromwell with: CROMWELL_VERSION: ${{ env.CROMWELL_VERSION }} @@ -273,11 +270,11 @@ jobs: - uses: actions/checkout@v2 with: fetch: 0 - - name: Set up java 8 + - name: Set up java 17 uses: actions/setup-java@v2 with: - java-version: '8' - distribution: 'adopt' + java-version: '17' + distribution: 'temurin' cache: gradle - uses: ./.github/actions/install-cromwell with: @@ -300,11 +297,11 @@ jobs: fetch-depth: 0 - name: pull lfs files run: git lfs pull - - name: Set up java 8 + - name: Set up java 17 uses: actions/setup-java@v2 with: - java-version: '8' - distribution: 'adopt' + java-version: '17' + distribution: 'temurin' cache: gradle - uses: ./.github/actions/install-cromwell with: diff --git a/Dockerfile b/Dockerfile index 22eda099cf0..2e85b4d8c84 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,11 +8,27 @@ ADD . /gatk WORKDIR /gatk # Get an updated gcloud signing key, in case the one in the base image has expired -RUN curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - +RUN rm /etc/apt/sources.list.d/google-cloud-sdk.list +RUN apt update +RUN apt-key list +RUN curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key --keyring /usr/share/keyrings/cloud.google.gpg add - RUN add-apt-repository universe && apt update RUN apt-get --assume-yes install git-lfs RUN git lfs install --force +##Get Java 17 temurin JDK +#RUN apt update && apt upgrade +RUN apt install wget +RUN wget https://github.com/adoptium/temurin17-binaries/releases/download/jdk-17.0.6%2B10/OpenJDK17U-jdk_x64_linux_hotspot_17.0.6_10.tar.gz +RUN tar -xvf OpenJDK17U-jdk_x64_linux_hotspot_17.*.tar.gz +RUN mv jdk-17.0.6+10 /opt/ +# +ENV JAVA_HOME /opt/jdk-17.0.6+10 +ENV PATH $JAVA_HOME/bin:$PATH +RUN echo $JAVA_HOME +RUN update-alternatives --install /usr/bin/java java /opt/jdk-17.0.6+10/bin/java 1 +RUN java -version + #Download only resources required for the build, not for testing RUN git lfs pull --include src/main/resources/large @@ -23,6 +39,23 @@ RUN unzip -o -j $( find /gatk/unzippedJar -name "gatkPython*.zip" ) -d /gatk/unz # Using OpenJDK 8 FROM broadinstitute/gatk:gatkbase-2.3.0 +RUN rm /etc/apt/sources.list.d/google-cloud-sdk.list +RUN apt update +RUN apt-key list + +#Get Java 17 temurin JDK +#RUN apt update && apt upgrade +RUN apt install wget +RUN wget https://github.com/adoptium/temurin17-binaries/releases/download/jdk-17.0.6%2B10/OpenJDK17U-jdk_x64_linux_hotspot_17.0.6_10.tar.gz +RUN tar -xvf OpenJDK17U-jdk_x64_linux_hotspot_17.*.tar.gz +RUN mv jdk-17.0.6+10 /opt/ + +ENV JAVA_HOME /opt/jdk-17.0.6+10 +ENV PATH $JAVA_HOME/bin:$PATH +RUN echo $JAVA_HOME +RUN update-alternatives --install /usr/bin/java java /opt/jdk-17.0.6+10/bin/java 1 +RUN java -version + WORKDIR /gatk # Location of the unzipped gatk bundle files @@ -56,11 +89,11 @@ RUN echo "source activate gatk" > /root/run_unit_tests.sh && \ echo "mkdir /gatk/srcdir" >> /root/run_unit_tests.sh && \ echo "cp -rp /gatkCloneMountPoint/src/main/java/* /gatk/srcdir" >> /root/run_unit_tests.sh && \ echo "export SOURCE_DIR=/gatk/srcdir" >> /root/run_unit_tests.sh && \ - echo "export GRADLE_OPTS=\"-Xmx1024m -Dorg.gradle.daemon=false\"" /root/run_unit_tests.sh && \ - echo "export CP_DIR=/gatk/testClasses" /root/run_unit_tests.sh && \ + echo "export GRADLE_OPTS=\"-Xmx1024m -Dorg.gradle.daemon=false --add-opens java.prefs/java.util.prefs=ALL-UNNAMED\"" >> /root/run_unit_tests.sh && \ + echo "export CP_DIR=/gatk/testClasses" >> /root/run_unit_tests.sh && \ echo "ln -s /gatkCloneMountPoint/src/ /gatkCloneMountPoint/scripts/docker/src" >> /root/run_unit_tests.sh && \ echo "ln -s /gatkCloneMountPoint/build/ /gatkCloneMountPoint/scripts/docker/build" >> /root/run_unit_tests.sh && \ - echo "cd /gatk/ && /gatkCloneMountPoint/gradlew -b /gatkCloneMountPoint/dockertest.gradle testOnPackagedReleaseJar jacocoTestReportOnPackagedReleaseJar -a -p /gatkCloneMountPoint" >> /root/run_unit_tests.sh + echo "cd /gatk/ && /gatkCloneMountPoint/gradlew -Dfile.encoding=UTF-8 -b /gatkCloneMountPoint/dockertest.gradle testOnPackagedReleaseJar jacocoTestReportOnPackagedReleaseJar -a -p /gatkCloneMountPoint" >> /root/run_unit_tests.sh WORKDIR /root RUN cp -r /root/run_unit_tests.sh /gatk diff --git a/README.md b/README.md index 5b6718695b3..1dc2bb366b8 100644 --- a/README.md +++ b/README.md @@ -51,18 +51,16 @@ releases of the toolkit. ## Requirements * To run GATK: - * Java 8 is needed to run or build GATK. + * Java 17 is needed to run or build GATK. We recommend one of the following: - * Download the Eclipse Foundation's distribution of OpenJDK 8 from [adoptium.net](https://adoptium.net/). Navigate to the [release archive](https://adoptium.net/temurin/archive/?version=8) to find downloads for Java 8. - * On Mac OS, you can install the [Homebrew package manager](https://brew.sh/) and run `brew tap homebrew/cask-versions` followed by `brew install --cask temurin8` to install the Eclipse Foundation's OpenJDK 8. - * Or you can download the older [OracleJDK 8](https://www.oracle.com/technetwork/java/javase/downloads/jdk8-downloads-2133151.html) - which requires an Oracle account to download and comes with restrictive [license conditions](https://www.oracle.com/downloads/licenses/javase-license1.html). + * Download the Eclipse Foundation's distribution of OpenJDK 17 from [adoptium.net](https://adoptium.net/). Navigate to the [release archive](https://adoptium.net/temurin/archive/?version=17) to find downloads for Java 17. + * On Mac OS, you can install the [Homebrew package manager](https://brew.sh/) and run `brew tap homebrew/cask-versions` followed by `brew install --cask temurin17` to install the Eclipse Foundation's OpenJDK 17. * Python 2.6 or greater (required to run the `gatk` frontend script) * Python 3.6.2, along with a set of additional Python packages, is required to run some tools and workflows. See [Python Dependencies](#python) for more information. * R 3.2.5 (needed for producing plots in certain tools) * To build GATK: - * A Java 8 JDK + * A Java 17 JDK * Git 2.5 or greater * [git-lfs](https://git-lfs.github.com/) 1.1.0 or greater. Required to download the large files used to build GATK, and test files required to run the test suite. Run `git lfs install` after downloading, followed by `git lfs pull` from @@ -455,7 +453,7 @@ We use [git-lfs](https://git-lfs.github.com/) to version and distribute test dat #### Creating a GATK project in the IntelliJ IDE (last tested with version 2016.2.4): -* Ensure that you have `gradle` and the Java 8 JDK installed +* Ensure that you have `gradle` and the Java 17 JDK installed * You may need to install the TestNG and Gradle plugins (in preferences) @@ -471,13 +469,13 @@ We use [git-lfs](https://git-lfs.github.com/) to version and distribute test dat * Select "Use auto-import" and "Use default gradle wrapper". -* Make sure the Gradle JVM points to Java 1.8. You may need to set this manually after creating the project, to do so find the gradle settings by clicking the wrench icon in the gradle tab on the right bar, from there edit "Gradle JVM" argument to point to Java 1.8. +* Make sure the Gradle JVM points to Java 17. You may need to set this manually after creating the project, to do so find the gradle settings by clicking the wrench icon in the gradle tab on the right bar, from there edit "Gradle JVM" argument to point to Java 17. * Click "Finish" * After downloading project dependencies, IntelliJ should open a new window with your GATK project -* Make sure that the Java version is set correctly by going to File -> "Project Structure" -> "Project". Check that the "Project SDK" is set to your Java 1.8 JDK, and "Project language level" to 8 (you may need to add your Java 8 JDK under "Platform Settings" -> SDKs if it isn't there already). Then click "Apply"/"Ok". +* Make sure that the Java version is set correctly by going to File -> "Project Structure" -> "Project". Check that the "Project SDK" is set to your Java 17 JDK, and "Project language level" to 17 (you may need to add your Java 17 JDK under "Platform Settings" -> SDKs if it isn't there already). Then click "Apply"/"Ok". #### Setting up debugging in IntelliJ diff --git a/build.gradle b/build.gradle index 07fb4420c9d..26d7e0a4fc3 100644 --- a/build.gradle +++ b/build.gradle @@ -21,10 +21,7 @@ plugins { import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar import de.undercouch.gradle.tasks.download.Download - -import javax.tools.ToolProvider import java.time.format.DateTimeFormatter -import java.time.ZonedDateTime mainClassName = "org.broadinstitute.hellbender.Main" @@ -48,7 +45,6 @@ task downloadGsaLibFile(type: Download) { repositories { mavenCentral() - jcenter() maven { url "https://broadinstitute.jfrog.io/broadinstitute/libs-snapshot/" //for htsjdk snapshots } @@ -60,11 +56,10 @@ repositories { mavenLocal() } -final htsjdkVersion = System.getProperty('htsjdk.version','3.0.1') -final picardVersion = System.getProperty('picard.version','2.27.5') -final barclayVersion = System.getProperty('barclay.version','4.1.0') -final sparkVersion = System.getProperty('spark.version', '2.4.5') -final scalaVersion = System.getProperty('scala.version', '2.11') +final htsjdkVersion = System.getProperty('htsjdk.version','3.0.5') +final picardVersion = System.getProperty('picard.version','3.0.0') +final barclayVersion = System.getProperty('barclay.version','5.0.0') +final sparkVersion = System.getProperty('spark.version', '3.3.1') final hadoopVersion = System.getProperty('hadoop.version', '3.3.1') final disqVersion = System.getProperty('disq.version','0.3.6') final genomicsdbVersion = System.getProperty('genomicsdb.version','1.4.4') @@ -138,19 +133,13 @@ def looksLikeWereInAGitRepository(){ // Ensure that we have a clone of the git repository, and resolve any required git-lfs // resource files that are needed to run the build but are still lfs stub files. def ensureBuildPrerequisites(largeResourcesFolder, buildPrerequisitesMessage, skipGitCheck) { - if (!JavaVersion.current().isJava8Compatible()) { - throw new GradleException( - "Java 8 or later is required to build GATK, but ${JavaVersion.current()} was found. " - + "$buildPrerequisitesMessage") + if (!JavaVersion.current().equals(JavaVersion.VERSION_17)) { + println("Warning: using Java ${JavaVersion.current()} but only Java 17 has been tested.") } - // Make sure we can get a ToolProvider class loader (for Java 8). If not we may have just a JRE. - if (JavaVersion.current().isJava8() && ToolProvider.getSystemToolClassLoader() == null) { + if (!JavaVersion.current().isCompatibleWith(JavaVersion.VERSION_17)) { throw new GradleException( - "The ClassLoader obtained from the Java ToolProvider is null. " - + "A full Java 8 or 11 JDK must be installed, check that you are not using a JRE. $buildPrerequisitesMessage") - } - if (!JavaVersion.current().isJava8() && !JavaVersion.current().isJava11()) { - println("Warning: using Java ${JavaVersion.current()} but only Java 8 and Java 11 have been tested.") + "A Java 17 compatible (Java 17 or later) version is required to build GATK, but ${JavaVersion.current()} was found. " + + "$buildPrerequisitesMessage") } if (!skipGitCheck && !looksLikeWereInAGitRepository() ) { throw new GradleException("This doesn't appear to be a git folder. " + @@ -182,11 +171,14 @@ configurations.all { // force testng dependency so we don't pick up a different version via GenomicsDB force 'org.testng:testng:' + testNGVersion force 'org.broadinstitute:barclay:' + barclayVersion - force 'com.twitter:chill_2.11:0.8.1' + force 'com.twitter:chill_2.12:0.10.0' + force 'org.apache.commons:commons-math3:3.5' // make sure we don't pick up an incorrect version of the GATK variant of the google-nio library // via Picard, etc. force googleCloudNioDependency + + force 'com.esotericsoftware:kryo:4.0.0' } all*.exclude group: 'org.slf4j', module: 'slf4j-jdk14' //exclude this to prevent slf4j complaining about to many slf4j bindings all*.exclude group: 'com.google.guava', module: 'guava-jdk5' @@ -225,28 +217,21 @@ configurations { // exclude Hadoop and Spark dependencies, since they are provided when running with Spark // (ref: http://unethicalblogger.com/2015/07/15/gradle-goodness-excluding-depends-from-shadow.html) exclude group: 'org.apache.hadoop' - exclude module: 'spark-core_2.11' + exclude module: 'spark-core_2.12' exclude group: 'org.slf4j' exclude module: 'jul-to-slf4j' exclude module: 'javax.servlet' exclude module: 'servlet-api' exclude group: 'com.esotericsoftware.kryo' - exclude module: 'spark-mllib_2.11' + exclude module: 'spark-mllib_2.12.15' exclude group: 'org.scala-lang' exclude module: 'kryo' } } -// Get the jdk files we need to run javaDoc. We need to use these during compile, testCompile, -// test execution, and gatkDoc generation, but we don't want them as part of the runtime -// classpath and we don't want to redistribute them in the uber jar. -final javadocJDKFiles = ToolProvider.getSystemToolClassLoader() == null ? files([]) : files(((URLClassLoader) ToolProvider.getSystemToolClassLoader()).getURLs()) - dependencies { - // javadoc utilities; compile/test only to prevent redistribution of sdk jars - compileOnly(javadocJDKFiles) - testImplementation(javadocJDKFiles) + implementation ('org.freemarker:freemarker:2.3.30') implementation 'org.broadinstitute:barclay:' + barclayVersion // Library for configuration: implementation 'org.aeonbits.owner:owner:1.0.9' @@ -312,7 +297,9 @@ dependencies { implementation ('org.ojalgo:ojalgo-commons-math3:1.0.0') { exclude group: 'org.apache.commons' } - implementation ('org.apache.spark:spark-mllib_' + scalaVersion + ':' + sparkVersion) { + + //there is no mllib_2.12.15:3.3.0, so stay use 2.12:3.3.0 + implementation ('org.apache.spark:spark-mllib_2.12:3.3.0') { // JUL is used by Google Dataflow as the backend logger, so exclude jul-to-slf4j to avoid a loop exclude module: 'jul-to-slf4j' exclude module: 'javax.servlet' @@ -321,7 +308,7 @@ dependencies { implementation 'com.thoughtworks.paranamer:paranamer:2.8' implementation 'org.bdgenomics.bdg-formats:bdg-formats:0.5.0' - implementation('org.bdgenomics.adam:adam-core-spark2_' + scalaVersion + ':0.28.0') { + implementation('org.bdgenomics.adam:adam-core-spark2_2.12:0.28.0') { exclude group: 'org.slf4j' exclude group: 'org.apache.hadoop' exclude group: 'org.scala-lang' @@ -336,7 +323,7 @@ dependencies { implementation('org.apache.hadoop:hadoop-client:' + hadoopVersion) // should be a 'provided' dependency implementation('com.github.jsr203hadoop:jsr203hadoop:1.0.3') - implementation('de.javakaffee:kryo-serializers:0.41') { + implementation('de.javakaffee:kryo-serializers:0.45') { exclude module: 'kryo' // use Spark's version } @@ -350,17 +337,6 @@ dependencies { implementation('com.github.fommil.netlib:netlib-native_system-linux-x86_64:1.1:natives') implementation('com.github.fommil.netlib:netlib-native_system-osx-x86_64:1.1:natives') - // Dependency change for including MLLib - implementation('com.esotericsoftware:kryo:3.0.3'){ - exclude group: 'com.esotericsoftware', module: 'reflectasm' - exclude group: 'org.ow2.asm', module: 'asm' - } - - // Dependency change for including MLLib - implementation('com.esotericsoftware:reflectasm:1.10.0:shaded') { - transitive = false - } - implementation('com.intel.gkl:gkl:0.8.8') { exclude module: 'htsjdk' } @@ -375,7 +351,7 @@ dependencies { // natural sort implementation('net.grey-panther:natural-comparator:1.1') - implementation('com.fasterxml.jackson.module:jackson-module-scala_' + scalaVersion + ':2.9.8') + implementation('com.fasterxml.jackson.module:jackson-module-scala_2.12:2.9.8') testUtilsImplementation sourceSets.main.output testUtilsImplementation 'org.testng:testng:' + testNGVersion @@ -387,6 +363,99 @@ dependencies { testImplementation "com.google.jimfs:jimfs:1.1" } +// This list needs to be kept in sync with the corresponding list in scripts/dockertest.gradle. +// +// The --add-open directives required to run GATK. These directives need to be: +// - included as properties in the manifest file in the jar(s) +// - passed to java via the gradle "jvmArgs" task property for any task that executes GATK code from a +// classpath that does use a jar (i.e., gradle run, test, javadoc, gatkDoc and jacoco tasks, etc.) +// - passed as java command line args when running from classes directly +// - included in any IntelliJ run/debug/profile configurations +// +final runtimeAddOpens = [ + // taken from the union of everything encountered by tests, plus everything defined here: + // https://github.com/apache/spark/blob/v3.3.0/launcher/src/main/java/org/apache/spark/launcher/JavaModuleOptions.java + 'java.base/java.lang=ALL-UNNAMED', + 'java.base/java.lang.invoke=ALL-UNNAMED', + 'java.base/java.lang.reflect=ALL-UNNAMED', + 'java.base/java.io=ALL-UNNAMED', + 'java.base/java.net=ALL-UNNAMED', + 'java.base/java.nio=ALL-UNNAMED', + 'java.base/java.util=ALL-UNNAMED', + 'java.base/java.util.concurrent=ALL-UNNAMED', + 'java.base/java.util.concurrent.atomic=ALL-UNNAMED', + 'java.base/sun.nio.ch=ALL-UNNAMED', + 'java.base/sun.nio.cs=ALL-UNNAMED', + 'java.base/sun.security.action=ALL-UNNAMED', + 'java.base/sun.util.calendar=ALL-UNNAMED', + 'java.base/sun.nio.fs=ALL-UNNAMED', + 'java.base/java.nio.channels.spi=ALL-UNNAMED', + 'java.base/jdk.internal.ref=ALL-UNNAMED', + 'java.base/java.lang.ref=ALL-UNNAMED', + 'java.base/java.util.zip=ALL-UNNAMED', + 'java.base/java.util.jar=ALL-UNNAMED', + 'java.base/java.nio.file.attribute=ALL-UNNAMED', + 'java.base/jdk.internal.loader=ALL-UNNAMED', + 'java.base/sun.net.www.protocol.jar=ALL-UNNAMED', + 'java.base/sun.invoke.util=ALL-UNNAMED', + 'java.base/java.util.concurrent.locks=ALL-UNNAMED', + 'java.base/java.security=ALL-UNNAMED', + 'java.base/sun.reflect.annotation=ALL-UNNAMED', + 'java.base/java.text=ALL-UNNAMED', + 'java.base/java.nio.charset=ALL-UNNAMED', + 'java.base/sun.reflect.generics.reflectiveObjects=ALL-UNNAMED', + 'java.management/com.sun.jmx.mbeanserver=ALL-UNNAMED', + 'java.management/javax.management=ALL-UNNAMED', + 'java.base/java.util.regex=ALL-UNNAMED', + 'java.base/sun.util.locale=ALL-UNNAMED', + 'java.base/jdk.internal.math=ALL-UNNAMED', + 'java.xml/com.sun.xml.internal.stream.util=ALL-UNNAMED', + 'java.base/java.time=ALL-UNNAMED', + 'java.base/sun.reflect.generics.factory=ALL-UNNAMED', + 'java.base/java.nio.channels=ALL-UNNAMED', + 'java.base/sun.security.util=ALL-UNNAMED', + 'java.base/java.time.zone=ALL-UNNAMED', + 'java.base/sun.reflect.generics.scope=ALL-UNNAMED', + 'java.base/sun.reflect.generics.tree=ALL-UNNAMED', + 'java.management/com.sun.jmx.interceptor=ALL-UNNAMED', + 'java.management/javax.management.openmbean=ALL-UNNAMED', + 'java.management/sun.management=ALL-UNNAMED', + 'jdk.management/com.sun.management.internal=ALL-UNNAMED', + 'jdk.management.jfr/jdk.management.jfr=ALL-UNNAMED', + 'jdk.jfr/jdk.jfr.internal.management=ALL-UNNAMED', + 'java.base/jdk.internal.module=ALL-UNNAMED', + 'java.base/java.lang.module=ALL-UNNAMED', + 'java.security.jgss/sun.security.krb5=ALL-UNNAMED' + ] + +final testAddOpens = [ + 'java.prefs/java.util.prefs=ALL-UNNAMED' // required for jacoco tasks +] + +run { + // transform the list of runtime configuration --add-opens args into command line argument format + final runtimeJVMArgs = runtimeAddOpens.stream() + .flatMap(openSpec -> ['--add-opens', openSpec].stream()) + .toList() + // add in any other required args + runtimeJVMArgs.add('-Dio.netty.tryReflectionSetAccessible=true') + jvmArgs = runtimeJVMArgs +} + +test { + // transform the list test configuration --add-opens (which must include both the runtime and test args) into + // command line argument format + final testJVMAddOpens = new ArrayList<>(); + testJVMAddOpens.addAll(runtimeAddOpens); + testJVMAddOpens.addAll(testAddOpens); + final testConfigurationJVMArgs = testJVMAddOpens.stream() + .flatMap(openSpec -> ['--add-opens', openSpec].stream()) + .toList() + // add in any other required args + testConfigurationJVMArgs.add('-Dio.netty.tryReflectionSetAccessible=true') + jvmArgs = testConfigurationJVMArgs +} + //add gatk launcher script to the jar as a resource processResources { from("gatk") @@ -399,8 +468,8 @@ processTestResources { include "org/broadinstitute/hellbender/utils/io/*" } -sourceCompatibility = 1.8 -targetCompatibility = 1.8 +sourceCompatibility = 1.17 +targetCompatibility = 1.17 def createSymlinks(archivePath, symlinkLocation) { exec { @@ -424,15 +493,22 @@ logger.info("build for version:" + version) group = 'org.broadinstitute' tasks.withType(Jar) { + // transform the list of --add-opens directives into manifest format, which requires only the source + // package (unlike the command line equivalent, in the manifest the "ALL-UNNAMED" target is implied + // and can't be included in the manifest entry syntax) + final manifestAddOpens = runtimeAddOpens.stream() + .map(o -> o.substring(0, (o.length() - "ALL-UNNAMED".length()) - 1)) + .collect(java.util.stream.Collectors.joining(' ')) manifest { attributes 'Implementation-Title': 'The Genome Analysis Toolkit (GATK)', - 'Implementation-Version': archiveVersion, + 'Implementation-Version': archiveVersion.get(), 'Toolkit-Short-Name' : 'GATK', 'Main-Class': project.mainClassName, 'Picard-Version': picardVersion, 'htsjdk-Version': htsjdkVersion, 'Spark-Version': sparkVersion, - 'Multi-Release': 'true' + 'Multi-Release': 'true', + 'Add-Opens': manifestAddOpens } } @@ -496,6 +572,7 @@ task shadowTestClassJar(type: ShadowJar){ // A minimal jar that only contains the extra dependencies needed for running the tests task shadowTestJar(type: ShadowJar){ + dependsOn 'compileTestUtilsJava', 'processTestUtilsResources' group = "Shadow" description = " A minimal jar that only contains the extra dependencies needed for running the tests that arent packaged in the main shadow jar" from { @@ -635,7 +712,7 @@ tasks.withType(Javadoc) { } javadoc { - // This is a hack to disable the java 8 default javadoc lint until we fix the html formatting + // This is a hack to disable the java default javadoc lint until we fix the html formatting // We only want to do this for the javadoc task, not gatkDoc options.addStringOption('Xdoclint:none', '-quiet') source = sourceSets.main.allJava + files(configurations.externalSourceConfiguration.collect { zipTree(it) }) @@ -644,7 +721,7 @@ javadoc { task testUtilsJavadoc(type: Javadoc) { - // This is a hack to disable the java 8 default javadoc lint until we fix the html formatting + // This is a hack to disable the java default javadoc lint until we fix the html formatting // We only want to do this for the javadoc task, not gatkDoc options.addStringOption('Xdoclint:none', '-quiet') source = sourceSets.testUtils.allJava @@ -687,8 +764,8 @@ task gatkDoc(type: Javadoc, dependsOn: classes) { include '**/*.java' // The gatkDoc process instantiates any documented feature classes, so to run it we need the entire - // runtime classpath, as well as jdk javadoc files such as tools.jar, where com.sun.javadoc lives. - classpath = sourceSets.main.runtimeClasspath + javadocJDKFiles + // runtime classpath. + classpath = sourceSets.main.runtimeClasspath options.docletpath = classpath.asType(List) options.doclet = "org.broadinstitute.hellbender.utils.help.GATKHelpDoclet" @@ -706,7 +783,7 @@ task gatkDoc(type: Javadoc, dependsOn: classes) { options.addStringOption("index-file-extension", phpExtension) } options.addStringOption("absolute-version", getVersion()) - options.addStringOption("build-timestamp", ZonedDateTime.now().format(DateTimeFormatter.RFC_1123_DATE_TIME)) + options.addStringOption("build-timestamp", ZonedDateTime.now().format(DateTimeFormatter.RFC_1123_DATE_TIME).toString()) } // Generate GATK Bash Tab Completion File @@ -726,8 +803,8 @@ task gatkTabComplete(type: Javadoc, dependsOn: classes) { include '**/*.java' // The gatkDoc process instantiates any documented feature classes, so to run it we need the entire - // runtime classpath, as well as jdk javadoc files such as tools.jar, where com.sun.javadoc lives, and Picard. - classpath = sourceSets.main.runtimeClasspath + javadocJDKFiles + // runtime classpath, including Picard. + classpath = sourceSets.main.runtimeClasspath options.docletpath = classpath.asType(List) options.doclet = "org.broadinstitute.barclay.help.BashTabCompletionDoclet" @@ -788,8 +865,8 @@ task gatkWDLGen(type: Javadoc, dependsOn: classes) { include '**/*.java' // The gatkWDLGen process instantiates any documented feature classes, so to run it we need the entire - // runtime classpath, as well as jdk javadoc files such as tools.jar, where com.sun.javadoc lives. - classpath = sourceSets.main.runtimeClasspath + javadocJDKFiles + // runtime classpath, including picard. + classpath = sourceSets.main.runtimeClasspath options.docletpath = classpath.asType(List) options.doclet = "org.broadinstitute.hellbender.utils.help.GATKWDLDoclet" diff --git a/gatk b/gatk index 45559912047..c0c455555af 100755 --- a/gatk +++ b/gatk @@ -39,10 +39,12 @@ EXTRA_JAVA_OPTIONS_SPARK= "-DGATK_STACKTRACE_ON_USER_EXCEPTION=true " \ "-Dsamjdk.use_async_io_write_tribble=false " \ "-Dsamjdk.compression_level=2 " -PACKAGED_LOCAL_JAR_OPTIONS= ["-Dsamjdk.use_async_io_read_samtools=false", +PACKAGED_LOCAL_JAR_OPTIONS= [ + "-Dsamjdk.use_async_io_read_samtools=false", "-Dsamjdk.use_async_io_write_samtools=true", "-Dsamjdk.use_async_io_write_tribble=false", - "-Dsamjdk.compression_level=2"] + "-Dsamjdk.compression_level=2" +] DEFAULT_SPARK_ARGS_PREFIX = '--conf' DEFAULT_SPARK_ARGS = { diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar index 7454180f2ae..41d9927a4d4 100644 Binary files a/gradle/wrapper/gradle-wrapper.jar and b/gradle/wrapper/gradle-wrapper.jar differ diff --git a/jitpack.yml b/jitpack.yml deleted file mode 100644 index 55044fd7bfc..00000000000 --- a/jitpack.yml +++ /dev/null @@ -1,5 +0,0 @@ -# configuration file for building snapshots and releases with jitpack.io -jdk: - - openjdk8 -before_install: - - ./scripts/prepareJitpackEnvironment.sh diff --git a/scripts/docker/dockertest.gradle b/scripts/docker/dockertest.gradle index 3623dc18b52..15b61e4c847 100644 --- a/scripts/docker/dockertest.gradle +++ b/scripts/docker/dockertest.gradle @@ -36,6 +36,88 @@ task unpackTestJar(type: Copy){ } } +// The --add-open directives required to run GATK. This list needs to be kept in sync with the corresponding +// list in the main project build.gradle. +// +final runtimeAddOpens = [ + // taken from the union of everything encountered by tests, plus everything defined here: + // https://github.com/apache/spark/blob/v3.3.0/launcher/src/main/java/org/apache/spark/launcher/JavaModuleOptions.java + 'java.base/java.lang=ALL-UNNAMED', + 'java.base/java.lang.invoke=ALL-UNNAMED', + 'java.base/java.lang.reflect=ALL-UNNAMED', + 'java.base/java.io=ALL-UNNAMED', + 'java.base/java.net=ALL-UNNAMED', + 'java.base/java.nio=ALL-UNNAMED', + 'java.base/java.util=ALL-UNNAMED', + 'java.base/java.util.concurrent=ALL-UNNAMED', + 'java.base/java.util.concurrent.atomic=ALL-UNNAMED', + 'java.base/sun.nio.ch=ALL-UNNAMED', + 'java.base/sun.nio.cs=ALL-UNNAMED', + 'java.base/sun.security.action=ALL-UNNAMED', + 'java.base/sun.util.calendar=ALL-UNNAMED', + 'java.base/sun.nio.fs=ALL-UNNAMED', + 'java.base/java.nio.channels.spi=ALL-UNNAMED', + 'java.base/jdk.internal.ref=ALL-UNNAMED', + 'java.base/java.lang.ref=ALL-UNNAMED', + 'java.base/java.util.zip=ALL-UNNAMED', + 'java.base/java.util.jar=ALL-UNNAMED', + 'java.base/java.nio.file.attribute=ALL-UNNAMED', + 'java.base/jdk.internal.loader=ALL-UNNAMED', + 'java.base/sun.net.www.protocol.jar=ALL-UNNAMED', + 'java.base/sun.invoke.util=ALL-UNNAMED', + 'java.base/java.util.concurrent.locks=ALL-UNNAMED', + 'java.base/java.security=ALL-UNNAMED', + 'java.base/sun.reflect.annotation=ALL-UNNAMED', + 'java.base/java.text=ALL-UNNAMED', + 'java.base/java.nio.charset=ALL-UNNAMED', + 'java.base/sun.reflect.generics.reflectiveObjects=ALL-UNNAMED', + 'java.management/com.sun.jmx.mbeanserver=ALL-UNNAMED', + 'java.management/javax.management=ALL-UNNAMED', + 'java.base/java.util.regex=ALL-UNNAMED', + 'java.base/sun.util.locale=ALL-UNNAMED', + 'java.base/jdk.internal.math=ALL-UNNAMED', + 'java.xml/com.sun.xml.internal.stream.util=ALL-UNNAMED', + 'java.base/java.time=ALL-UNNAMED', + 'java.base/sun.reflect.generics.factory=ALL-UNNAMED', + 'java.base/java.nio.channels=ALL-UNNAMED', + 'java.base/sun.security.util=ALL-UNNAMED', + 'java.base/java.time.zone=ALL-UNNAMED', + 'java.base/sun.reflect.generics.scope=ALL-UNNAMED', + 'java.base/sun.reflect.generics.tree=ALL-UNNAMED', + 'java.management/com.sun.jmx.interceptor=ALL-UNNAMED', + 'java.management/javax.management.openmbean=ALL-UNNAMED', + 'java.management/sun.management=ALL-UNNAMED', + 'jdk.management/com.sun.management.internal=ALL-UNNAMED', + 'jdk.management.jfr/jdk.management.jfr=ALL-UNNAMED', + 'jdk.jfr/jdk.jfr.internal.management=ALL-UNNAMED', + 'java.base/jdk.internal.module=ALL-UNNAMED', + 'java.base/java.lang.module=ALL-UNNAMED', + 'java.security.jgss/sun.security.krb5=ALL-UNNAMED' +] + +final testAddOpens = [ + 'java.prefs/java.util.prefs=ALL-UNNAMED' // required for jacoco tasks +] + +def getJVMArgs(runtimeAddOpens, testAddOpens) { + final testJVMAddOpens = new ArrayList<>(); + testJVMAddOpens.addAll(runtimeAddOpens); + testJVMAddOpens.addAll(testAddOpens); + final testConfigurationJVMArgs = testJVMAddOpens.stream() + .flatMap(openSpec -> ['--add-opens', openSpec].stream()) + .toList() + // add in any other required args + testConfigurationJVMArgs.add('-Dio.netty.tryReflectionSetAccessible=true') + return testConfigurationJVMArgs +} + +test { + jvmArgs = getJVMArgs(runtimeAddOpens, testAddOpens) + jacoco { + jvmArgs = getJVMArgs(runtimeAddOpens, testAddOpens) + } +} + task testOnPackagedReleaseJar(type: Test){ String testClassesJar = "$System.env.TEST_JAR" String testDependencyJar = "$System.env.TEST_DEPENDENCY_JAR" @@ -60,6 +142,8 @@ task testOnPackagedReleaseJar(type: Test){ classpath.each {println it} } + jvmArgs = getJVMArgs(runtimeAddOpens, testAddOpens) + classpath = files( gatkJar, testDependencyJar, testClassesJar) testClassesDirs = files(testClassesUnpacked) } diff --git a/src/main/java/org/broadinstitute/hellbender/cmdline/GATKPlugin/GATKAnnotationPluginDescriptor.java b/src/main/java/org/broadinstitute/hellbender/cmdline/GATKPlugin/GATKAnnotationPluginDescriptor.java index 9cfd91e9d71..526121739fb 100644 --- a/src/main/java/org/broadinstitute/hellbender/cmdline/GATKPlugin/GATKAnnotationPluginDescriptor.java +++ b/src/main/java/org/broadinstitute/hellbender/cmdline/GATKPlugin/GATKAnnotationPluginDescriptor.java @@ -11,11 +11,15 @@ import org.broadinstitute.hellbender.exceptions.GATKException; import org.broadinstitute.hellbender.tools.walkers.annotator.Annotation; import org.broadinstitute.hellbender.tools.walkers.annotator.PedigreeAnnotation; +import org.broadinstitute.hellbender.utils.SerializableConsumer; +import org.broadinstitute.hellbender.utils.SerializableFunction; +import org.broadinstitute.hellbender.utils.SerializablePredicate; import org.broadinstitute.hellbender.tools.walkers.annotator.flow.FlowAnnotatorBase; import org.broadinstitute.hellbender.utils.Utils; import org.broadinstitute.hellbender.utils.config.ConfigFactory; import org.broadinstitute.hellbender.utils.config.GATKConfig; +import java.io.Serializable; import java.lang.reflect.Modifier; import java.util.*; import java.util.stream.Collectors; @@ -35,7 +39,9 @@ * NOTE: this class enforces that annotations with required arguments must see their arguments, yet this is not currently tested * as no such annotations exist in the GATK. */ -public class GATKAnnotationPluginDescriptor extends CommandLinePluginDescriptor { +public class GATKAnnotationPluginDescriptor extends CommandLinePluginDescriptor implements Serializable { + private static final long serialVersionUID = 1L; + /** * At startup, set the plugin package name to the one(s) in the configuration file. */ @@ -115,7 +121,7 @@ public List getPackageNames() { public GATKAnnotationPluginDescriptor(final GATKAnnotationArgumentCollection userArgs, final List toolDefaultAnnotations, final List> toolDefaultGroups) { this.userArgs = userArgs; if (null != toolDefaultAnnotations) { - toolDefaultAnnotations.forEach(f -> { + toolDefaultAnnotations.forEach((SerializableConsumer) (f -> { final Class annotClass = f.getClass(); // anonymous classes have a 0-length simple name, and thus cannot be accessed or // controlled by the user via the command line, but they should still be valid @@ -127,15 +133,16 @@ public GATKAnnotationPluginDescriptor(final GATKAnnotationArgumentCollection use } populateAnnotationGroups(className, f); this.toolDefaultAnnotations.put(className, f); - }); + })); } if (null != toolDefaultGroups) { - toolDefaultGroups.forEach(a -> {if (a.isInterface() && a!=Annotation.class) { - this.toolDefaultGroups.add(a.getSimpleName()); - } else { - throw new GATKException(String.format("Tool specified annotation group %s is not a valid annotation group, must be an interface extending Annotation", a.getSimpleName())); - } - }); + toolDefaultGroups.forEach((SerializableConsumer>) (a -> { + if (a.isInterface() && a!=Annotation.class) { + this.toolDefaultGroups.add(a.getSimpleName()); + } else { + throw new GATKException(String.format("Tool specified annotation group %s is not a valid annotation group, must be an interface extending Annotation", a.getSimpleName())); + } + })); } } /** @@ -256,7 +263,9 @@ public Set getAllowedValuesForDescriptorHelp(String longArgName) { return allDiscoveredAnnotations.keySet(); } if (longArgName.equals(StandardArgumentDefinitions.ANNOTATIONS_TO_EXCLUDE_LONG_NAME)) { - Set annotations = toolDefaultGroups.stream().map(k -> discoveredGroups.get(k).keySet()).flatMap(Collection::stream).collect(Collectors.toSet()); + Set annotations = toolDefaultGroups.stream().map( + k -> discoveredGroups.get(k).keySet()) + .flatMap(Collection::stream).collect(Collectors.toSet()); annotations.addAll(toolDefaultAnnotations.keySet()); return annotations; } @@ -280,9 +289,11 @@ public boolean isDependentArgumentAllowed(final Class predecessorClass) { if (!isAllowed) { // Check whether any of the annotations have been added via groups (either tool default or user enabled) isAllowed = Stream.of(userArgs.getUserEnabledAnnotationGroups(), toolDefaultGroups) - .flatMap(Collection::stream) - .anyMatch(group -> - discoveredGroups.containsKey(group) && discoveredGroups.get(group).keySet().stream().anyMatch(s -> s.equals(predecessorName))); + .flatMap((SerializableFunction, Stream>) t -> t.stream()) + .anyMatch((SerializablePredicate) (group -> + discoveredGroups.containsKey(group) && + discoveredGroups.get(group).keySet().stream() + .anyMatch((SerializablePredicate) (s -> s.equals(predecessorName))))); } if (isAllowed) { // Keep track of the ones we allow so we can validate later that they weren't subsequently disabled @@ -325,29 +336,29 @@ public void validateAndResolvePlugins() throws CommandLineException { } // throw if a disabled annotation doesn't exist; warn if it wasn't enabled by the tool in the first place - userArgs.getUserDisabledAnnotationNames().forEach(s -> { + userArgs.getUserDisabledAnnotationNames().forEach((SerializableConsumer) (s -> { if (!allDiscoveredAnnotations.containsKey(s)) { throw new CommandLineException.BadArgumentValue(String.format("Disabled annotation (%s) does not exist", s)); } else if (!toolDefaultAnnotations.containsKey(s)) { logger.warn(String.format("Disabled annotation (%s) is not enabled by this tool", s)); } - }); + })); // warn if an annotation is both default and enabled by the user final Set redundantAnnots = new HashSet<>(toolDefaultAnnotations.keySet()); redundantAnnots.retainAll(userArgs.getUserEnabledAnnotationNames()); - redundantAnnots.forEach( - s -> { + redundantAnnots.forEach((SerializableConsumer) + (s -> { logger.warn(String.format("Redundant enabled annotation (%s) is enabled for this tool by default", s)); - }); + })); // warn if an annotation group is both default and enabled by the user final Set redundantGroups = new HashSet<>(toolDefaultGroups); redundantGroups.retainAll(userArgs.getUserEnabledAnnotationGroups()); - redundantGroups.forEach( - s -> { + redundantGroups.forEach((SerializableConsumer) + (s -> { logger.warn(String.format("Redundant enabled annotation group (%s) is enabled for this tool by default", s)); - }); + })); // Throw if args were specified for an annotation that was also disabled, or that was not enabled by the // tool by default. @@ -375,21 +386,21 @@ public void validateAndResolvePlugins() throws CommandLineException { }); // throw if an annotation name was specified that has no corresponding instance - userArgs.getUserEnabledAnnotationNames().forEach(s -> { + userArgs.getUserEnabledAnnotationNames().forEach((SerializableConsumer) (s -> { Annotation ta = allDiscoveredAnnotations.get(s); if (null == ta) { if (!toolDefaultAnnotations.containsKey(s)) { throw new CommandLineException("Unrecognized annotation name: " + s); } } - }); + })); // throw if an annotation group was specified that has no corresponding instance - userArgs.getUserEnabledAnnotationGroups().forEach(s -> { + userArgs.getUserEnabledAnnotationGroups().forEach((SerializableConsumer) (s -> { if (!discoveredGroups.containsKey(s)) { throw new CommandLineException("Unrecognized annotation group name: " + s); } - }); + })); // Populating the tool default annotations with the ones requested by groups for (String group : toolDefaultGroups ) { @@ -417,6 +428,7 @@ public void validateAndResolvePlugins() throws CommandLineException { allDiscoveredAnnotations.values().stream().filter(PedigreeAnnotation.class::isInstance).map(a -> a.getClass().getSimpleName()).collect(Collectors.joining(", ")))); } + //TODO: fix these lambdas to have serializable types // Populating any discovered flow annotations with the flowOrder arguments from the command line. if (flowOrder!=null && !flowOrder.isEmpty() && getResolvedInstances().stream() .filter(FlowAnnotatorBase.class::isInstance) @@ -464,8 +476,8 @@ public List getDefaultInstances() { @Override public List getResolvedInstances() { if (resolvedInstances == null) { - final SortedSet annotations = new TreeSet<>(Comparator.comparing(t -> t.getClass().getSimpleName())); - + final SortedSet annotations = new TreeSet<>(Comparator.comparing( + (SerializableFunction) t -> t.getClass().getSimpleName())); if (!userArgs.getDisableToolDefaultAnnotations()) { annotations.addAll(toolDefaultAnnotations.values()); } diff --git a/src/main/java/org/broadinstitute/hellbender/tools/HaplotypeCallerSpark.java b/src/main/java/org/broadinstitute/hellbender/tools/HaplotypeCallerSpark.java index 8ff76a1ae6c..e30ad0b62e0 100644 --- a/src/main/java/org/broadinstitute/hellbender/tools/HaplotypeCallerSpark.java +++ b/src/main/java/org/broadinstitute/hellbender/tools/HaplotypeCallerSpark.java @@ -3,7 +3,6 @@ import com.google.common.collect.Iterators; import htsjdk.samtools.SAMFileHeader; import htsjdk.samtools.SAMSequenceDictionary; -import htsjdk.samtools.reference.ReferenceSequenceFile; import htsjdk.samtools.util.FileExtensions; import htsjdk.variant.variantcontext.VariantContext; import org.apache.logging.log4j.Logger; @@ -29,6 +28,11 @@ import org.broadinstitute.hellbender.exceptions.UserException; import org.broadinstitute.hellbender.tools.walkers.annotator.Annotation; import org.broadinstitute.hellbender.tools.walkers.annotator.VariantAnnotatorEngine; +import org.broadinstitute.hellbender.tools.walkers.haplotypecaller.HaplotypeCaller; +import org.broadinstitute.hellbender.tools.walkers.haplotypecaller.HaplotypeCallerArgumentCollection; +import org.broadinstitute.hellbender.tools.walkers.haplotypecaller.HaplotypeCallerEngine; +import org.broadinstitute.hellbender.tools.walkers.haplotypecaller.ReferenceConfidenceMode; +import org.broadinstitute.hellbender.utils.SerializableSupplier; import org.broadinstitute.hellbender.tools.walkers.haplotypecaller.*; import org.broadinstitute.hellbender.utils.Utils; import org.broadinstitute.hellbender.utils.fasta.CachingIndexedFastaSequenceFile; @@ -142,7 +146,7 @@ private static void processAssemblyRegions( final VariantAnnotatorEngine variantannotatorEngine = new VariantAnnotatorEngine(annotations, hcArgs.dbsnp.dbsnp, hcArgs.comps, hcArgs.emitReferenceConfidence != ReferenceConfidenceMode.NONE, false); final Path referencePath = IOUtils.getPath(reference); - final ReferenceSequenceFile driverReferenceSequenceFile = new CachingIndexedFastaSequenceFile(referencePath); + final CachingIndexedFastaSequenceFile driverReferenceSequenceFile = new CachingIndexedFastaSequenceFile(referencePath); final HaplotypeCallerEngine hcEngine = new HaplotypeCallerEngine(hcArgs, assemblyRegionArgs, false, false, header, driverReferenceSequenceFile, variantannotatorEngine); final String referenceFileName = referencePath.getFileName().toString(); final Broadcast hcArgsBroadcast = ctx.broadcast(hcArgs); @@ -167,7 +171,7 @@ private static FlatMapFunction, VariantCon final Broadcast annotatorEngineBroadcast) { return (FlatMapFunction, VariantContext>) contexts -> { // HaplotypeCallerEngine isn't serializable but is expensive to instantiate, so construct and reuse one for every partition - final ReferenceSequenceFile taskReferenceSequenceFile = taskReferenceSequenceFile(referenceFileName); + final CachingIndexedFastaSequenceFile taskReferenceSequenceFile = taskReferenceSequenceFile(referenceFileName); final HaplotypeCallerEngine hcEngine = new HaplotypeCallerEngine(hcArgsBroadcast.value(), assemblyRegionArgsBroadcast.value(), false, false, header, taskReferenceSequenceFile, annotatorEngineBroadcast.getValue()); Iterator> iterators = Utils.stream(contexts).map(context -> { AssemblyRegion region = context.getAssemblyRegion(); @@ -194,7 +198,7 @@ protected Broadcast> assemblyRegionEvaluatorSu final Path referencePath = IOUtils.getPath(referenceArguments.getReferenceFileName()); final String referenceFileName = referencePath.getFileName().toString(); final String pathOnExecutor = SparkFiles.get(referenceFileName); - final ReferenceSequenceFile taskReferenceSequenceFile = new CachingIndexedFastaSequenceFile(IOUtils.getPath(pathOnExecutor)); + final CachingIndexedFastaSequenceFile taskReferenceSequenceFile = new CachingIndexedFastaSequenceFile(IOUtils.getPath(pathOnExecutor)); final Collection annotations = makeVariantAnnotations(); final VariantAnnotatorEngine annotatorEngine = new VariantAnnotatorEngine(annotations, hcArgs.dbsnp.dbsnp, hcArgs.comps, hcArgs.emitReferenceConfidence != ReferenceConfidenceMode.NONE, false); return assemblyRegionEvaluatorSupplierBroadcastFunction(ctx, hcArgs, assemblyRegionArgs, getHeaderForReads(), taskReferenceSequenceFile, annotatorEngine); @@ -208,12 +212,12 @@ private static Broadcast> assemblyRegionEvalua final Collection annotations) { final Path referencePath = IOUtils.getPath(reference); final String referenceFileName = referencePath.getFileName().toString(); - final ReferenceSequenceFile taskReferenceSequenceFile = taskReferenceSequenceFile(referenceFileName); + final CachingIndexedFastaSequenceFile taskReferenceSequenceFile = taskReferenceSequenceFile(referenceFileName); final VariantAnnotatorEngine annotatorEngine = new VariantAnnotatorEngine(annotations, hcArgs.dbsnp.dbsnp, hcArgs.comps, hcArgs.emitReferenceConfidence != ReferenceConfidenceMode.NONE, false); return assemblyRegionEvaluatorSupplierBroadcastFunction(ctx, hcArgs, assemblyRegionArgs, header, taskReferenceSequenceFile, annotatorEngine); } - private static ReferenceSequenceFile taskReferenceSequenceFile(final String referenceFileName) { + private static CachingIndexedFastaSequenceFile taskReferenceSequenceFile(final String referenceFileName) { final String pathOnExecutor = SparkFiles.get(referenceFileName); return new CachingIndexedFastaSequenceFile(IOUtils.getPath(pathOnExecutor)); } @@ -222,9 +226,9 @@ private static Broadcast> assemblyRegionEvalua final JavaSparkContext ctx, final HaplotypeCallerArgumentCollection hcArgs, AssemblyRegionArgumentCollection assemblyRegionArgs, final SAMFileHeader header, - final ReferenceSequenceFile taskReferenceSequenceFile, + final CachingIndexedFastaSequenceFile taskReferenceSequenceFile, final VariantAnnotatorEngine annotatorEngine) { - Supplier supplier = new Supplier() { + SerializableSupplier supplier = new SerializableSupplier() { @Override public AssemblyRegionEvaluator get() { return new HaplotypeCallerEngine(hcArgs, assemblyRegionArgs, false, false, header, taskReferenceSequenceFile, annotatorEngine); diff --git a/src/main/java/org/broadinstitute/hellbender/tools/dragstr/CalibrateDragstrModel.java b/src/main/java/org/broadinstitute/hellbender/tools/dragstr/CalibrateDragstrModel.java index cd2b7cbb601..51133a5ab90 100644 --- a/src/main/java/org/broadinstitute/hellbender/tools/dragstr/CalibrateDragstrModel.java +++ b/src/main/java/org/broadinstitute/hellbender/tools/dragstr/CalibrateDragstrModel.java @@ -263,6 +263,7 @@ private void checkSequenceDictionaryCompatibility(final SAMSequenceDictionary re } } + @SuppressWarnings("deprecation") private PrintWriter openSitesOutputWriter(final String sitesOutput) { return sitesOutput == null ? new PrintWriter(NullOutputStream.NULL_OUTPUT_STREAM) : new PrintWriter(BucketUtils.createFile(sitesOutput)); diff --git a/src/main/java/org/broadinstitute/hellbender/tools/walkers/annotator/AssemblyComplexity.java b/src/main/java/org/broadinstitute/hellbender/tools/walkers/annotator/AssemblyComplexity.java index c71d6faabb4..a33796672e0 100644 --- a/src/main/java/org/broadinstitute/hellbender/tools/walkers/annotator/AssemblyComplexity.java +++ b/src/main/java/org/broadinstitute/hellbender/tools/walkers/annotator/AssemblyComplexity.java @@ -4,7 +4,6 @@ import htsjdk.samtools.util.Locatable; import htsjdk.variant.variantcontext.Allele; import htsjdk.variant.variantcontext.VariantContext; -import jdk.nashorn.internal.runtime.regexp.joni.constants.EncloseType; import org.apache.commons.lang.mutable.MutableInt; import org.broadinstitute.barclay.argparser.Argument; import org.apache.commons.lang3.tuple.Triple; diff --git a/src/main/java/org/broadinstitute/hellbender/tools/walkers/haplotypecaller/HaplotypeCallerEngine.java b/src/main/java/org/broadinstitute/hellbender/tools/walkers/haplotypecaller/HaplotypeCallerEngine.java index 791b368075f..a7d8e2e33c2 100644 --- a/src/main/java/org/broadinstitute/hellbender/tools/walkers/haplotypecaller/HaplotypeCallerEngine.java +++ b/src/main/java/org/broadinstitute/hellbender/tools/walkers/haplotypecaller/HaplotypeCallerEngine.java @@ -25,6 +25,7 @@ import org.broadinstitute.hellbender.tools.walkers.genotyper.OutputMode; import org.broadinstitute.hellbender.tools.walkers.genotyper.StandardCallerArgumentCollection; import org.broadinstitute.hellbender.tools.walkers.haplotypecaller.readthreading.ReadThreadingAssembler; +import org.broadinstitute.hellbender.utils.fasta.CachingIndexedFastaSequenceFile; import org.broadinstitute.hellbender.utils.pileup.PileupBasedAlleles; import org.broadinstitute.hellbender.transformers.IUPACReadTransformer; import org.broadinstitute.hellbender.transformers.ReadTransformer; @@ -169,7 +170,7 @@ public class HaplotypeCallerEngine implements AssemblyRegionEvaluator { */ public HaplotypeCallerEngine(final HaplotypeCallerArgumentCollection hcArgs, AssemblyRegionArgumentCollection assemblyRegionArgs, boolean createBamOutIndex, boolean createBamOutMD5, final SAMFileHeader readsHeader, - ReferenceSequenceFile referenceReader, VariantAnnotatorEngine annotationEngine) { + CachingIndexedFastaSequenceFile referenceReader, VariantAnnotatorEngine annotationEngine) { this.dragstrParams = DragstrParamUtils.parse(hcArgs.likelihoodArgs.dragstrParams); this.hcArgs = Utils.nonNull(hcArgs); this.readsHeader = Utils.nonNull(readsHeader); diff --git a/src/main/java/org/broadinstitute/hellbender/tools/walkers/haplotypecaller/RampedHaplotypeCallerEngine.java b/src/main/java/org/broadinstitute/hellbender/tools/walkers/haplotypecaller/RampedHaplotypeCallerEngine.java index f0db1b52bac..b66bc6e7202 100644 --- a/src/main/java/org/broadinstitute/hellbender/tools/walkers/haplotypecaller/RampedHaplotypeCallerEngine.java +++ b/src/main/java/org/broadinstitute/hellbender/tools/walkers/haplotypecaller/RampedHaplotypeCallerEngine.java @@ -14,6 +14,7 @@ import org.broadinstitute.hellbender.tools.walkers.haplotypecaller.ramps.*; import org.broadinstitute.hellbender.utils.SimpleInterval; import org.broadinstitute.hellbender.utils.downsampling.ReservoirDownsampler; +import org.broadinstitute.hellbender.utils.fasta.CachingIndexedFastaSequenceFile; import org.broadinstitute.hellbender.utils.genotyper.AlleleLikelihoods; import org.broadinstitute.hellbender.utils.haplotype.EventMap; import org.broadinstitute.hellbender.utils.haplotype.Haplotype; @@ -61,7 +62,7 @@ public class RampedHaplotypeCallerEngine extends HaplotypeCallerEngine { public RampedHaplotypeCallerEngine(final HaplotypeCallerArgumentCollection hcArgs, AssemblyRegionArgumentCollection assemblyRegionArgs, boolean createBamOutIndex, boolean createBamOutMD5, final SAMFileHeader readsHeader, - ReferenceSequenceFile referenceReader, VariantAnnotatorEngine annotationEngine, + CachingIndexedFastaSequenceFile referenceReader, VariantAnnotatorEngine annotationEngine, RampedHaplotypeCallerArgumentCollection rpArgs) { super(hcArgs, assemblyRegionArgs, createBamOutIndex, diff --git a/src/main/java/org/broadinstitute/hellbender/tools/walkers/haplotypecaller/graphs/Path.java b/src/main/java/org/broadinstitute/hellbender/tools/walkers/haplotypecaller/graphs/Path.java index 90f80c04dcb..52040450438 100644 --- a/src/main/java/org/broadinstitute/hellbender/tools/walkers/haplotypecaller/graphs/Path.java +++ b/src/main/java/org/broadinstitute/hellbender/tools/walkers/haplotypecaller/graphs/Path.java @@ -2,7 +2,6 @@ import com.google.common.annotations.VisibleForTesting; import htsjdk.samtools.Cigar; -import joptsimple.internal.Strings; import org.apache.commons.lang3.ArrayUtils; import org.broadinstitute.gatk.nativebindings.smithwaterman.SWOverhangStrategy; import org.broadinstitute.gatk.nativebindings.smithwaterman.SWParameters; @@ -153,7 +152,7 @@ public boolean containsVertex(final V v) { @Override public String toString() { - final String joinedPath = Strings.join(getVertices().stream().map(BaseVertex::getSequenceString).collect(Collectors.toList()), "->"); + final String joinedPath = getVertices().stream().map(BaseVertex::getSequenceString).collect(Collectors.joining("->")); return String.format("Path{path=%s}", joinedPath); } diff --git a/src/main/java/org/broadinstitute/hellbender/utils/SerializableConsumer.java b/src/main/java/org/broadinstitute/hellbender/utils/SerializableConsumer.java new file mode 100644 index 00000000000..2f61106669e --- /dev/null +++ b/src/main/java/org/broadinstitute/hellbender/utils/SerializableConsumer.java @@ -0,0 +1,6 @@ +package org.broadinstitute.hellbender.utils; + +import java.io.Serializable; +import java.util.function.Consumer; + +public interface SerializableConsumer extends Consumer, Serializable {} diff --git a/src/main/java/org/broadinstitute/hellbender/utils/SerializablePredicate.java b/src/main/java/org/broadinstitute/hellbender/utils/SerializablePredicate.java new file mode 100644 index 00000000000..3da23d638bd --- /dev/null +++ b/src/main/java/org/broadinstitute/hellbender/utils/SerializablePredicate.java @@ -0,0 +1,6 @@ +package org.broadinstitute.hellbender.utils; + +import java.io.Serializable; +import java.util.function.Predicate; + +public interface SerializablePredicate extends Predicate, Serializable {} diff --git a/src/main/java/org/broadinstitute/hellbender/utils/SerializableSupplier.java b/src/main/java/org/broadinstitute/hellbender/utils/SerializableSupplier.java new file mode 100644 index 00000000000..b62df11bc71 --- /dev/null +++ b/src/main/java/org/broadinstitute/hellbender/utils/SerializableSupplier.java @@ -0,0 +1,6 @@ +package org.broadinstitute.hellbender.utils; + +import java.io.Serializable; +import java.util.function.Supplier; + +public interface SerializableSupplier extends Supplier, Serializable {} \ No newline at end of file diff --git a/src/main/java/org/broadinstitute/hellbender/utils/fasta/CachingIndexedFastaSequenceFile.java b/src/main/java/org/broadinstitute/hellbender/utils/fasta/CachingIndexedFastaSequenceFile.java index 1ead048346e..6cd2d678347 100644 --- a/src/main/java/org/broadinstitute/hellbender/utils/fasta/CachingIndexedFastaSequenceFile.java +++ b/src/main/java/org/broadinstitute/hellbender/utils/fasta/CachingIndexedFastaSequenceFile.java @@ -19,6 +19,7 @@ import org.broadinstitute.hellbender.utils.Utils; import java.io.IOException; +import java.io.Serializable; import java.nio.file.Files; import java.nio.file.Path; import java.util.Arrays; @@ -31,10 +32,13 @@ * * Instances of this class should be closed when they are no longer needed. */ -public final class CachingIndexedFastaSequenceFile implements ReferenceSequenceFile { +public final class CachingIndexedFastaSequenceFile implements ReferenceSequenceFile, Serializable { + private static final long serialVersionUID = 1L; + protected static final Logger logger = LogManager.getLogger(CachingIndexedFastaSequenceFile.class); - private final ReferenceSequenceFile sequenceFile; + // TODO: this gets around numerous serialization issues with non-serializable lambdas + private transient final ReferenceSequenceFile sequenceFile; /** do we want to print debugging information about cache efficiency? */ private static final boolean PRINT_EFFICIENCY = false; diff --git a/src/main/java/org/broadinstitute/hellbender/utils/help/GATKDocWorkUnit.java b/src/main/java/org/broadinstitute/hellbender/utils/help/GATKDocWorkUnit.java index 1dce5a8422b..3c1891dcc6a 100644 --- a/src/main/java/org/broadinstitute/hellbender/utils/help/GATKDocWorkUnit.java +++ b/src/main/java/org/broadinstitute/hellbender/utils/help/GATKDocWorkUnit.java @@ -3,8 +3,8 @@ import org.broadinstitute.barclay.help.DocWorkUnit; import org.broadinstitute.barclay.help.DocWorkUnitHandler; import org.broadinstitute.barclay.help.DocumentedFeature; - import org.broadinstitute.hellbender.utils.runtime.RuntimeUtils; +import javax.lang.model.element.Element; /** * Custom DocWorkUnit used for generating GATK help/documentation. Overrides the defaults to provide tool @@ -19,10 +19,10 @@ public class GATKDocWorkUnit extends DocWorkUnit { public GATKDocWorkUnit( final DocWorkUnitHandler workUnitHandler, - final DocumentedFeature documentedFeatureAnnotation, - final com.sun.javadoc.ClassDoc classDoc, - final Class clazz) { - super(workUnitHandler, documentedFeatureAnnotation, classDoc, clazz); + final Element classElement, + final Class clazz, + final DocumentedFeature documentedFeatureAnnotation) { + super(workUnitHandler, classElement, clazz, documentedFeatureAnnotation); } @Override diff --git a/src/main/java/org/broadinstitute/hellbender/utils/help/GATKGSONWorkUnit.java b/src/main/java/org/broadinstitute/hellbender/utils/help/GATKGSONWorkUnit.java index ab93c5a5670..11c5f1e5259 100644 --- a/src/main/java/org/broadinstitute/hellbender/utils/help/GATKGSONWorkUnit.java +++ b/src/main/java/org/broadinstitute/hellbender/utils/help/GATKGSONWorkUnit.java @@ -12,9 +12,7 @@ public class GATKGSONWorkUnit extends GSONWorkUnit { private String walkerType; - public GATKGSONWorkUnit(DocWorkUnit workUnit) { - super(workUnit); - } + public GATKGSONWorkUnit(DocWorkUnit workUnit) { super(workUnit); } public void setWalkerType(final String walkerType){ this.walkerType = walkerType; diff --git a/src/main/java/org/broadinstitute/hellbender/utils/help/GATKHelpDocWorkUnitHandler.java b/src/main/java/org/broadinstitute/hellbender/utils/help/GATKHelpDocWorkUnitHandler.java index ab2cf1a9d06..098e81caf75 100644 --- a/src/main/java/org/broadinstitute/hellbender/utils/help/GATKHelpDocWorkUnitHandler.java +++ b/src/main/java/org/broadinstitute/hellbender/utils/help/GATKHelpDocWorkUnitHandler.java @@ -3,7 +3,6 @@ import org.broadinstitute.barclay.argparser.CommandLineProgramProperties; import org.broadinstitute.barclay.help.DefaultDocWorkUnitHandler; import org.broadinstitute.barclay.help.DocWorkUnit; - import org.broadinstitute.barclay.help.HelpDoclet; /** diff --git a/src/main/java/org/broadinstitute/hellbender/utils/help/GATKHelpDoclet.java b/src/main/java/org/broadinstitute/hellbender/utils/help/GATKHelpDoclet.java index 9262f176c13..6e3b222f9cc 100644 --- a/src/main/java/org/broadinstitute/hellbender/utils/help/GATKHelpDoclet.java +++ b/src/main/java/org/broadinstitute/hellbender/utils/help/GATKHelpDoclet.java @@ -5,7 +5,7 @@ import org.broadinstitute.barclay.help.GSONWorkUnit; import org.broadinstitute.barclay.help.HelpDoclet; -import java.io.IOException; +import javax.lang.model.element.Element; import java.util.List; import java.util.Map; @@ -22,15 +22,6 @@ public class GATKHelpDoclet extends HelpDoclet { private final static String GATK_FREEMARKER_INDEX_TEMPLATE_NAME = "generic.index.template.html"; private final static String WALKER_TYPE_MAP_ENTRY = "walkertype"; // populated from javadoc custom tag - /** - * Create a doclet of the appropriate type and generate the FreeMarker templates properties. - * @param rootDoc - * @throws IOException - */ - public static boolean start(final com.sun.javadoc.RootDoc rootDoc) throws IOException { - return new GATKHelpDoclet().startProcessDocs(rootDoc); - } - /** * Return the name of the freemarker template to be used for the index generated by Barclay. * Must reside in the folder passed to the Barclay Javadc Doclet via the "-settings-dir" parameter. @@ -45,22 +36,22 @@ public String getIndexTemplateName() { * @return Create and return a DocWorkUnit-derived object to handle documentation * for the target feature(s) represented by documentedFeature. * - * @param documentedFeature DocumentedFeature annotation for the target feature - * @param classDoc javadoc classDoc for the target feature + * @param classElement Element for the target feature * @param clazz class of the target feature + * @param documentedFeature DocumentedFeature annotation for the target feature * @return DocWorkUnit to be used for this feature */ @Override - protected DocWorkUnit createWorkUnit( - final DocumentedFeature documentedFeature, - final com.sun.javadoc.ClassDoc classDoc, - final Class clazz) + public DocWorkUnit createWorkUnit( + final Element classElement, + final Class clazz, + final DocumentedFeature documentedFeature) { return new GATKDocWorkUnit( new GATKHelpDocWorkUnitHandler(this), - documentedFeature, - classDoc, - clazz); + classElement, + clazz, + documentedFeature); } /** diff --git a/src/main/java/org/broadinstitute/hellbender/utils/help/GATKWDLDoclet.java b/src/main/java/org/broadinstitute/hellbender/utils/help/GATKWDLDoclet.java index f0a83f00f4d..b90b1d57cf5 100644 --- a/src/main/java/org/broadinstitute/hellbender/utils/help/GATKWDLDoclet.java +++ b/src/main/java/org/broadinstitute/hellbender/utils/help/GATKWDLDoclet.java @@ -3,14 +3,16 @@ import freemarker.template.Configuration; import freemarker.template.Template; import freemarker.template.TemplateException; +import jdk.javadoc.doclet.Doclet; import org.apache.commons.io.FilenameUtils; -import org.broadinstitute.barclay.argparser.CommandLineProgramProperties; -import org.broadinstitute.barclay.argparser.WorkflowProperties; import org.broadinstitute.barclay.help.*; +import javax.lang.model.element.Element; import java.io.*; +import java.util.LinkedHashSet; import java.util.List; import java.util.Map; +import java.util.Set; /** * Custom Barclay-based Javadoc Doclet used for generating tool WDL. @@ -22,7 +24,7 @@ @SuppressWarnings("removal") public class GATKWDLDoclet extends WDLDoclet { - // emit an index file with links to all of the .wdl files + // emit an index file with links to all the .wdl files private final static String GATK_FREEMARKER_INDEX_TEMPLATE_NAME = "wdlIndexTemplate.html.ftl"; // the directory where the wdlgen build is running @@ -32,38 +34,20 @@ public class GATKWDLDoclet extends WDLDoclet { @Override public String getIndexFileExtension() { return "html"; } - /** - * Validates the given options against options supported by this doclet. - * - * @param option Option to validate. - * @return Number of potential parameters; 0 if not supported. - */ - public static int optionLength(final String option) { - // Any arguments used for the doclet need to be recognized here. Many javadoc plugins (ie. gradle) - // automatically add some such as "-doctitle", "-windowtitle", which we ignore. - if (option.equals(OPT_BUILD_DIR)) { - return 2; - } - return WDLDoclet.optionLength(option); - } - @Override - protected boolean parseOption(final String[] option) { - if (option[0].equals(OPT_BUILD_DIR)) { - buildDir = option[1]; - return true; - } else { - return super.parseOption(option); - } - } - - /** - * Create a WDL doclet and generate the FreeMarker templates properties. - * @param rootDoc - * @throws IOException - */ - public static boolean start(final com.sun.javadoc.RootDoc rootDoc) throws IOException { - return new GATKWDLDoclet().startProcessDocs(rootDoc); + public Set getSupportedOptions() { + final Set