Move to Java 17! (#8035)

* Improve Java Version by more than 2x! ``` _ _ _____ | | __ ___ ____ _ / |___ | _ | |/ _` \ \ / / _` | | | / / | |_| | (_| |\ V / (_| | | | / / \___/ \__,_| \_/ \__,_| |_|/_/ ``` * Update spark to 3.x * Many necessary changes to support the upgrade. --------- Co-authored-by: Louis Bergelson <[email protected]> Co-authored-by: Samuel Lee <[email protected]>
broadinstitute · Mar 2, 2023 · 330c59a · 330c59a
1 parent 4c1fb92
commit 330c59a
Show file tree

Hide file tree

Showing 74 changed files with 1,531 additions and 1,579 deletions.
diff --git a/.github/actions/upload-gatk-test-results/action.yml b/.github/actions/upload-gatk-test-results/action.yml
@@ -11,10 +11,10 @@ inputs:
     description: 'The job-unique identifier to use for uploading the results of this run into a google bucket'
     required: true
   is-docker:
-    description: 'Whether to label the uplaod artifact as coming from a docker test'
+    description: 'Whether to label the upload artifact as coming from a docker test'
     required: false
   identifier:
-    description: 'Identifier to use for finding the unique name for jobs in order to determine html logs location (eg. "Java 8 build and test integration")'
+    description: 'Identifier to use for finding the unique name for jobs in order to determine html logs location (eg. "Java 17 build and test integration")'
     required: true
 ## Secrets and token inputs
   repo-token:
@@ -23,7 +23,7 @@ inputs:
   bot-comment-key:
     description: 'Key corresponding to the user account to be used for making comments on github about test failures'
     required: false
-## option to skip all but the artifact uplaod
+## option to skip all but the artifact upload
   only-artifact:
     description: 'if "true" this will skip any uploading steps that require permissions and only upload the artifact file'
     required: false

diff --git a/.github/workflows/gatk-tests.yml b/.github/workflows/gatk-tests.yml
@@ -7,7 +7,7 @@ on:
   workflow_dispatch:
 
 env:
-  CROMWELL_VERSION: 51
+  CROMWELL_VERSION: 84
   TERM: dumb
   GRADLE_OPTS: "-Xmx2048m -Dorg.gradle.daemon=false"
   HELLBENDER_TEST_INPUTS: gs://hellbender/test/resources/
@@ -73,19 +73,10 @@ jobs:
     needs: check-secrets
     strategy:
       matrix:
-        java: [ 8, 11.0.11+9 ]
+        java: [ 17.0.6+10 ]
         experimental: [ false ]
-        scalaVersion: [ 2.11, 2.12 ]
+        scalaVersion: [ 2.13 ]
         testType: [ cloud, integration, unit ]
-        exclude:
-          - java: 11.0.11+9
-            scalaVersion: 2.11
-          - java: 8
-            scalaVersion: 2.12
-          - java: 8
-            testType: integration
-          - java: 8
-            testType: unit
       fail-fast: false
     continue-on-error: ${{ matrix.experimental }}
     env:
@@ -100,7 +91,7 @@ jobs:
         uses: actions/setup-java@v2
         with:
           java-version: ${{ matrix.Java }}
-          distribution: 'adopt'
+          distribution: 'temurin'
           cache: gradle
 
       - name: 'Compile with Gradle'
@@ -134,7 +125,7 @@ jobs:
         if: ${{ needs.check-secrets.outputs.google-credentials == 'true' || matrix.testType != 'cloud'}}
         id: jacoco-tests
         run: |
-          ./gradlew -Dscala.version=${{ env.SCALA_VERSION }} jacocoTestReport
+          ./gradlew --daemon -Dscala.version=${{ env.SCALA_VERSION }} jacocoTestReport
 
       - uses: ./.github/actions/upload-gatk-test-results
         if: always()
@@ -154,9 +145,9 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        java: [ 8 ]
+        java: [ 17.0.6+10 ]
         experimental: [ false ]
-        scalaVersion: [ 2.11 ]
+        scalaVersion: [ 2.13 ]
         testType: [ integration, unit, variantcalling, conda ]
       fail-fast: false
     continue-on-error: ${{ matrix.experimental }}
@@ -174,7 +165,7 @@ jobs:
         uses: actions/setup-java@v2
         with:
           java-version: ${{ matrix.Java }}
-          distribution: 'adopt'
+          distribution: 'temurin'
           cache: gradle
 
       #Google Cloud stuff
@@ -259,6 +250,12 @@ jobs:
       - uses: actions/checkout@v2
         with:
           fetch: 0
+      - name: Set up java 17
+        uses: actions/setup-java@v2
+        with:
+          java-version: '17'
+          distribution: 'temurin'
+          cache: gradle
       - uses: ./.github/actions/install-cromwell
         with:
           CROMWELL_VERSION: ${{ env.CROMWELL_VERSION }}
@@ -273,11 +270,11 @@ jobs:
       - uses: actions/checkout@v2
         with:
           fetch: 0
-      - name: Set up java 8
+      - name: Set up java 17
         uses: actions/setup-java@v2
         with:
-          java-version: '8'
-          distribution: 'adopt'
+          java-version: '17'
+          distribution: 'temurin'
           cache: gradle
       - uses: ./.github/actions/install-cromwell
         with:
@@ -300,11 +297,11 @@ jobs:
           fetch-depth: 0
       - name: pull lfs files
         run: git lfs pull
-      - name: Set up java 8
+      - name: Set up java 17
         uses: actions/setup-java@v2
         with:
-          java-version: '8'
-          distribution: 'adopt'
+          java-version: '17'
+          distribution: 'temurin'
           cache: gradle
       - uses: ./.github/actions/install-cromwell
         with:

diff --git a/Dockerfile b/Dockerfile
@@ -8,11 +8,27 @@ ADD . /gatk
 WORKDIR /gatk
 
 # Get an updated gcloud signing key, in case the one in the base image has expired
-RUN curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add -
+RUN rm /etc/apt/sources.list.d/google-cloud-sdk.list
+RUN apt update
+RUN apt-key list
+RUN curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key --keyring /usr/share/keyrings/cloud.google.gpg add -
 RUN add-apt-repository universe && apt update
 RUN apt-get --assume-yes install git-lfs
 RUN git lfs install --force
 
+##Get Java 17 temurin JDK
+#RUN apt update && apt upgrade
+RUN apt install wget
+RUN wget https://github.com/adoptium/temurin17-binaries/releases/download/jdk-17.0.6%2B10/OpenJDK17U-jdk_x64_linux_hotspot_17.0.6_10.tar.gz
+RUN tar -xvf OpenJDK17U-jdk_x64_linux_hotspot_17.*.tar.gz
+RUN mv jdk-17.0.6+10 /opt/
+#
+ENV JAVA_HOME /opt/jdk-17.0.6+10
+ENV PATH $JAVA_HOME/bin:$PATH
+RUN echo $JAVA_HOME
+RUN update-alternatives --install /usr/bin/java java /opt/jdk-17.0.6+10/bin/java 1
+RUN java -version
+
 #Download only resources required for the build, not for testing
 RUN git lfs pull --include src/main/resources/large
 
@@ -23,6 +39,23 @@ RUN unzip -o -j $( find /gatk/unzippedJar -name "gatkPython*.zip" ) -d /gatk/unz
 # Using OpenJDK 8
 FROM broadinstitute/gatk:gatkbase-2.3.0
 
+RUN rm /etc/apt/sources.list.d/google-cloud-sdk.list
+RUN apt update
+RUN apt-key list
+
+#Get Java 17 temurin JDK
+#RUN apt update && apt upgrade
+RUN apt install wget
+RUN wget https://github.com/adoptium/temurin17-binaries/releases/download/jdk-17.0.6%2B10/OpenJDK17U-jdk_x64_linux_hotspot_17.0.6_10.tar.gz
+RUN tar -xvf OpenJDK17U-jdk_x64_linux_hotspot_17.*.tar.gz
+RUN mv jdk-17.0.6+10 /opt/
+
+ENV JAVA_HOME /opt/jdk-17.0.6+10
+ENV PATH $JAVA_HOME/bin:$PATH
+RUN echo $JAVA_HOME
+RUN update-alternatives --install /usr/bin/java java /opt/jdk-17.0.6+10/bin/java 1
+RUN java -version
+
 WORKDIR /gatk
 
 # Location of the unzipped gatk bundle files
@@ -56,11 +89,11 @@ RUN echo "source activate gatk" > /root/run_unit_tests.sh && \
     echo "mkdir /gatk/srcdir" >> /root/run_unit_tests.sh && \
     echo "cp -rp /gatkCloneMountPoint/src/main/java/* /gatk/srcdir" >> /root/run_unit_tests.sh && \
     echo "export SOURCE_DIR=/gatk/srcdir" >> /root/run_unit_tests.sh && \
-    echo "export GRADLE_OPTS=\"-Xmx1024m -Dorg.gradle.daemon=false\"" /root/run_unit_tests.sh && \
-    echo "export CP_DIR=/gatk/testClasses" /root/run_unit_tests.sh && \
+    echo "export GRADLE_OPTS=\"-Xmx1024m -Dorg.gradle.daemon=false --add-opens java.prefs/java.util.prefs=ALL-UNNAMED\"" >> /root/run_unit_tests.sh && \
+    echo "export CP_DIR=/gatk/testClasses" >> /root/run_unit_tests.sh && \
     echo "ln -s /gatkCloneMountPoint/src/ /gatkCloneMountPoint/scripts/docker/src" >> /root/run_unit_tests.sh && \
     echo "ln -s /gatkCloneMountPoint/build/ /gatkCloneMountPoint/scripts/docker/build" >> /root/run_unit_tests.sh && \
-    echo "cd /gatk/ && /gatkCloneMountPoint/gradlew -b /gatkCloneMountPoint/dockertest.gradle testOnPackagedReleaseJar jacocoTestReportOnPackagedReleaseJar -a -p /gatkCloneMountPoint" >> /root/run_unit_tests.sh
+    echo "cd /gatk/ && /gatkCloneMountPoint/gradlew -Dfile.encoding=UTF-8 -b /gatkCloneMountPoint/dockertest.gradle testOnPackagedReleaseJar jacocoTestReportOnPackagedReleaseJar -a -p /gatkCloneMountPoint" >> /root/run_unit_tests.sh
 
 WORKDIR /root
 RUN cp -r /root/run_unit_tests.sh /gatk

diff --git a/README.md b/README.md
@@ -51,18 +51,16 @@ releases of the toolkit.
 
 ## <a name="requirements">Requirements</a>
 * To run GATK:
-    * Java 8 is needed to run or build GATK. 
+    * Java 17 is needed to run or build GATK. 
     We recommend one of the following:
-        * Download the Eclipse Foundation's distribution of OpenJDK 8 from [adoptium.net](https://adoptium.net/). Navigate to the [release archive](https://adoptium.net/temurin/archive/?version=8) to find downloads for Java 8.
-        * On Mac OS, you can install the [Homebrew package manager](https://brew.sh/) and run `brew tap homebrew/cask-versions` followed by `brew install --cask temurin8` to install the Eclipse Foundation's OpenJDK 8. 
-        * Or you can download the older [OracleJDK 8](https://www.oracle.com/technetwork/java/javase/downloads/jdk8-downloads-2133151.html)
-    which requires an Oracle account to download and comes with restrictive [license conditions](https://www.oracle.com/downloads/licenses/javase-license1.html).
+        * Download the Eclipse Foundation's distribution of OpenJDK 17 from [adoptium.net](https://adoptium.net/). Navigate to the [release archive](https://adoptium.net/temurin/archive/?version=17) to find downloads for Java 17.
+        * On Mac OS, you can install the [Homebrew package manager](https://brew.sh/) and run `brew tap homebrew/cask-versions` followed by `brew install --cask temurin17` to install the Eclipse Foundation's OpenJDK 17. 
     * Python 2.6 or greater (required to run the `gatk` frontend script)
     * Python 3.6.2, along with a set of additional Python packages, is required to run some tools and workflows.
       See [Python Dependencies](#python) for more information.
     * R 3.2.5 (needed for producing plots in certain tools)
 * To build GATK:
-    * A Java 8 JDK
+    * A Java 17 JDK
     * Git 2.5 or greater
     * [git-lfs](https://git-lfs.github.com/) 1.1.0 or greater. Required to download the large files used to build GATK, and
       test files required to run the test suite. Run `git lfs install` after downloading, followed by `git lfs pull` from
@@ -455,7 +453,7 @@ We use [git-lfs](https://git-lfs.github.com/) to version and distribute test dat
 
 #### <a name="intellij">Creating a GATK project in the IntelliJ IDE (last tested with version 2016.2.4):</a>
 
-* Ensure that you have `gradle` and the Java 8 JDK installed
+* Ensure that you have `gradle` and the Java 17 JDK installed
 
 * You may need to install the TestNG and Gradle plugins (in preferences)
 
@@ -471,13 +469,13 @@ We use [git-lfs](https://git-lfs.github.com/) to version and distribute test dat
 
 * Select "Use auto-import" and "Use default gradle wrapper".
 
-* Make sure the Gradle JVM points to Java 1.8. You may need to set this manually after creating the project, to do so find the gradle settings by clicking the wrench icon in the gradle tab on the right bar, from there edit "Gradle JVM" argument to point to Java 1.8.
+* Make sure the Gradle JVM points to Java 17. You may need to set this manually after creating the project, to do so find the gradle settings by clicking the wrench icon in the gradle tab on the right bar, from there edit "Gradle JVM" argument to point to Java 17.
 
 * Click "Finish"
 
 * After downloading project dependencies, IntelliJ should open a new window with your GATK project
 
-* Make sure that the Java version is set correctly by going to File -> "Project Structure" -> "Project". Check that the "Project SDK" is set to your Java 1.8 JDK, and "Project language level" to 8 (you may need to add your Java 8 JDK under "Platform Settings" -> SDKs if it isn't there already). Then click "Apply"/"Ok".
+* Make sure that the Java version is set correctly by going to File -> "Project Structure" -> "Project". Check that the "Project SDK" is set to your Java 17 JDK, and "Project language level" to 17 (you may need to add your Java 17 JDK under "Platform Settings" -> SDKs if it isn't there already). Then click "Apply"/"Ok".
 
 #### <a name="debugging">Setting up debugging in IntelliJ</a>