Spark 3.3.0 - WIP.

.github/actions/upload-gatk-test-results/action.yml

@@ -11,10 +11,10 @@ inputs:
     description: 'The job-unique identifier to use for uploading the results of this run into a google bucket'
     required: true
   is-docker:
-    description: 'Whether to label the uplaod artifact as coming from a docker test'
+    description: 'Whether to label the upload artifact as coming from a docker test'
     required: false
   identifier:
-    description: 'Identifier to use for finding the unique name for jobs in order to determine html logs location (eg. "Java 8 build and test integration")'
+    description: 'Identifier to use for finding the unique name for jobs in order to determine html logs location (eg. "Java 17 build and test integration")'
     required: true
 ## Secrets and token inputs
   repo-token:
@@ -23,7 +23,7 @@ inputs:
   bot-comment-key:
     description: 'Key corresponding to the user account to be used for making comments on github about test failures'
     required: false
-## option to skip all but the artifact uplaod
+## option to skip all but the artifact upload
   only-artifact:
     description: 'if "true" this will skip any uploading steps that require permissions and only upload the artifact file'
     required: false

.github/workflows/gatk-tests.yml

@@ -73,19 +73,10 @@ jobs:
     needs: check-secrets
     strategy:
       matrix:
-        java: [ 8, 11 ]
+        java: [ 17 ]
         experimental: [ false ]
-        scalaVersion: [ 2.11, 2.12 ]
+        scalaVersion: [ 2.12 ]
         testType: [ cloud, integration, unit ]
-        exclude:
-          - java: 11
-            scalaVersion: 2.11
-          - java: 8
-            scalaVersion: 2.12
-          - java: 8
-            testType: integration
-          - java: 8
-            testType: unit
       fail-fast: false
     continue-on-error: ${{ matrix.experimental }}
     env:
@@ -100,7 +91,7 @@ jobs:
         uses: actions/setup-java@v2
         with:
           java-version: ${{ matrix.Java }}
-          distribution: 'adopt'
+          distribution: 'temurin'
           cache: gradle
 
       - name: 'Compile with Gradle'
@@ -120,6 +111,10 @@ jobs:
         if: needs.check-secrets.outputs.google-credentials == 'true'
         run: echo "HELLBENDER_JSON_SERVICE_ACCOUNT_KEY=${{ steps.auth.outputs.credentials_file_path }}" >> $GITHUB_ENV
 
+      # See https://cloud.google.com/compute/docs/troubleshooting/known-issues#ubuntu-systems
+      - name: "get the updated signature key for google cloud"
+        run: curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key --keyring /usr/share/keyrings/cloud.google.gpg add -
+
       - name: 'Set up Cloud SDK'
         if: needs.check-secrets.outputs.google-credentials == 'true'
         uses: google-github-actions/setup-gcloud@v0
@@ -134,7 +129,7 @@ jobs:
         if: ${{ needs.check-secrets.outputs.google-credentials == 'true' || matrix.testType != 'cloud'}}
         id: jacoco-tests
         run: |
-          ./gradlew -Dscala.version=${{ env.SCALA_VERSION }} jacocoTestReport
+          ./gradlew --daemon -Dscala.version=${{ env.SCALA_VERSION }} jacocoTestReport
 
       - uses: ./.github/actions/upload-gatk-test-results
         if: always()
@@ -154,9 +149,9 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        java: [ 8 ]
+        java: [ 17 ]
         experimental: [ false ]
-        scalaVersion: [ 2.11 ]
+        scalaVersion: [ 2.12 ]
         testType: [ integration, unit, variantcalling, conda ]
       fail-fast: false
     continue-on-error: ${{ matrix.experimental }}
@@ -174,7 +169,7 @@ jobs:
         uses: actions/setup-java@v2
         with:
           java-version: ${{ matrix.Java }}
-          distribution: 'adopt'
+          distribution: 'temurin'
           cache: gradle
 
       #Google Cloud stuff
@@ -190,6 +185,10 @@ jobs:
         if: needs.check-secrets.outputs.google-credentials == 'true'
         run: echo "HELLBENDER_JSON_SERVICE_ACCOUNT_KEY=${{ steps.auth.outputs.credentials_file_path }}" >> $GITHUB_ENV
 
+      # See https://cloud.google.com/compute/docs/troubleshooting/known-issues#ubuntu-systems
+      - name: "get the updated signature key for google cloud"
+        run: curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key --keyring /usr/share/keyrings/cloud.google.gpg add -
+
       - name: 'Set up Cloud SDK'
         if: needs.check-secrets.outputs.google-credentials == 'true'
         uses: google-github-actions/setup-gcloud@v0
@@ -273,11 +272,11 @@ jobs:
       - uses: actions/checkout@v2
         with:
           fetch: 0
-      - name: Set up java 8
+      - name: Set up java 17
         uses: actions/setup-java@v2
         with:
-          java-version: '8'
-          distribution: 'adopt'
+          java-version: '17'
+          distribution: 'temurin'
           cache: gradle
       - uses: ./.github/actions/install-cromwell
         with:
@@ -300,11 +299,11 @@ jobs:
           fetch-depth: 0
       - name: pull lfs files
         run: git lfs pull
-      - name: Set up java 8
+      - name: Set up java 17
         uses: actions/setup-java@v2
         with:
-          java-version: '8'
-          distribution: 'adopt'
+          java-version: '17'
+          distribution: 'temurin'
           cache: gradle
       - uses: ./.github/actions/install-cromwell
         with:
@@ -335,6 +334,8 @@ jobs:
         run: |
           echo "Running CNV somatic workflows";
           bash scripts/cnv_cromwell_tests/somatic/run_cnv_somatic_workflows.sh;
+          echo "Gather CNV somatic cromwell workflows logs";
+          bash -c "find /home/runner/work/gatk/gatk/scripts/cnv_cromwell_tests/somatic/cromwell-executions -exec cat {} \;";
 
       - name: "M2_WDL_TEST"
         if: ${{ matrix.wdlTest == 'RUN_M2_WDL' }}

Dockerfile

@@ -8,11 +8,41 @@ ADD . /gatk
 WORKDIR /gatk
 
 # Get an updated gcloud signing key, in case the one in the base image has expired
-RUN curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add -
+RUN rm /etc/apt/sources.list.d/google-cloud-sdk.list
+RUN apt update
+RUN apt-key list
+RUN curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key --keyring /usr/share/keyrings/cloud.google.gpg add -
+#
+##Get Java 17 temurin JDK
+##RUN apt update && apt upgrade
+#RUN apt install wget
+#RUN wget https://github.com/adoptium/temurin17-binaries/releases/download/jdk-17.0.1%2B12/OpenJDK17U-jdk_x64_linux_hotspot_17.0.1_12.tar.gz
+#RUN tar -xvf OpenJDK17U-jdk_x64_linux_hotspot_17.*.tar.gz
+#RUN mv jdk-17.0.1+12 /opt/
+#
+#ENV JAVA_HOME /opt/jdk-17.0.1+12
+#ENV PATH $JAVA_HOME/bin:$PATH
+#RUN echo $JAVA_HOME
+#RUN update-alternatives --install /usr/bin/java java /opt/jdk-17.0.1+12/bin/java 1
+#RUN java -version
+
 RUN add-apt-repository universe && apt update
 RUN apt-get --assume-yes install git-lfs
 RUN git lfs install --force
 
+##Get Java 17 temurin JDK
+#RUN apt update && apt upgrade
+RUN apt install wget
+RUN wget https://github.com/adoptium/temurin17-binaries/releases/download/jdk-17.0.1%2B12/OpenJDK17U-jdk_x64_linux_hotspot_17.0.1_12.tar.gz
+RUN tar -xvf OpenJDK17U-jdk_x64_linux_hotspot_17.*.tar.gz
+RUN mv jdk-17.0.1+12 /opt/
+#
+ENV JAVA_HOME /opt/jdk-17.0.1+12
+ENV PATH $JAVA_HOME/bin:$PATH
+RUN echo $JAVA_HOME
+RUN update-alternatives --install /usr/bin/java java /opt/jdk-17.0.1+12/bin/java 1
+RUN java -version
+
 #Download only resources required for the build, not for testing
 RUN git lfs pull --include src/main/resources/large
 
@@ -23,6 +53,23 @@ RUN unzip -o -j $( find /gatk/unzippedJar -name "gatkPython*.zip" ) -d /gatk/unz
 # Using OpenJDK 8
 FROM broadinstitute/gatk:gatkbase-2.3.0
 
+RUN rm /etc/apt/sources.list.d/google-cloud-sdk.list
+RUN apt update
+RUN apt-key list
+
+#Get Java 17 temurin JDK
+#RUN apt update && apt upgrade
+RUN apt install wget
+RUN wget https://github.com/adoptium/temurin17-binaries/releases/download/jdk-17.0.1%2B12/OpenJDK17U-jdk_x64_linux_hotspot_17.0.1_12.tar.gz
+RUN tar -xvf OpenJDK17U-jdk_x64_linux_hotspot_17.*.tar.gz
+RUN mv jdk-17.0.1+12 /opt/
+
+ENV JAVA_HOME /opt/jdk-17.0.1+12
+ENV PATH $JAVA_HOME/bin:$PATH
+RUN echo $JAVA_HOME
+RUN update-alternatives --install /usr/bin/java java /opt/jdk-17.0.1+12/bin/java 1
+RUN java -version
+
 WORKDIR /gatk
 
 # Location of the unzipped gatk bundle files
@@ -43,6 +90,9 @@ RUN mkdir .gradle
 
 WORKDIR /gatk
 
+# use UTF-8 encoding to get around (??) scala class name shapeless/$tilde$qmark$greater$?.class
+ENV LANG="en_US.UTF-8" LANGUAGE="en_US:en" LC_ALL="en_US.UTF-8"
+
 # Create a simple unit test runner
 ENV CI true
 RUN echo "source activate gatk" > /root/run_unit_tests.sh && \
@@ -58,7 +108,7 @@ RUN echo "source activate gatk" > /root/run_unit_tests.sh && \
     echo "export CP_DIR=/gatk/testClasses" /root/run_unit_tests.sh && \
     echo "ln -s /gatkCloneMountPoint/src/ /gatkCloneMountPoint/scripts/docker/src" >> /root/run_unit_tests.sh && \
     echo "ln -s /gatkCloneMountPoint/build/ /gatkCloneMountPoint/scripts/docker/build" >> /root/run_unit_tests.sh && \
-    echo "cd /gatk/ && /gatkCloneMountPoint/gradlew -b /gatkCloneMountPoint/dockertest.gradle testOnPackagedReleaseJar jacocoTestReportOnPackagedReleaseJar -a -p /gatkCloneMountPoint" >> /root/run_unit_tests.sh
+    echo "cd /gatk/ && /gatkCloneMountPoint/gradlew -Dfile.encoding=UTF-8 -b /gatkCloneMountPoint/dockertest.gradle testOnPackagedReleaseJar jacocoTestReportOnPackagedReleaseJar -a -p /gatkCloneMountPoint" >> /root/run_unit_tests.sh
 
 WORKDIR /root
 RUN cp -r /root/run_unit_tests.sh /gatk

README.md

@@ -51,7 +51,7 @@ releases of the toolkit.
 
 ## <a name="requirements">Requirements</a>
 * To run GATK:
-    * Java 8 is needed to run or build GATK. 
+    * Java 17 is needed to run or build GATK. 
     We recommend either of the following:
         * OpenJDK 8 with Hotspot from [AdoptOpenJdk](https://adoptopenjdk.net/)
         * [OracleJDK 8](https://www.oracle.com/technetwork/java/javase/downloads/jdk8-downloads-2133151.html)
@@ -61,7 +61,7 @@ releases of the toolkit.
       See [Python Dependencies](#python) for more information.
     * R 3.2.5 (needed for producing plots in certain tools)
 * To build GATK:
-    * A Java 8 JDK
+    * A Java 17 JDK
     * Git 2.5 or greater
     * [git-lfs](https://git-lfs.github.com/) 1.1.0 or greater. Required to download the large files used to build GATK, and
       test files required to run the test suite. Run `git lfs install` after downloading, followed by `git lfs pull` from
@@ -454,7 +454,7 @@ We use [git-lfs](https://git-lfs.github.com/) to version and distribute test dat
 
 #### <a name="intellij">Creating a GATK project in the IntelliJ IDE (last tested with version 2016.2.4):</a>
 
-* Ensure that you have `gradle` and the Java 8 JDK installed
+* Ensure that you have `gradle` and the Java 17 JDK installed
 
 * You may need to install the TestNG and Gradle plugins (in preferences)
 
@@ -476,7 +476,7 @@ We use [git-lfs](https://git-lfs.github.com/) to version and distribute test dat
 
 * After downloading project dependencies, IntelliJ should open a new window with your GATK project
 
-* Make sure that the Java version is set correctly by going to File -> "Project Structure" -> "Project". Check that the "Project SDK" is set to your Java 1.8 JDK, and "Project language level" to 8 (you may need to add your Java 8 JDK under "Platform Settings" -> SDKs if it isn't there already). Then click "Apply"/"Ok".
+* Make sure that the Java version is set correctly by going to File -> "Project Structure" -> "Project". Check that the "Project SDK" is set to your Java 1.17 JDK, and "Project language level" to 17 (you may need to add your Java 17 JDK under "Platform Settings" -> SDKs if it isn't there already). Then click "Apply"/"Ok".
 
 #### <a name="debugging">Setting up debugging in IntelliJ</a>