Skip to content

Commit

Permalink
Update dependencies to address security vulnerabilities, and add a se…
Browse files Browse the repository at this point in the history
…curity scanner to build.gradle (#8607)

* Updated many GATK dependencies to address known security vulnerabilities

* Added a security scanner to build.gradle

* There are still some remaining vulnerabilities in GATK dependencies, but this eliminates most of them
  • Loading branch information
droazen committed Dec 8, 2023
1 parent bf24519 commit e2c5fab
Show file tree
Hide file tree
Showing 51 changed files with 120 additions and 102 deletions.
39 changes: 26 additions & 13 deletions build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ plugins {
id "com.github.johnrengelman.shadow" version "8.1.1" //used to build the shadow and sparkJars
id "com.github.ben-manes.versions" version "0.12.0" //used for identifying dependencies that need updating
id 'com.palantir.git-version' version '0.5.1' //version helper
id 'org.sonatype.gradle.plugins.scan' version '2.6.1' // scans for security vulnerabilities in our dependencies
}


Expand Down Expand Up @@ -60,16 +61,16 @@ final htsjdkVersion = System.getProperty('htsjdk.version','4.0.2')
final picardVersion = System.getProperty('picard.version','3.1.1')
final barclayVersion = System.getProperty('barclay.version','5.0.0')
final sparkVersion = System.getProperty('spark.version', '3.3.1')
final hadoopVersion = System.getProperty('hadoop.version', '3.3.1')
final disqVersion = System.getProperty('disq.version','0.3.6')
final hadoopVersion = System.getProperty('hadoop.version', '3.3.6')
final disqVersion = System.getProperty('disq.version','0.3.8')
final genomicsdbVersion = System.getProperty('genomicsdb.version','1.5.0')
final bigQueryVersion = System.getProperty('bigQuery.version', '2.31.0')
final bigQueryStorageVersion = System.getProperty('bigQueryStorage.version', '2.41.0')
final guavaVersion = System.getProperty('guava.version', '32.1.2-jre')
final bigQueryVersion = System.getProperty('bigQuery.version', '2.35.0')
final bigQueryStorageVersion = System.getProperty('bigQueryStorage.version', '2.47.0')
final guavaVersion = System.getProperty('guava.version', '32.1.3-jre')
final log4j2Version = System.getProperty('log4j2Version', '2.17.1')
final testNGVersion = '7.0.0'

final googleCloudNioDependency = 'com.google.cloud:google-cloud-nio:0.127.0'
final googleCloudNioDependency = 'com.google.cloud:google-cloud-nio:0.127.8'

final baseJarName = 'gatk'
final secondaryBaseJarName = 'hellbender'
Expand Down Expand Up @@ -267,12 +268,12 @@ dependencies {
// are routed to log4j
implementation 'org.apache.logging.log4j:log4j-jcl:' + log4j2Version

implementation 'org.apache.commons:commons-lang3:3.5'
implementation 'org.apache.commons:commons-math3:3.5'
implementation 'org.apache.commons:commons-lang3:3.14.0'
implementation 'org.apache.commons:commons-math3:3.6.1'
implementation 'org.hipparchus:hipparchus-stat:2.0'
implementation 'org.apache.commons:commons-collections4:4.1'
implementation 'org.apache.commons:commons-vfs2:2.0'
implementation 'org.apache.commons:commons-configuration2:2.4'
implementation 'org.apache.commons:commons-collections4:4.4'
implementation 'org.apache.commons:commons-vfs2:2.9.0'
implementation 'org.apache.commons:commons-configuration2:2.9.0'
constraints {
implementation('org.apache.commons:commons-text') {
version {
Expand All @@ -298,7 +299,7 @@ dependencies {
}

//there is no mllib_2.12.15:3.3.0, so stay use 2.12:3.3.0
implementation ('org.apache.spark:spark-mllib_2.12:3.3.0') {
implementation ('org.apache.spark:spark-mllib_2.12:3.5.0') {
// JUL is used by Google Dataflow as the backend logger, so exclude jul-to-slf4j to avoid a loop
exclude module: 'jul-to-slf4j'
exclude module: 'javax.servlet'
Expand Down Expand Up @@ -338,7 +339,7 @@ dependencies {
implementation 'org.broadinstitute:http-nio:0.1.0-rc1'

// Required for COSMIC Funcotator data source:
implementation 'org.xerial:sqlite-jdbc:3.36.0.3'
implementation 'org.xerial:sqlite-jdbc:3.44.1.0'

// natural sort
implementation('net.grey-panther:natural-comparator:1.1')
Expand Down Expand Up @@ -968,6 +969,18 @@ task gatkValidateGeneratedWdl(dependsOn: [gatkWDLGen, shadowJar]) {
}
}

// scan-gradle-plugin security vulnerability scan
ossIndexAudit {
allConfigurations = false // if true includes the dependencies in all resolvable configurations. By default is false, meaning only 'compileClasspath', 'runtimeClasspath', 'releaseCompileClasspath' and 'releaseRuntimeClasspath' are considered
useCache = true // true by default
outputFormat = 'DEFAULT' // Optional, other values are: 'DEPENDENCY_GRAPH' prints dependency graph showing direct/transitive dependencies, 'JSON_CYCLONE_DX_1_4' prints a CycloneDX 1.4 SBOM in JSON format.
showAll = false // if true prints all dependencies. By default is false, meaning only dependencies with vulnerabilities will be printed.
printBanner = true // if true will print ASCII text banner. By default is true.

// ossIndexAudit can be configured to exclude vulnerabilities from matching
// excludeVulnerabilityIds = ['39d74cc8-457a-4e57-89ef-a258420138c5'] // list containing ids of vulnerabilities to be ignored
// excludeCoordinates = ['commons-fileupload:commons-fileupload:1.3'] // list containing coordinate of components which if vulnerable should be ignored
}

/**
*This specifies what artifacts will be built and uploaded when performing a maven upload.
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
package org.broadinstitute.hellbender.tools.dragstr;

import org.apache.commons.lang.math.IntRange;
import org.apache.commons.lang3.Range;
import org.broadinstitute.hellbender.exceptions.UserException;
import org.broadinstitute.hellbender.utils.MathUtils;
import org.broadinstitute.hellbender.utils.dragstr.DragstrParams;
Expand Down Expand Up @@ -143,38 +143,38 @@ private void estimatePeriod(final int period, final DragstrParamsBuilder destina
}
}

final ArrayDeque<IntRange> pending = new ArrayDeque<>(hyperParameters.maxRepeatLength);
final ArrayDeque<Range<Integer>> pending = new ArrayDeque<>(hyperParameters.maxRepeatLength);

if (rightFlank >= leftFlank) {
// We fill 'pending' with the repeat-length groups that will be analyze:
// [1 .. leftFlank], leftFlank + 1, leftFlank + 2, ... , [rightFlank .. maxRepeats+]
pending.add(new IntRange(1, leftFlank));
pending.add(Range.of(1, leftFlank));
for (leftFlank++; leftFlank <= rightFlank; leftFlank++) {
pending.add(new IntRange(leftFlank));
pending.add(Range.is(leftFlank));
}
pending.add(new IntRange(++rightFlank, hyperParameters.maxRepeatLength));
pending.add(Range.of(++rightFlank, hyperParameters.maxRepeatLength));
} else { // no enough data we simply put all repeat lengths is one single group:
pending.add(new IntRange(1, hyperParameters.maxRepeatLength));
pending.add(Range.of(1, hyperParameters.maxRepeatLength));
}

IntRange last = null;
Range<Integer> last = null;

// Done will contain the ranges already processed.
final ArrayDeque<IntRange> done = new ArrayDeque<>(hyperParameters.maxRepeatLength);
final ArrayDeque<Range<Integer>> done = new ArrayDeque<>(hyperParameters.maxRepeatLength);

do {
final IntRange next = pending.pop();
final Range<Integer> next = pending.pop();
estimatePeriodRepeatInterval(period, next, destination, cases);
final double gp1 = destination.gp(period, next.getMinimumInteger());
final double api1 = destination.api(period, next.getMinimumInteger());
final double gp1 = destination.gp(period, next.getMinimum());
final double api1 = destination.api(period, next.getMinimum());
// if GP and API are "decreasing" with respect those from smaller repeat length
// then we accepted them:
if (last == null || (destination.gp(period, last.getMaximumInteger()) >= gp1 &&
destination.api(period, last.getMaximumInteger()) + hyperParameters.apiMonothresh >= api1)) {
if (last == null || (destination.gp(period, last.getMaximum()) >= gp1 &&
destination.api(period, last.getMaximum()) + hyperParameters.apiMonothresh >= api1)) {
done.addLast(last = next);
// if not, the we group back this repeat-length group/range with last one and re-estimate (next-loop).
} else {
pending.push(new IntRange(last.getMinimumNumber(), next.getMaximumNumber()));
pending.push(Range.of(last.getMinimum(), next.getMaximum()));
done.removeLast();
last = !done.isEmpty() ? done.getLast() : null;
}
Expand All @@ -183,14 +183,14 @@ private void estimatePeriod(final int period, final DragstrParamsBuilder destina

// Given a observed het/hom ratio the total number of reads, the variable ones and the length of the STR in bases.
// calculate the optimal gp and api that maximizes the likelihood.
private void estimatePeriodRepeatInterval(final int period, final IntRange repeatRange,
private void estimatePeriodRepeatInterval(final int period, final Range<Integer> repeatRange,
final DragstrParamsBuilder builder,
final StratifiedDragstrLocusCases cases) {
int maxApiIdx = -1;
int maxGpIdx = -1;
double maxLog10Prob = Double.NEGATIVE_INFINITY;
final int minRepeat = repeatRange.getMinimumInteger();
final int maxRepeat = repeatRange.getMaximumInteger();
final int minRepeat = repeatRange.getMinimum();
final int maxRepeat = repeatRange.getMaximum();
final int periodIdx = period - 1;
final double maxLog10PHet = log10HetOverHomVar - Math.log10(1 + hyperParameters.hetToHomRatio);
for (int i = 0; i < log10ApiValues.length; i++) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
import htsjdk.variant.variantcontext.VariantContext;
import htsjdk.variant.vcf.VCFHeader;
import htsjdk.variant.vcf.VCFHeaderLine;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.broadinstitute.hellbender.exceptions.GATKException;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@
import org.broadinstitute.hellbender.utils.tsv.TableColumnCollection;
import org.broadinstitute.hellbender.utils.tsv.TableUtils;
import org.broadinstitute.hellbender.utils.tsv.TableWriter;
import org.codehaus.plexus.util.StringUtils;

import java.io.IOException;
import java.nio.file.Path;
Expand Down Expand Up @@ -356,7 +355,15 @@ excludedOutputFields, createColumnNameToAliasesMap(Resource.getResourceContentsA

@VisibleForTesting
static String[] splitAndTrim(final String text, final String separator ) {
return Stream.of(StringUtils.split(text, separator)).map(String::trim).toArray(String[]::new);
final String[] tokens = Utils.split(text, separator).stream().map(String::trim).toArray(String[]::new);

// If we get a single empty token after trimming, return an empty array, as this is what
// the tests for this class expect:
if ( tokens.length == 1 && tokens[0].isEmpty() ) {
return new String[]{};
}

return tokens;
}

@VisibleForTesting
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package org.broadinstitute.hellbender.tools.spark;

import com.google.common.io.Files;
import htsjdk.samtools.*;
import htsjdk.samtools.BAMSBIIndexer;
import htsjdk.samtools.seekablestream.SeekableFileStream;
Expand All @@ -18,7 +19,6 @@
import org.broadinstitute.hellbender.exceptions.UserException;
import org.broadinstitute.hellbender.utils.io.IOUtils;
import org.broadinstitute.hellbender.utils.read.ReadConstants;
import org.codehaus.plexus.util.FileUtils;
import picard.cmdline.programgroups.OtherProgramGroup;

import java.io.*;
Expand Down Expand Up @@ -166,7 +166,7 @@ private static void assertBamIsCoordinateSorted(final SAMFileHeader header) {
private static void assertIsBam(final File inputBam) {
if(!BamFileIoUtils.isBamFile(inputBam)) {
throw new UserException.BadInput("A splitting index is only relevant for a bam file, but a "
+ "file with extension "+ FileUtils.getExtension(inputBam.getName()) + " was specified.");
+ "file with extension "+ Files.getFileExtension(inputBam.getName()) + " was specified.");
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
import htsjdk.samtools.CigarElement;
import htsjdk.samtools.CigarOperator;
import org.aeonbits.owner.util.Collections;
import org.apache.commons.lang.ArrayUtils;
import org.apache.commons.lang3.ArrayUtils;
import org.broadinstitute.barclay.argparser.Argument;
import org.broadinstitute.barclay.argparser.CommandLineProgramProperties;
import org.broadinstitute.barclay.argparser.ExperimentalFeature;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
import htsjdk.variant.variantcontext.Allele;
import htsjdk.variant.variantcontext.Genotype;
import htsjdk.variant.variantcontext.VariantContext;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang3.StringUtils;
import org.broadinstitute.hellbender.tools.walkers.annotator.allelespecific.*;
import org.broadinstitute.hellbender.utils.genotyper.AlleleLikelihoods;
import org.broadinstitute.hellbender.utils.read.GATKRead;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
import htsjdk.samtools.util.Locatable;
import htsjdk.variant.variantcontext.Allele;
import htsjdk.variant.variantcontext.VariantContext;
import org.apache.commons.lang.mutable.MutableInt;
import org.apache.commons.lang3.mutable.MutableInt;
import org.broadinstitute.barclay.argparser.Argument;
import org.apache.commons.lang3.tuple.Triple;
import org.broadinstitute.barclay.help.DocumentedFeature;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
import htsjdk.variant.variantcontext.Genotype;
import htsjdk.variant.variantcontext.GenotypeBuilder;
import htsjdk.variant.variantcontext.VariantContext;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang3.StringUtils;
import org.broadinstitute.barclay.help.DocumentedFeature;
import org.broadinstitute.gatk.nativebindings.smithwaterman.SWOverhangStrategy;
import org.broadinstitute.hellbender.engine.FeatureContext;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
import htsjdk.variant.variantcontext.Genotype;
import htsjdk.variant.variantcontext.GenotypeBuilder;
import htsjdk.variant.variantcontext.VariantContext;
import org.apache.commons.lang.mutable.MutableInt;
import org.apache.commons.lang3.mutable.MutableInt;
import org.broadinstitute.barclay.help.DocumentedFeature;
import org.broadinstitute.hellbender.engine.FeatureContext;
import org.broadinstitute.hellbender.engine.ReferenceContext;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
import htsjdk.variant.variantcontext.VariantContext;
import htsjdk.variant.vcf.VCFHeaderLineType;
import htsjdk.variant.vcf.VCFInfoHeaderLine;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang3.StringUtils;
import org.broadinstitute.barclay.help.DocumentedFeature;
import org.broadinstitute.hellbender.engine.ReferenceContext;
import org.broadinstitute.hellbender.utils.Utils;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
import htsjdk.variant.vcf.VCFCompoundHeaderLine;
import htsjdk.variant.vcf.VCFHeaderLine;
import htsjdk.variant.vcf.VCFInfoHeaderLine;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang3.StringUtils;
import org.broadinstitute.barclay.help.DocumentedFeature;
import org.broadinstitute.hellbender.engine.ReferenceContext;
import org.broadinstitute.hellbender.tools.walkers.annotator.AnnotationUtils;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
import htsjdk.variant.variantcontext.GenotypesContext;
import htsjdk.variant.variantcontext.VariantContext;
import htsjdk.variant.vcf.VCFConstants;
import org.apache.commons.lang.ArrayUtils;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.broadinstitute.hellbender.engine.ReferenceContext;
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
package org.broadinstitute.hellbender.tools.walkers.contamination;

import htsjdk.samtools.util.OverlapDetector;
import org.apache.commons.lang.mutable.MutableDouble;
import org.apache.commons.lang3.mutable.MutableDouble;
import org.apache.commons.lang3.tuple.ImmutablePair;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.commons.math3.optim.univariate.UnivariatePointValuePair;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
import java.util.stream.IntStream;

public class ContaminationSegmenter {
public static final Range<Double> ALT_FRACTIONS_FOR_SEGMENTATION = Range.between(0.1, 0.9);
public static final Range<Double> ALT_FRACTIONS_FOR_SEGMENTATION = Range.of(0.1, 0.9);
public static final double KERNEL_SEGMENTER_LINEAR_COST = 1.0;
public static final double KERNEL_SEGMENTER_LOG_LINEAR_COST = 1.0;
public static final int KERNEL_SEGMENTER_DIMENSION = 100;
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
package org.broadinstitute.hellbender.tools.walkers.featuremapping;

import htsjdk.samtools.CigarElement;
import org.apache.commons.lang.ArrayUtils;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.commons.text.similarity.LevenshteinDistance;
import org.broadinstitute.hellbender.engine.ReferenceContext;
import org.broadinstitute.hellbender.exceptions.GATKException;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
import com.google.common.primitives.Ints;
import htsjdk.variant.variantcontext.*;
import htsjdk.variant.vcf.VCFConstants;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.tuple.Pair;
import org.broadinstitute.hellbender.exceptions.UserException;
import org.broadinstitute.hellbender.tools.walkers.annotator.*;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
import htsjdk.samtools.util.Locatable;
import htsjdk.samtools.util.SequenceUtil;
import htsjdk.samtools.util.Tuple;
import org.apache.commons.lang.ArrayUtils;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

import com.google.common.collect.ArrayListMultimap;
import com.google.common.collect.Multimap;
import org.apache.commons.lang.mutable.MutableInt;
import org.apache.commons.lang3.mutable.MutableInt;
import org.apache.commons.lang3.tuple.ImmutablePair;
import org.apache.commons.lang3.tuple.Pair;
import org.broadinstitute.hellbender.tools.walkers.mutect.Mutect2Engine;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
import htsjdk.samtools.util.OverlapDetector;
import htsjdk.variant.variantcontext.Allele;
import htsjdk.variant.variantcontext.VariantContext;
import org.apache.commons.lang.ArrayUtils;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.commons.lang3.tuple.Pair;
import org.broadinstitute.hellbender.tools.spark.sv.utils.Strand;
import org.broadinstitute.hellbender.utils.SimpleInterval;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
import htsjdk.variant.variantcontext.VariantContextBuilder;
import htsjdk.variant.variantcontext.writer.VariantContextWriter;
import htsjdk.variant.vcf.*;
import org.apache.commons.lang.mutable.MutableInt;
import org.apache.commons.lang3.mutable.MutableInt;
import org.broadinstitute.barclay.argparser.Argument;
import org.broadinstitute.barclay.argparser.CommandLineProgramProperties;
import org.broadinstitute.barclay.help.DocumentedFeature;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

import htsjdk.variant.variantcontext.VariantContext;
import htsjdk.variant.vcf.*;
import org.apache.commons.lang.mutable.MutableLong;
import org.apache.commons.lang3.mutable.MutableLong;
import org.broadinstitute.barclay.argparser.Argument;
import org.broadinstitute.barclay.argparser.CommandLineProgramProperties;
import org.broadinstitute.barclay.help.DocumentedFeature;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
import htsjdk.variant.variantcontext.writer.VariantContextWriter;
import htsjdk.variant.vcf.*;
import org.apache.commons.collections4.Predicate;
import org.apache.commons.lang.mutable.MutableLong;
import org.apache.commons.lang3.mutable.MutableLong;
import org.broadinstitute.barclay.argparser.Argument;
import org.broadinstitute.barclay.argparser.CommandLineProgramProperties;
import org.broadinstitute.barclay.help.DocumentedFeature;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
import htsjdk.variant.variantcontext.VariantContext;
import htsjdk.variant.variantcontext.VariantContextBuilder;
import htsjdk.variant.vcf.VCFConstants;
import org.apache.commons.lang.ArrayUtils;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.commons.lang3.AnnotationUtils;
import org.apache.logging.log4j.Logger;
import org.apache.logging.log4j.LogManager;
Expand Down
Loading

0 comments on commit e2c5fab

Please sign in to comment.