Merge branch 'develop'

.gitattributes

@@ -3,3 +3,4 @@
 *.MD text
 *.java text
 *.html text
+docs/api/RESTAPI.md linguist-generated=true

.github/issue_template.md

@@ -1,27 +1,13 @@
-###
-### IMPORTANT: Please file new issues over in our Jira issue tracker!
-###
-### https://broadworkbench.atlassian.net/projects/BA/issues
-###
-### You may need to create an account before you can view/create issues.
-###
-
 <!--
 Hi!  Thanks for taking the time to report feedback.
 
-Before posting an issue over in Jira tracker, please check whether your question is already answered in our:
-  forum  https://gatkforums.broadinstitute.org/wdl/categories/ask-the-wdl-team
-  documentation http://cromwell.readthedocs.io/en/develop/
-
-Other forums:
-FireCloud https://gatkforums.broadinstitute.org/firecloud/categories/ask-the-firecloud-team
-WDL https://gatkforums.broadinstitute.org/wdl/categories/ask-the-wdl-team
-CWL https://www.biostars.org/
+Please check whether your question is already answered in our:
+  Documentation  http://cromwell.readthedocs.io/en/develop/
+  Bioinformatics Stack Exchange  https://bioinformatics.stackexchange.com/search?q=cromwell
+  Slack  https://join.slack.com/t/cromwellhq/shared_invite/zt-dxmmrtye-JHxwKE53rfKE_ZWdOHIB4g
 -->
 
-<!-- Are you seeing something that looks like a bug?  Then great!  You're almost in the right place.  -->
-
-<!-- You'll want to go to https://broadworkbench.atlassian.net/projects/BA/issues and then tell us: -->
+<!-- Are you seeing something that looks like a bug?  Please attach as much information as possible.  -->
 
 <!-- Which backend are you running?  -->
 

.github/workflows/chart_update_on_merge.yml

@@ -0,0 +1,71 @@
+name: chart-update-on-merge
+
+on:
+  pull_request:
+    types:
+      - closed
+
+jobs:
+  chart-update:
+    name: Cromwhelm Chart Auto Updater
+    if: github.event.pull_request.merged == true
+    runs-on: self-hosted # Faster machines; see https://github.com/broadinstitute/cromwell/settings/actions/runners
+    steps:
+    - name: Clone Cromwell
+      uses: actions/checkout@v2
+      with:
+        repository: broadinstitute/cromwell
+        token: ${{ secrets.BROADBOT_GITHUB_TOKEN }} # Has to be set at checkout AND later when pushing to work
+        path: cromwell
+    - uses: olafurpg/setup-scala@v10
+      with:
+        java-version: [email protected]
+    - name: Clone Cromwhelm
+      uses: actions/checkout@v2
+      with:
+        repository: broadinstitute/cromwhelm
+        token: ${{ secrets.BROADBOT_GITHUB_TOKEN }} # Has to be set at checkout AND later when pushing to work
+        path: cromwhelm
+    - name: Find Cromwell short SHA
+      run: |
+        set -e
+        cd cromwell
+        echo "CROMWELL_SHORT_SHA=`git rev-parse --short $GITHUB_SHA`" >> $GITHUB_ENV
+    - name: Find Cromwell release number
+      run: |
+        set -e
+        previous_version=$(curl -X GET https://api.github.com/repos/broadinstitute/cromwell/releases/latest | jq .tag_name | xargs)
+        if ! [[ "${previous_version}" =~ ^[0-9][0-9]+$ ]]; then
+          exit 1
+        fi
+        echo "CROMWELL_NUMBER=$((previous_version + 1))" >> $GITHUB_ENV
+    - name: Save complete image ID
+      run: |
+        echo "CROMWELL_SNAP_VERSION=`echo "$CROMWELL_NUMBER-$CROMWELL_SHORT_SHA-SNAP"`" >> $GITHUB_ENV
+    # `DSDEJENKINS_PASSWORD` auto syncs from vault with https://github.com/broadinstitute/terraform-ap-deployments/pull/614
+    - name: Login to Docker Hub
+      uses: docker/login-action@v1
+      with:
+        username: dsdejenkins
+        password: ${{ secrets.DSDEJENKINS_PASSWORD }}
+    - name: Build Cromwell Docker
+      run: |
+        set -e
+        cd cromwell
+        sbt server/docker
+        docker push broadinstitute/cromwell:$CROMWELL_SNAP_VERSION
+    - name: Edit & push chart
+      env:
+        BROADBOT_GITHUB_TOKEN: ${{ secrets.BROADBOT_GITHUB_TOKEN }}
+      run: |
+        set -e
+        cd cromwhelm
+        git checkout main
+        ls -la
+        sed -i "s/appVersion.*/appVersion: \"$CROMWELL_SNAP_VERSION\"/" cromwell-helm/Chart.yaml
+        sed -i "s/image: broadinstitute\/cromwell.*/image: broadinstitute\/cromwell:$CROMWELL_SNAP_VERSION/" cromwell-helm/templates/cromwell.yaml
+        git diff
+        git config --global user.name "broadbot"
+        git config --global user.email "[email protected]"
+        git commit -am "Auto update to Cromwell $CROMWELL_SNAP_VERSION"
+        git push https://broadbot:[email protected]/broadinstitute/cromwhelm.git main

.scala-steward.conf

@@ -25,7 +25,7 @@
 #
 # Default: @asap
 #
-pullRequests.frequency = "0 0 1 1,4,7,10 ?" # Run at 00:00 on the 1st day of Jan,Apr,Jul,Oct (whatever day that is)
+pullRequests.frequency = "0 0 1 3,6,9,12 ?" # Run at 00:00 on the 1st day of Mar,Jun,Sep,Dec (whatever day of the week that is)
 
 # Only these dependencies which match the given patterns are updated.
 #

CHANGELOG.md

@@ -1,5 +1,42 @@
 # Cromwell Change Log
 
+## 79 Release Notes
+
+### Last release with CWL support
+
+Cromwell 79 is the last release with CWL. Support will be removed in Cromwell 80 and above.
+
+CWL will be re-introduced at a later date in the [Terra platform](https://terra.bio/), using a solution other than Cromwell. See the blog post ["Terra’s roadmap to supporting more workflow languages"](https://terra.bio/terras-roadmap-to-supporting-more-workflow-languages/) for details.
+
+| Product                                   | Language | Support               |
+|-------------------------------------------|----------|-----------------------|
+| Cromwell standalone                       | WDL      | :white_check_mark:    |
+| Cromwell standalone                       | CWL      | :x:                    |
+| [Terra SaaS platform](https://terra.bio/) | WDL      | :white_check_mark:    |
+| [Terra SaaS platform](https://terra.bio/) | CWL      | Future support planned |
+
+### Last release with Alibaba Cloud
+
+The BCS backend and OSS filesystem (both of which support Alibaba Cloud) will be removed in version 80.
+
+### WES endpoints preview
+
+As a means to stay on top of endpoints within our repo, 3 new Workflow Execution Service (WES) endpoints are now documented in the Cromwell Swagger (others to follow as part of later work):
+
+| HTTP verb | Endpoint path | Description   |
+| --------- | ------------- |---------------|
+| GET | /api/ga4gh/wes/v1/service-info | Server info |
+| POST | /api/ga4gh/wes/v1/runs/{run_id}/cancel | Abort workflow |
+| GET | /api/ga4gh/wes/v1/runs/{run_id}/status | Workflow status |
+
+### Scala 2.13
+
+Cromwell is now built with Scala version 2.13. This change should not be noticeable to users but may be of interest to developers of Cromwell backend implementations.
+
+### Bug Fixes
+
+ * Fixed a call caching bug in which an invalid cache entry could cause a valid cache entry to be ignored.
+
 ## 75 Release Notes
 
 ### New `AwaitingCloudQuota` backend status

CromIAM/src/main/scala/cromiam/server/CromIamServer.scala

@@ -46,7 +46,7 @@ object CromIamServer extends HttpApp with CromIamApiService with SwaggerService
 
   override val routes: Route = allRoutes ~ swaggerUiResourceRoute
 
-  override val statusService: StatusService = new StatusService(() => Map(Cromwell -> cromwellClient.subsystemStatus, Sam -> samClient.subsystemStatus))
+  override val statusService: StatusService = new StatusService(() => Map(Cromwell -> cromwellClient.subsystemStatus(), Sam -> samClient.subsystemStatus()))
 
   // Override default shutdownsignal which was just "hit return/enter"
   override def waitForShutdownSignal(actorSystem: ActorSystem)(implicit executionContext: ExecutionContext): Future[Done] = {

CromIAM/src/main/scala/cromiam/server/config/CromIamServerConfig.scala

@@ -9,7 +9,7 @@ import common.validation.Validation._
 import cromiam.server.config.CromIamServerConfig._
 import net.ceedubs.ficus.Ficus._
 
-import scala.collection.JavaConverters._
+import scala.jdk.CollectionConverters._
 import scala.util.{Failure, Success, Try}
 
 final case class CromIamServerConfig(cromIamConfig: CromIamConfig,

CromIAM/src/main/scala/cromiam/server/status/StatusCheckedSubsystem.scala

@@ -19,7 +19,7 @@ trait StatusCheckedSubsystem {
     * error messages, otherwise OK = false and include the response body
     */
   def subsystemStatus()(implicit ec: ExecutionContext): Future[SubsystemStatus] = {
-    sttp.get(statusUri).send map { x =>
+    sttp.get(statusUri).send() map { x =>
       x.body match {
         case Right(_) => SubsystemStatus(true, None)
         case Left(errors) => SubsystemStatus(false, Option(List(errors)))

CromIAM/src/main/scala/cromiam/webservice/QuerySupport.scala

@@ -1,6 +1,7 @@
 package cromiam.webservice
 
 import akka.event.LoggingAdapter
+import akka.http.scaladsl.model.Uri.Query
 import akka.http.scaladsl.model._
 import akka.http.scaladsl.server.Directives._
 import akka.http.scaladsl.server._
@@ -86,7 +87,7 @@ trait QuerySupport extends RequestSupport {
       // DO NOT REMOVE THE NEXT LINE WITHOUT READING THE SCALADOC ON ensureNoLabelOrs
       ensureNoLabelOrs(user, labelOrs)
 
-      val newQueryBuilder = query.newBuilder
+      val newQueryBuilder = Query.newBuilder
       newQueryBuilder ++= query
 
       val collectionLabels = userCollectionLabels(user, collections)

CromIAM/src/main/scala/cromiam/webservice/SubmissionSupport.scala

@@ -109,7 +109,7 @@ object SubmissionSupport {
   }
 
   def extractInputAux: Directive1[Map[String, String]] = {
-    formFieldMap.map(_.filterKeys(_.startsWith(WorkflowInputsAuxPrefix)))
+    formFieldMap.map(_.view.filterKeys(_.startsWith(WorkflowInputsAuxPrefix)).toMap)
   }
 
   // FIXME: Much like CromwellClient see if there are ways of unifying this a bit w/ the mothership

CromIAM/src/test/scala/cromiam/sam/SamClientSpec.scala

@@ -15,11 +15,10 @@ import org.broadinstitute.dsde.workbench.model.WorkbenchUserId
 import org.scalatest.BeforeAndAfterAll
 import org.scalatest.flatspec.AsyncFlatSpec
 import org.scalatest.matchers.should.Matchers
-import org.specs2.mock.Mockito
 
 import scala.concurrent.ExecutionContextExecutor
 
-class SamClientSpec extends AsyncFlatSpec with Matchers with BeforeAndAfterAll with Mockito {
+class SamClientSpec extends AsyncFlatSpec with Matchers with BeforeAndAfterAll {
 
   implicit val actorSystem: ActorSystem = ActorSystem("SamClientSpec")
   implicit val ece: ExecutionContextExecutor = actorSystem.dispatcher
@@ -28,20 +27,20 @@ class SamClientSpec extends AsyncFlatSpec with Matchers with BeforeAndAfterAll w
   private val expectedErrorResponse =
     HttpResponse(StatusCodes.InternalServerError, entity = HttpEntity("expected error"))
 
-  val authorization = Authorization(OAuth2BearerToken("my-token"))
-  val authorizedUserWithCollection = User(WorkbenchUserId(MockSamClient.AuthorizedUserCollectionStr), authorization)
-  val unauthorizedUserWithNoCollection =
+  private val authorization = Authorization(OAuth2BearerToken("my-token"))
+  private val authorizedUserWithCollection = User(WorkbenchUserId(MockSamClient.AuthorizedUserCollectionStr), authorization)
+  private val unauthorizedUserWithNoCollection =
     User(WorkbenchUserId(MockSamClient.UnauthorizedUserCollectionStr), authorization)
-  val notWhitelistedUser = User(WorkbenchUserId(MockSamClient.NotWhitelistedUser), authorization)
+  private val notWhitelistedUser = User(WorkbenchUserId(MockSamClient.NotWhitelistedUser), authorization)
 
-  val authorizedCollection = Collection(MockSamClient.AuthorizedUserCollectionStr)
-  val unauthorizedCollection = Collection(MockSamClient.UnauthorizedUserCollectionStr)
-  val authorizedCollectionRequest =
+  private val authorizedCollection = Collection(MockSamClient.AuthorizedUserCollectionStr)
+  private val unauthorizedCollection = Collection(MockSamClient.UnauthorizedUserCollectionStr)
+  private val authorizedCollectionRequest =
     CollectionAuthorizationRequest(authorizedUserWithCollection, authorizedCollection, "add")
-  val unauthorizedCollectionRequest =
+  private val unauthorizedCollectionRequest =
     CollectionAuthorizationRequest(unauthorizedUserWithNoCollection, unauthorizedCollection, "add")
 
-  val emptyHttpRequest: HttpRequest = HttpRequest()
+  private val emptyHttpRequest: HttpRequest = HttpRequest()
 
   override protected def afterAll(): Unit = {
     actorSystem.terminate()

CromIAM/src/test/scala/cromiam/webservice/SwaggerServiceSpec.scala

@@ -14,7 +14,7 @@ import org.yaml.snakeyaml.error.YAMLException
 import org.yaml.snakeyaml.nodes.MappingNode
 import org.yaml.snakeyaml.{Yaml => SnakeYaml}
 
-import scala.collection.JavaConverters._
+import scala.jdk.CollectionConverters._
 
 
 class SwaggerServiceSpec extends AnyFlatSpec with CromwellTimeoutSpec with SwaggerService with ScalatestRouteTest with Matchers

CromwellRefdiskManifestCreator/pom.xml

@@ -48,7 +48,7 @@
         <dependency>
             <groupId>com.fasterxml.jackson.core</groupId>
             <artifactId>jackson-databind</artifactId>
-            <version>2.11.1</version>
+            <version>2.13.2.2</version>
         </dependency>
         <dependency>
             <groupId>org.apache.logging.log4j</groupId>

README.md

@@ -22,28 +22,24 @@ Talk to us:
 
 ### Capabilities and roadmap
 
-A majority of Cromwell users today run their workflows in [Terra](https://app.terra.bio/), a fully-managed cloud-native bioinformatics computing platform. See [here](https://support.terra.bio/hc/en-us/articles/360036379771-Get-started-running-workflows) for a quick-start guide.
+Many users today run their WDL workflows in [Terra](https://app.terra.bio/), a managed cloud bioinformatics platform with built-in WDL support provided by Cromwell. See [here](https://support.terra.bio/hc/en-us/articles/360036379771-Get-started-running-workflows) for a quick-start guide.
 
 Users with specialized needs who wish to install and maintain their own Cromwell instances can [download](https://github.com/broadinstitute/cromwell/releases) a JAR or Docker image. The development team accepts reproducible bug reports from self-managed instances, but cannot feasibly provide direct support.
 
-[Cromwell's backends](https://cromwell.readthedocs.io/en/stable/backends/Backends/) receive development resources proportional to customer demand. The team is actively developing for Google Cloud and AWS. Maintenance of other backends is primarily community-based.
+[Cromwell's backends](https://cromwell.readthedocs.io/en/stable/backends/Backends/) receive development resources proportional to user demand. The team is actively developing for Google Cloud and Microsoft Azure. Maintenance of other backends is primarily community-based.
 
-Cromwell [supports](https://cromwell.readthedocs.io/en/stable/LanguageSupport/) the WDL and CWL workflow languages. The Cromwell team is actively developing WDL, while maintenance for CWL is primarily community-based.  
+Cromwell [supports](https://cromwell.readthedocs.io/en/stable/LanguageSupport/) the WDL workflow language. Cromwell version 80 and above no longer support CWL.
+
+CWL will be re-introduced at a later date in the [Terra platform](https://terra.bio/), using a solution other than Cromwell. See the blog post ["Terra’s roadmap to supporting more workflow languages"](https://terra.bio/terras-roadmap-to-supporting-more-workflow-languages/) for details.
 
 ### Security reports
 
 If you believe you have found a security issue please contact `[email protected]`.
 
-### Issue tracking in JIRA
-
-<!--
-AEN external issue filing tested 2020-12-08 with `[email protected]` / `https://broadworkbench.atlassian.net/browse/CROM-6681`
--->
-
-Need to file an issue? Head over to [our JIRA](https://broadworkbench.atlassian.net/jira/software/c/projects/CROM/issues). You must create a free profile to view or create.
+### Issue tracking
 
-[Issues in Github](https://github.com/broadinstitute/cromwell/issues) remain available for discussion among community members but are not actively monitored by the development team.
+Need to file an issue? Head over to [Github Issues](https://github.com/broadinstitute/cromwell/issues).
 
-![Cromwell JIRA](docs/img/cromwell_jira.png)
+If you previously filed an issue in JIRA, the link is [here](https://broadworkbench.atlassian.net/jira/software/c/projects/CROM/issues). New signups are no longer accepted.
 
 ![Jamie, the Cromwell pig](docs/jamie_the_cromwell_pig.png)

backend/src/main/scala/cromwell/backend/BackendLifecycleActor.scala

@@ -40,7 +40,7 @@ trait BackendLifecycleActor extends Actor {
   protected def performActionThenRespond(operation: => Future[BackendWorkflowLifecycleActorResponse],
                                          onFailure: Throwable => BackendWorkflowLifecycleActorResponse,
                                          andThen: => Unit = ()) = {
-    val respondTo: ActorRef = sender
+    val respondTo: ActorRef = sender()
     operation onComplete {
       case Success(r) =>
         respondTo ! r

backend/src/main/scala/cromwell/backend/BackendWorkflowFinalizationActor.scala

@@ -28,7 +28,7 @@ object BackendWorkflowFinalizationActor {
 trait BackendWorkflowFinalizationActor extends BackendWorkflowLifecycleActor with ActorLogging {
 
   def receive: Receive = LoggingReceive {
-    case Finalize => performActionThenRespond(afterAll map { _ => FinalizationSuccess }, onFailure = FinalizationFailed)
+    case Finalize => performActionThenRespond(afterAll() map { _ => FinalizationSuccess }, onFailure = FinalizationFailed)
   }
 
   /**

backend/src/main/scala/cromwell/backend/OutputEvaluator.scala

@@ -78,7 +78,7 @@ object OutputEvaluator {
       case Success(Invalid(errors)) => InvalidJobOutputs(errors)
       case Failure(exception) => JobOutputsEvaluationException(exception)
     }
-    
+
     /*
       * Because Cromwell doesn't trust anyone, if custom evaluation is provided,
       * still make sure that all the output ports have been filled with values
@@ -90,7 +90,7 @@ object OutputEvaluator {
         case Nil =>
           val errorMessagePrefix = "Error applying postMapper in short-circuit output evaluation"
           TryUtil.sequenceMap(outputs map { case (k, v) => (k, postMapper(v))}, errorMessagePrefix) match {
-            case Failure(e) => InvalidJobOutputs(NonEmptyList.of(e.getMessage, e.getStackTrace.take(5).map(_.toString):_*))
+            case Failure(e) => InvalidJobOutputs(NonEmptyList.of(e.getMessage, e.getStackTrace.take(5).toIndexedSeq.map(_.toString):_*))
             case Success(postMappedOutputs) => ValidJobOutputs(CallOutputs(postMappedOutputs))
           }
         case head :: tail => InvalidJobOutputs(NonEmptyList.of(toError(head), tail.map(toError): _*))

backend/src/main/scala/cromwell/backend/dummy/DummyInitializationActor.scala

@@ -17,7 +17,7 @@ class DummyInitializationActor(pipelinesParams: StandardInitializationActorParam
   val backendAttributeValidation: RuntimeAttributesValidation[String] = new RuntimeAttributesValidation[String] {
     override def key: String = "backend"
 
-    override def coercion: Traversable[WomType] = Vector(WomStringType)
+    override def coercion: Iterable[WomType] = Vector(WomStringType)
 
     override protected def validateValue: PartialFunction[WomValue, ErrorOr[String]] = {
       case WomString("Dummy") => "Dummy".validNel