Bump libraries and run centaur bcs.

.gitignore

@@ -29,6 +29,7 @@ scripts/docker-compose-mysql/compose/mysql/data
 artifactory_credentials.properties
 aws_credentials
 bcs_application.conf
+bcs_login.inc.sh
 centaur_secure.inc.conf
 cromwell-centaur-requester-pays-service-account.json
 cromwell-centaur-service-account.json

centaur/src/main/resources/standardTestCases/labels_bad_workflow.test

@@ -12,7 +12,7 @@ files {
 
 metadata {
   status: Failed
-  "submittedFiles.labels": "{\"label-key-1\":\"label-value-1\",\"label-key-2\":\"label-value-2\",\"only-key\":\"\",\"fc-id\":\"0123-abcd-4567-efgh\"}"
+  "submittedFiles.labels": "{\"fc-id\":\"0123-abcd-4567-efgh\",\"label-key-1\":\"label-value-1\",\"label-key-2\":\"label-value-2\",\"only-key\":\"\"}"
   "labels.label-key-1": "label-value-1"
   "labels.label-key-2": "label-value-2"
   "labels.only-key": ""

centaur/src/main/resources/standardTestCases/prefix.test

@@ -3,6 +3,8 @@ testFormat: workflowsuccess
 
 files {
   workflow: prefix/prefix.wdl
+  # https://github.com/broadinstitute/cromwell/issues/4590
+  options: prefix/prefix.options
 }
 
 metadata {

centaur/src/main/resources/standardTestCases/prefix/prefix.options

@@ -0,0 +1,3 @@
+{
+  "read_from_cache": false
+}

centaur/src/main/resources/standardTestCases/valid_labels.test

@@ -11,7 +11,7 @@ files {
 metadata {
   workflowName: wf_hello
   status: Succeeded
-  "submittedFiles.labels": "{\"label-key-1\":\"label-value-1\",\"label-key-2\":\"label-value-2\",\"only-key\":\"\",\"fc-id\":\"0123-abcd-4567-efgh\"}"
+  "submittedFiles.labels": "{\"fc-id\":\"0123-abcd-4567-efgh\",\"label-key-1\":\"label-value-1\",\"label-key-2\":\"label-value-2\",\"only-key\":\"\"}"
   "labels.label-key-1": "label-value-1"
   "labels.label-key-2": "label-value-2"
   "labels.only-key": ""

centaurCwlRunner/src/main/resources/application.conf

@@ -1,6 +1,15 @@
 akka {
   loggers = ["akka.event.slf4j.Slf4jLogger"]
   logging-filter = "akka.event.slf4j.Slf4jLoggingFilter"
+
+  # Silence Akka-Http warning logging of:
+  # [WARN] [01/28/2019 06:50:04.293] [centaur-acting-like-a-system-akka.actor.default-dispatcher-3] [centaur-acting-like-a-system/Pool(shared->http://localhost:8000)] Connection attempt failed. Backing off new connection attempts for at least 100 milliseconds.
+  #
+  # via:
+  # https://github.com/akka/akka/blob/v2.5.19/akka-actor/src/main/resources/reference.conf#L41-L44
+  # https://github.com/akka/akka/blob/v2.5.19/akka-actor/src/main/scala/akka/event/Logging.scala#L377
+  # https://github.com/akka/akka-http/blob/v10.1.7/akka-http-core/src/main/scala/akka/http/impl/engine/client/pool/NewHostConnectionPool.scala#L134
+  stdout-loglevel = "ERROR"
 }
 
 centaur {

centaurCwlRunner/src/main/resources/logback.xml

@@ -27,4 +27,22 @@
     https://github.com/owlcs/owlapi/blob/owlapi-parent-5.1.7/api/src/main/java/org/semanticweb/owlapi/utilities/Injector.java#L383
     -->
     <logger name="org.semanticweb.owlapi.utilities.Injector" level="error" additivity="false"/>
+
+    <!--
+    Silence Akka warning logging of:
+    10:18:20.629 [centaur-acting-like-a-system-akka.actor.default-dispatcher-8] WARN  akka.io.InetAddressDnsResolver - No caching TTL defined. Using default value Ttl(30 seconds).
+
+    via:
+    https://github.com/akka/akka/blob/v2.5.19/akka-actor/src/main/scala/akka/io/InetAddressDnsResolver.scala#L40
+    -->
+    <logger name="akka.io.InetAddressDnsResolver" level="error" additivity="false"/>
+
+    <!--
+    Silence Akka-Http warning logging of:
+    00:53:57.537 [centaur-acting-like-a-system-akka.actor.default-dispatcher-10] WARN  akka.http.impl.engine.client.PoolGateway - Connection attempt failed. Backing off new connection attempts for at least 100 milliseconds.
+
+    via:
+    https://github.com/akka/akka-http/blob/v10.1.7/akka-http-core/src/main/scala/akka/http/impl/engine/client/pool/NewHostConnectionPool.scala#L134
+    -->
+    <logger name="akka.http.impl.engine.client.PoolGateway" level="error" additivity="false"/>
 </configuration>

cloud-nio/cloud-nio-impl-drs/src/main/scala/cloud/nio/impl/drs/DrsPathResolver.scala

@@ -15,10 +15,6 @@ import org.apache.http.impl.client.HttpClientBuilder
 import org.apache.http.util.EntityUtils
 import org.apache.http.{HttpResponse, HttpStatus}
 
-//Do not remove this import. This import is required to compile, but IntelliJ doesn't correctly recognize that.
-import cats.syntax.functor._
-
-
 case class DrsPathResolver(drsConfig: DrsConfig, httpClientBuilder: HttpClientBuilder) {
 
   implicit lazy val urlDecoder: Decoder[Url] = deriveDecoder

cloudSupport/src/main/scala/cromwell/cloudsupport/aws/AwsConfiguration.scala

@@ -42,7 +42,7 @@ import common.validation.Validation._
 import cromwell.cloudsupport.aws.auth.{AssumeRoleMode, AwsAuthMode, CustomKeyMode, DefaultMode}
 import net.ceedubs.ficus.Ficus._
 import org.slf4j.LoggerFactory
-import software.amazon.awssdk.core.regions.Region
+import software.amazon.awssdk.regions.Region
 
 final case class AwsConfiguration private (applicationName: String,
                                            authsByName: Map[String, AwsAuthMode],
@@ -99,10 +99,7 @@ object AwsConfiguration {
       }
 
       def assumeRoleAuth(authConfig: Config, name: String, region: Option[String]): ErrorOr[AwsAuthMode] = validate {
-        val externalId = authConfig.hasPath("external-id") match {
-          case true => authConfig.getString("external-id")
-          case _ => ""
-        }
+        val externalId = authConfig.getOrElse("external-id", "")
         AssumeRoleMode(
           name,
           // We won't do anything with this now, but it is required for

cloudSupport/src/main/scala/cromwell/cloudsupport/aws/auth/AwsAuthMode.scala

@@ -30,20 +30,14 @@
  */
 package cromwell.cloudsupport.aws.auth
 
-import software.amazon.awssdk.core.auth.{AwsCredentials,
-                                         AwsSessionCredentials,
-                                         AnonymousCredentialsProvider,
-                                         DefaultCredentialsProvider,
-                                         StaticCredentialsProvider}
-import software.amazon.awssdk.core.regions.Region
-import software.amazon.awssdk.services.sts.{STSClient}
-import software.amazon.awssdk.services.sts.model.{GetCallerIdentityRequest,
-                                                  AssumeRoleRequest}
-
+import com.google.api.client.json.jackson2.JacksonFactory
 import cromwell.cloudsupport.aws.auth.AwsAuthMode.OptionLookup
-
 import org.slf4j.LoggerFactory
-import com.google.api.client.json.jackson2.JacksonFactory
+import software.amazon.awssdk.auth.credentials._
+import software.amazon.awssdk.regions.Region
+import software.amazon.awssdk.services.sts.StsClient
+import software.amazon.awssdk.services.sts.model.{AssumeRoleRequest, GetCallerIdentityRequest}
+
 import scala.util.{Failure, Success, Try}
 
 object AwsAuthMode {
@@ -71,9 +65,9 @@ sealed trait AwsAuthMode {
     * All traits in this file are sealed, all classes final, meaning things
     * like Mockito or other java/scala overrides cannot work.
     */
-   private[auth] var credentialValidation: ((AwsCredentials, Option[String]) => Unit) =
+   private[auth] var credentialValidation: (AwsCredentials, Option[String]) => Unit =
      (credentials: AwsCredentials, region: Option[String]) => {
-       val builder = STSClient.builder
+       val builder = StsClient.builder
 
        //If the region argument exists in config, set it in the builder.
        //Otherwise it is left unset and the AwsCredential builder will look in various places to supply,
@@ -97,7 +91,7 @@ sealed trait AwsAuthMode {
 case object MockAuthMode extends AwsAuthMode {
   override val name = "no_auth"
 
-  lazy val _credential = AnonymousCredentialsProvider.create.getCredentials
+  lazy val _credential = AnonymousCredentialsProvider.create.resolveCredentials()
 
   override def credential(options: OptionLookup): AwsCredentials = _credential
 }
@@ -113,7 +107,7 @@ final case class CustomKeyMode(override val name: String,
     // Validate credentials synchronously here, without retry.
     // It's very unlikely to fail as it should not happen more than a few times
     // (one for the engine and for each backend using it) per Cromwell instance.
-    validateCredential(AwsCredentials.create(accessKey, secretKey), region)
+    validateCredential(AwsBasicCredentials.create(accessKey, secretKey), region)
   }
 
   override def credential(options: OptionLookup): AwsCredentials = _credential
@@ -130,7 +124,7 @@ final case class DefaultMode(override val name: String, region: Option[String])
     // Validate credentials synchronously here, without retry.
     // It's very unlikely to fail as it should not happen more than a few times
     // (one for the engine and for each backend using it) per Cromwell instance.
-    validateCredential(DefaultCredentialsProvider.create.getCredentials, region)
+    validateCredential(DefaultCredentialsProvider.create.resolveCredentials(), region)
   }
 
   override def credential(options: OptionLookup): AwsCredentials = _credential
@@ -157,7 +151,7 @@ final case class AssumeRoleMode(override val name: String,
     if (! externalId.isEmpty) requestBuilder.externalId(externalId)
     val request = requestBuilder.build
 
-    val builder = STSClient.builder
+    val builder = StsClient.builder
     region.foreach(str => builder.region(Region.of(str)))
     // See comment above regarding builder
     baseAuthObj match{

cloudSupport/src/main/scala/cromwell/cloudsupport/aws/s3/S3Storage.scala

@@ -30,43 +30,42 @@
  */
 package cromwell.cloudsupport.aws.s3
 
-import software.amazon.awssdk.services.s3.S3AdvancedConfiguration
-import software.amazon.awssdk.services.s3.S3Client
-import software.amazon.awssdk.core.auth.{AwsCredentials, StaticCredentialsProvider}
 import com.typesafe.config.ConfigFactory
 import net.ceedubs.ficus.Ficus._
-import software.amazon.awssdk.core.regions.Region
+import software.amazon.awssdk.auth.credentials.{AwsCredentials, StaticCredentialsProvider}
+import software.amazon.awssdk.regions.Region
+import software.amazon.awssdk.services.s3.{S3Client, S3Configuration}
 
 object S3Storage {
   val DefaultConfiguration = {
     val accelerateModeEnabled = ConfigFactory.load().as[Option[Boolean]]("s3.accelerate-mode").getOrElse(false)
     val dualstackEnabled = ConfigFactory.load().as[Option[Boolean]]("s3.dual-stack").getOrElse(false)
     val pathStyleAccessEnabled = ConfigFactory.load().as[Option[Boolean]]("s3.path-style-access").getOrElse(false)
 
-    S3AdvancedConfiguration.builder
+    S3Configuration.builder
       .accelerateModeEnabled(accelerateModeEnabled)
       .dualstackEnabled(dualstackEnabled)
       .pathStyleAccessEnabled(pathStyleAccessEnabled)
       .build
   }
 
-  def s3Client(configuration: S3AdvancedConfiguration, credentials: AwsCredentials, region: Option[Region]): S3Client = {
+  def s3Client(configuration: S3Configuration, credentials: AwsCredentials, region: Option[Region]): S3Client = {
     val builder = S3Client.builder
-      .advancedConfiguration(configuration)
+      .serviceConfiguration(configuration)
       .credentialsProvider(StaticCredentialsProvider.create(credentials))
-      region.foreach(builder.region)
-      builder.build
+    region.foreach(builder.region)
+    builder.build
   }
 
   def s3Client(credentials: AwsCredentials, region: Option[Region]): S3Client = {
-    s3Client(s3AdvancedConfiguration(), credentials, region)
+    s3Client(s3Configuration(), credentials, region)
   }
 
-  def s3AdvancedConfiguration(accelerateModeEnabled: Boolean = false,
-                              dualstackEnabled: Boolean = false,
-                              pathStyleAccessEnabled: Boolean = false): S3AdvancedConfiguration = {
+  def s3Configuration(accelerateModeEnabled: Boolean = false,
+                      dualstackEnabled: Boolean = false,
+                      pathStyleAccessEnabled: Boolean = false): S3Configuration = {
 
-    S3AdvancedConfiguration.builder
+    S3Configuration.builder
       .accelerateModeEnabled(accelerateModeEnabled)
       .dualstackEnabled(dualstackEnabled)
       .pathStyleAccessEnabled(pathStyleAccessEnabled)

cloudSupport/src/test/scala/cromwell/cloudsupport/aws/s3/S3StorageSpec.scala

@@ -29,9 +29,9 @@
  *  POSSIBILITY OF SUCH DAMAGE.
  */
 package cromwell.cloudsupport.aws.s3
-import software.amazon.awssdk.core.auth.AnonymousCredentialsProvider
+import software.amazon.awssdk.auth.credentials.AnonymousCredentialsProvider
 import org.scalatest.{FlatSpec, Matchers, Tag}
-import software.amazon.awssdk.core.regions.Region
+import software.amazon.awssdk.regions.Region
 
 class S3StorageSpec extends FlatSpec with Matchers {
 
@@ -45,14 +45,17 @@ class S3StorageSpec extends FlatSpec with Matchers {
   }
 
   it should "build s3 storage" taggedAs S3StorageSpecUtils.AwsTest in {
-    val configuration = S3Storage.s3AdvancedConfiguration(false, true)
+    val configuration = S3Storage.s3Configuration(dualstackEnabled = true)
     configuration.accelerateModeEnabled should be(false)
     configuration.dualstackEnabled should be(true)
     configuration.pathStyleAccessEnabled should be(false)
   }
 
   it should "build s3 client with credentials" taggedAs S3StorageSpecUtils.AwsTest in {
-    S3Storage.s3Client(AnonymousCredentialsProvider.create.getCredentials, Option(Region.US_EAST_1))
+    S3Storage.s3Client(
+      AnonymousCredentialsProvider.create.resolveCredentials(),
+      Option(Region.US_EAST_1)
+    )
   }
 
 }

dockerHashing/src/main/scala/cromwell/docker/DockerInfoActor.scala

@@ -9,14 +9,15 @@ import com.google.common.cache.CacheBuilder
 import com.typesafe.config.Config
 import common.validation.ErrorOr.ErrorOr
 import common.validation.Validation._
-import cromwell.core.{Dispatcher, DockerConfiguration}
 import cromwell.core.actor.StreamIntegration.{BackPressure, StreamContext}
+import cromwell.core.{Dispatcher, DockerConfiguration}
 import cromwell.docker.DockerInfoActor._
 import cromwell.docker.registryv2.DockerRegistryV2Abstract
 import cromwell.docker.registryv2.flows.dockerhub.DockerHubRegistry
 import cromwell.docker.registryv2.flows.gcr.GcrRegistry
 import cromwell.docker.registryv2.flows.quay.QuayRegistry
 import cromwell.util.GracefulShutdownHelper.ShutdownCommand
+import fs2.Pipe
 import fs2.concurrent.{NoneTerminatedQueue, Queue}
 import net.ceedubs.ficus.Ficus._
 import org.http4s.client.blaze.BlazeClientBuilder
@@ -112,7 +113,7 @@ final class DockerInfoActor(
   /*
    * Sends back responses and adds to cache in case of success. Used as the sink for each registry stream.
    */
-  val streamSink = fs2.Sink[IO, (DockerInfoResponse, DockerInfoContext)] {
+  val streamSink: Pipe[IO, (DockerInfoResponse, DockerInfoContext), Unit] = _ evalMap {
     case (response: DockerInfoSuccessResponse, dockerInfoContext) =>
       dockerInfoContext.replyTo ! response
       IO.pure(cache.put(dockerInfoContext.dockerImageID, response.dockerInformation))
@@ -148,7 +149,7 @@ final class DockerInfoActor(
           // Run requests in parallel - allow nbThreads max concurrent requests - order doesn't matter
           .parEvalMapUnordered(registry.config.nbThreads)({ request => registry.run(request)(client) })
           // Send to the sink for finalization of the result
-          .to(streamSink)
+          .through(streamSink)
       })
 
       // Start the stream now asynchronously. It will keep running until we terminate the queue by sending None to it