diff --git a/.gitignore b/.gitignore
index 1f20691f99e..42680886bf9 100644
--- a/.gitignore
+++ b/.gitignore
@@ -29,6 +29,7 @@ scripts/docker-compose-mysql/compose/mysql/data
artifactory_credentials.properties
aws_credentials
bcs_application.conf
+bcs_login.inc.sh
centaur_secure.inc.conf
cromwell-centaur-requester-pays-service-account.json
cromwell-centaur-service-account.json
diff --git a/centaur/src/main/resources/standardTestCases/labels_bad_workflow.test b/centaur/src/main/resources/standardTestCases/labels_bad_workflow.test
index 426be8c4e73..9413985f25f 100644
--- a/centaur/src/main/resources/standardTestCases/labels_bad_workflow.test
+++ b/centaur/src/main/resources/standardTestCases/labels_bad_workflow.test
@@ -12,7 +12,7 @@ files {
metadata {
status: Failed
- "submittedFiles.labels": "{\"label-key-1\":\"label-value-1\",\"label-key-2\":\"label-value-2\",\"only-key\":\"\",\"fc-id\":\"0123-abcd-4567-efgh\"}"
+ "submittedFiles.labels": "{\"fc-id\":\"0123-abcd-4567-efgh\",\"label-key-1\":\"label-value-1\",\"label-key-2\":\"label-value-2\",\"only-key\":\"\"}"
"labels.label-key-1": "label-value-1"
"labels.label-key-2": "label-value-2"
"labels.only-key": ""
diff --git a/centaur/src/main/resources/standardTestCases/prefix.test b/centaur/src/main/resources/standardTestCases/prefix.test
index 457e01b8b35..a836843066e 100644
--- a/centaur/src/main/resources/standardTestCases/prefix.test
+++ b/centaur/src/main/resources/standardTestCases/prefix.test
@@ -3,6 +3,8 @@ testFormat: workflowsuccess
files {
workflow: prefix/prefix.wdl
+ # https://github.com/broadinstitute/cromwell/issues/4590
+ options: prefix/prefix.options
}
metadata {
diff --git a/centaur/src/main/resources/standardTestCases/prefix/prefix.options b/centaur/src/main/resources/standardTestCases/prefix/prefix.options
new file mode 100644
index 00000000000..1840c33cd39
--- /dev/null
+++ b/centaur/src/main/resources/standardTestCases/prefix/prefix.options
@@ -0,0 +1,3 @@
+{
+ "read_from_cache": false
+}
diff --git a/centaur/src/main/resources/standardTestCases/valid_labels.test b/centaur/src/main/resources/standardTestCases/valid_labels.test
index 36010acd6de..1ab748f8a86 100644
--- a/centaur/src/main/resources/standardTestCases/valid_labels.test
+++ b/centaur/src/main/resources/standardTestCases/valid_labels.test
@@ -11,7 +11,7 @@ files {
metadata {
workflowName: wf_hello
status: Succeeded
- "submittedFiles.labels": "{\"label-key-1\":\"label-value-1\",\"label-key-2\":\"label-value-2\",\"only-key\":\"\",\"fc-id\":\"0123-abcd-4567-efgh\"}"
+ "submittedFiles.labels": "{\"fc-id\":\"0123-abcd-4567-efgh\",\"label-key-1\":\"label-value-1\",\"label-key-2\":\"label-value-2\",\"only-key\":\"\"}"
"labels.label-key-1": "label-value-1"
"labels.label-key-2": "label-value-2"
"labels.only-key": ""
diff --git a/centaurCwlRunner/src/main/resources/application.conf b/centaurCwlRunner/src/main/resources/application.conf
index c6a5039833e..fcdd3da9c57 100644
--- a/centaurCwlRunner/src/main/resources/application.conf
+++ b/centaurCwlRunner/src/main/resources/application.conf
@@ -1,6 +1,15 @@
akka {
loggers = ["akka.event.slf4j.Slf4jLogger"]
logging-filter = "akka.event.slf4j.Slf4jLoggingFilter"
+
+ # Silence Akka-Http warning logging of:
+ # [WARN] [01/28/2019 06:50:04.293] [centaur-acting-like-a-system-akka.actor.default-dispatcher-3] [centaur-acting-like-a-system/Pool(shared->http://localhost:8000)] Connection attempt failed. Backing off new connection attempts for at least 100 milliseconds.
+ #
+ # via:
+ # https://github.com/akka/akka/blob/v2.5.19/akka-actor/src/main/resources/reference.conf#L41-L44
+ # https://github.com/akka/akka/blob/v2.5.19/akka-actor/src/main/scala/akka/event/Logging.scala#L377
+ # https://github.com/akka/akka-http/blob/v10.1.7/akka-http-core/src/main/scala/akka/http/impl/engine/client/pool/NewHostConnectionPool.scala#L134
+ stdout-loglevel = "ERROR"
}
centaur {
diff --git a/centaurCwlRunner/src/main/resources/logback.xml b/centaurCwlRunner/src/main/resources/logback.xml
index c21755aa774..d92bac61949 100644
--- a/centaurCwlRunner/src/main/resources/logback.xml
+++ b/centaurCwlRunner/src/main/resources/logback.xml
@@ -27,4 +27,22 @@
https://github.com/owlcs/owlapi/blob/owlapi-parent-5.1.7/api/src/main/java/org/semanticweb/owlapi/utilities/Injector.java#L383
-->
+
+
+
+
+
+
diff --git a/cloud-nio/cloud-nio-impl-drs/src/main/scala/cloud/nio/impl/drs/DrsPathResolver.scala b/cloud-nio/cloud-nio-impl-drs/src/main/scala/cloud/nio/impl/drs/DrsPathResolver.scala
index e56ecdf85cf..cc8bbf7dd26 100644
--- a/cloud-nio/cloud-nio-impl-drs/src/main/scala/cloud/nio/impl/drs/DrsPathResolver.scala
+++ b/cloud-nio/cloud-nio-impl-drs/src/main/scala/cloud/nio/impl/drs/DrsPathResolver.scala
@@ -15,10 +15,6 @@ import org.apache.http.impl.client.HttpClientBuilder
import org.apache.http.util.EntityUtils
import org.apache.http.{HttpResponse, HttpStatus}
-//Do not remove this import. This import is required to compile, but IntelliJ doesn't correctly recognize that.
-import cats.syntax.functor._
-
-
case class DrsPathResolver(drsConfig: DrsConfig, httpClientBuilder: HttpClientBuilder) {
implicit lazy val urlDecoder: Decoder[Url] = deriveDecoder
diff --git a/cloudSupport/src/main/scala/cromwell/cloudsupport/aws/AwsConfiguration.scala b/cloudSupport/src/main/scala/cromwell/cloudsupport/aws/AwsConfiguration.scala
index 1e332eaf07d..396504afeff 100644
--- a/cloudSupport/src/main/scala/cromwell/cloudsupport/aws/AwsConfiguration.scala
+++ b/cloudSupport/src/main/scala/cromwell/cloudsupport/aws/AwsConfiguration.scala
@@ -42,7 +42,7 @@ import common.validation.Validation._
import cromwell.cloudsupport.aws.auth.{AssumeRoleMode, AwsAuthMode, CustomKeyMode, DefaultMode}
import net.ceedubs.ficus.Ficus._
import org.slf4j.LoggerFactory
-import software.amazon.awssdk.core.regions.Region
+import software.amazon.awssdk.regions.Region
final case class AwsConfiguration private (applicationName: String,
authsByName: Map[String, AwsAuthMode],
@@ -99,10 +99,7 @@ object AwsConfiguration {
}
def assumeRoleAuth(authConfig: Config, name: String, region: Option[String]): ErrorOr[AwsAuthMode] = validate {
- val externalId = authConfig.hasPath("external-id") match {
- case true => authConfig.getString("external-id")
- case _ => ""
- }
+ val externalId = authConfig.getOrElse("external-id", "")
AssumeRoleMode(
name,
// We won't do anything with this now, but it is required for
diff --git a/cloudSupport/src/main/scala/cromwell/cloudsupport/aws/auth/AwsAuthMode.scala b/cloudSupport/src/main/scala/cromwell/cloudsupport/aws/auth/AwsAuthMode.scala
index f2338e98a00..43fb8844eb1 100644
--- a/cloudSupport/src/main/scala/cromwell/cloudsupport/aws/auth/AwsAuthMode.scala
+++ b/cloudSupport/src/main/scala/cromwell/cloudsupport/aws/auth/AwsAuthMode.scala
@@ -30,20 +30,14 @@
*/
package cromwell.cloudsupport.aws.auth
-import software.amazon.awssdk.core.auth.{AwsCredentials,
- AwsSessionCredentials,
- AnonymousCredentialsProvider,
- DefaultCredentialsProvider,
- StaticCredentialsProvider}
-import software.amazon.awssdk.core.regions.Region
-import software.amazon.awssdk.services.sts.{STSClient}
-import software.amazon.awssdk.services.sts.model.{GetCallerIdentityRequest,
- AssumeRoleRequest}
-
+import com.google.api.client.json.jackson2.JacksonFactory
import cromwell.cloudsupport.aws.auth.AwsAuthMode.OptionLookup
-
import org.slf4j.LoggerFactory
-import com.google.api.client.json.jackson2.JacksonFactory
+import software.amazon.awssdk.auth.credentials._
+import software.amazon.awssdk.regions.Region
+import software.amazon.awssdk.services.sts.StsClient
+import software.amazon.awssdk.services.sts.model.{AssumeRoleRequest, GetCallerIdentityRequest}
+
import scala.util.{Failure, Success, Try}
object AwsAuthMode {
@@ -71,9 +65,9 @@ sealed trait AwsAuthMode {
* All traits in this file are sealed, all classes final, meaning things
* like Mockito or other java/scala overrides cannot work.
*/
- private[auth] var credentialValidation: ((AwsCredentials, Option[String]) => Unit) =
+ private[auth] var credentialValidation: (AwsCredentials, Option[String]) => Unit =
(credentials: AwsCredentials, region: Option[String]) => {
- val builder = STSClient.builder
+ val builder = StsClient.builder
//If the region argument exists in config, set it in the builder.
//Otherwise it is left unset and the AwsCredential builder will look in various places to supply,
@@ -97,7 +91,7 @@ sealed trait AwsAuthMode {
case object MockAuthMode extends AwsAuthMode {
override val name = "no_auth"
- lazy val _credential = AnonymousCredentialsProvider.create.getCredentials
+ lazy val _credential = AnonymousCredentialsProvider.create.resolveCredentials()
override def credential(options: OptionLookup): AwsCredentials = _credential
}
@@ -113,7 +107,7 @@ final case class CustomKeyMode(override val name: String,
// Validate credentials synchronously here, without retry.
// It's very unlikely to fail as it should not happen more than a few times
// (one for the engine and for each backend using it) per Cromwell instance.
- validateCredential(AwsCredentials.create(accessKey, secretKey), region)
+ validateCredential(AwsBasicCredentials.create(accessKey, secretKey), region)
}
override def credential(options: OptionLookup): AwsCredentials = _credential
@@ -130,7 +124,7 @@ final case class DefaultMode(override val name: String, region: Option[String])
// Validate credentials synchronously here, without retry.
// It's very unlikely to fail as it should not happen more than a few times
// (one for the engine and for each backend using it) per Cromwell instance.
- validateCredential(DefaultCredentialsProvider.create.getCredentials, region)
+ validateCredential(DefaultCredentialsProvider.create.resolveCredentials(), region)
}
override def credential(options: OptionLookup): AwsCredentials = _credential
@@ -157,7 +151,7 @@ final case class AssumeRoleMode(override val name: String,
if (! externalId.isEmpty) requestBuilder.externalId(externalId)
val request = requestBuilder.build
- val builder = STSClient.builder
+ val builder = StsClient.builder
region.foreach(str => builder.region(Region.of(str)))
// See comment above regarding builder
baseAuthObj match{
diff --git a/cloudSupport/src/main/scala/cromwell/cloudsupport/aws/s3/S3Storage.scala b/cloudSupport/src/main/scala/cromwell/cloudsupport/aws/s3/S3Storage.scala
index ab0db7b1517..6a16a0c557c 100644
--- a/cloudSupport/src/main/scala/cromwell/cloudsupport/aws/s3/S3Storage.scala
+++ b/cloudSupport/src/main/scala/cromwell/cloudsupport/aws/s3/S3Storage.scala
@@ -30,12 +30,11 @@
*/
package cromwell.cloudsupport.aws.s3
-import software.amazon.awssdk.services.s3.S3AdvancedConfiguration
-import software.amazon.awssdk.services.s3.S3Client
-import software.amazon.awssdk.core.auth.{AwsCredentials, StaticCredentialsProvider}
import com.typesafe.config.ConfigFactory
import net.ceedubs.ficus.Ficus._
-import software.amazon.awssdk.core.regions.Region
+import software.amazon.awssdk.auth.credentials.{AwsCredentials, StaticCredentialsProvider}
+import software.amazon.awssdk.regions.Region
+import software.amazon.awssdk.services.s3.{S3Client, S3Configuration}
object S3Storage {
val DefaultConfiguration = {
@@ -43,30 +42,30 @@ object S3Storage {
val dualstackEnabled = ConfigFactory.load().as[Option[Boolean]]("s3.dual-stack").getOrElse(false)
val pathStyleAccessEnabled = ConfigFactory.load().as[Option[Boolean]]("s3.path-style-access").getOrElse(false)
- S3AdvancedConfiguration.builder
+ S3Configuration.builder
.accelerateModeEnabled(accelerateModeEnabled)
.dualstackEnabled(dualstackEnabled)
.pathStyleAccessEnabled(pathStyleAccessEnabled)
.build
}
- def s3Client(configuration: S3AdvancedConfiguration, credentials: AwsCredentials, region: Option[Region]): S3Client = {
+ def s3Client(configuration: S3Configuration, credentials: AwsCredentials, region: Option[Region]): S3Client = {
val builder = S3Client.builder
- .advancedConfiguration(configuration)
+ .serviceConfiguration(configuration)
.credentialsProvider(StaticCredentialsProvider.create(credentials))
- region.foreach(builder.region)
- builder.build
+ region.foreach(builder.region)
+ builder.build
}
def s3Client(credentials: AwsCredentials, region: Option[Region]): S3Client = {
- s3Client(s3AdvancedConfiguration(), credentials, region)
+ s3Client(s3Configuration(), credentials, region)
}
- def s3AdvancedConfiguration(accelerateModeEnabled: Boolean = false,
- dualstackEnabled: Boolean = false,
- pathStyleAccessEnabled: Boolean = false): S3AdvancedConfiguration = {
+ def s3Configuration(accelerateModeEnabled: Boolean = false,
+ dualstackEnabled: Boolean = false,
+ pathStyleAccessEnabled: Boolean = false): S3Configuration = {
- S3AdvancedConfiguration.builder
+ S3Configuration.builder
.accelerateModeEnabled(accelerateModeEnabled)
.dualstackEnabled(dualstackEnabled)
.pathStyleAccessEnabled(pathStyleAccessEnabled)
diff --git a/cloudSupport/src/test/scala/cromwell/cloudsupport/aws/s3/S3StorageSpec.scala b/cloudSupport/src/test/scala/cromwell/cloudsupport/aws/s3/S3StorageSpec.scala
index 87b3091bc2d..4b23916e09d 100644
--- a/cloudSupport/src/test/scala/cromwell/cloudsupport/aws/s3/S3StorageSpec.scala
+++ b/cloudSupport/src/test/scala/cromwell/cloudsupport/aws/s3/S3StorageSpec.scala
@@ -29,9 +29,9 @@
* POSSIBILITY OF SUCH DAMAGE.
*/
package cromwell.cloudsupport.aws.s3
-import software.amazon.awssdk.core.auth.AnonymousCredentialsProvider
+import software.amazon.awssdk.auth.credentials.AnonymousCredentialsProvider
import org.scalatest.{FlatSpec, Matchers, Tag}
-import software.amazon.awssdk.core.regions.Region
+import software.amazon.awssdk.regions.Region
class S3StorageSpec extends FlatSpec with Matchers {
@@ -45,14 +45,17 @@ class S3StorageSpec extends FlatSpec with Matchers {
}
it should "build s3 storage" taggedAs S3StorageSpecUtils.AwsTest in {
- val configuration = S3Storage.s3AdvancedConfiguration(false, true)
+ val configuration = S3Storage.s3Configuration(dualstackEnabled = true)
configuration.accelerateModeEnabled should be(false)
configuration.dualstackEnabled should be(true)
configuration.pathStyleAccessEnabled should be(false)
}
it should "build s3 client with credentials" taggedAs S3StorageSpecUtils.AwsTest in {
- S3Storage.s3Client(AnonymousCredentialsProvider.create.getCredentials, Option(Region.US_EAST_1))
+ S3Storage.s3Client(
+ AnonymousCredentialsProvider.create.resolveCredentials(),
+ Option(Region.US_EAST_1)
+ )
}
}
diff --git a/dockerHashing/src/main/scala/cromwell/docker/DockerInfoActor.scala b/dockerHashing/src/main/scala/cromwell/docker/DockerInfoActor.scala
index 60745e8c460..fda13b4a135 100644
--- a/dockerHashing/src/main/scala/cromwell/docker/DockerInfoActor.scala
+++ b/dockerHashing/src/main/scala/cromwell/docker/DockerInfoActor.scala
@@ -9,14 +9,15 @@ import com.google.common.cache.CacheBuilder
import com.typesafe.config.Config
import common.validation.ErrorOr.ErrorOr
import common.validation.Validation._
-import cromwell.core.{Dispatcher, DockerConfiguration}
import cromwell.core.actor.StreamIntegration.{BackPressure, StreamContext}
+import cromwell.core.{Dispatcher, DockerConfiguration}
import cromwell.docker.DockerInfoActor._
import cromwell.docker.registryv2.DockerRegistryV2Abstract
import cromwell.docker.registryv2.flows.dockerhub.DockerHubRegistry
import cromwell.docker.registryv2.flows.gcr.GcrRegistry
import cromwell.docker.registryv2.flows.quay.QuayRegistry
import cromwell.util.GracefulShutdownHelper.ShutdownCommand
+import fs2.Pipe
import fs2.concurrent.{NoneTerminatedQueue, Queue}
import net.ceedubs.ficus.Ficus._
import org.http4s.client.blaze.BlazeClientBuilder
@@ -112,7 +113,7 @@ final class DockerInfoActor(
/*
* Sends back responses and adds to cache in case of success. Used as the sink for each registry stream.
*/
- val streamSink = fs2.Sink[IO, (DockerInfoResponse, DockerInfoContext)] {
+ val streamSink: Pipe[IO, (DockerInfoResponse, DockerInfoContext), Unit] = _ evalMap {
case (response: DockerInfoSuccessResponse, dockerInfoContext) =>
dockerInfoContext.replyTo ! response
IO.pure(cache.put(dockerInfoContext.dockerImageID, response.dockerInformation))
@@ -148,7 +149,7 @@ final class DockerInfoActor(
// Run requests in parallel - allow nbThreads max concurrent requests - order doesn't matter
.parEvalMapUnordered(registry.config.nbThreads)({ request => registry.run(request)(client) })
// Send to the sink for finalization of the result
- .to(streamSink)
+ .through(streamSink)
})
// Start the stream now asynchronously. It will keep running until we terminate the queue by sending None to it
diff --git a/docs/api/RESTAPI.md b/docs/api/RESTAPI.md
index ff40bf28615..45879915e31 100644
--- a/docs/api/RESTAPI.md
+++ b/docs/api/RESTAPI.md
@@ -1,5 +1,5 @@