diff --git a/.gitignore b/.gitignore index 1f20691f99e..42680886bf9 100644 --- a/.gitignore +++ b/.gitignore @@ -29,6 +29,7 @@ scripts/docker-compose-mysql/compose/mysql/data artifactory_credentials.properties aws_credentials bcs_application.conf +bcs_login.inc.sh centaur_secure.inc.conf cromwell-centaur-requester-pays-service-account.json cromwell-centaur-service-account.json diff --git a/centaur/src/main/resources/standardTestCases/labels_bad_workflow.test b/centaur/src/main/resources/standardTestCases/labels_bad_workflow.test index 426be8c4e73..9413985f25f 100644 --- a/centaur/src/main/resources/standardTestCases/labels_bad_workflow.test +++ b/centaur/src/main/resources/standardTestCases/labels_bad_workflow.test @@ -12,7 +12,7 @@ files { metadata { status: Failed - "submittedFiles.labels": "{\"label-key-1\":\"label-value-1\",\"label-key-2\":\"label-value-2\",\"only-key\":\"\",\"fc-id\":\"0123-abcd-4567-efgh\"}" + "submittedFiles.labels": "{\"fc-id\":\"0123-abcd-4567-efgh\",\"label-key-1\":\"label-value-1\",\"label-key-2\":\"label-value-2\",\"only-key\":\"\"}" "labels.label-key-1": "label-value-1" "labels.label-key-2": "label-value-2" "labels.only-key": "" diff --git a/centaur/src/main/resources/standardTestCases/prefix.test b/centaur/src/main/resources/standardTestCases/prefix.test index 457e01b8b35..a836843066e 100644 --- a/centaur/src/main/resources/standardTestCases/prefix.test +++ b/centaur/src/main/resources/standardTestCases/prefix.test @@ -3,6 +3,8 @@ testFormat: workflowsuccess files { workflow: prefix/prefix.wdl + # https://github.com/broadinstitute/cromwell/issues/4590 + options: prefix/prefix.options } metadata { diff --git a/centaur/src/main/resources/standardTestCases/prefix/prefix.options b/centaur/src/main/resources/standardTestCases/prefix/prefix.options new file mode 100644 index 00000000000..1840c33cd39 --- /dev/null +++ b/centaur/src/main/resources/standardTestCases/prefix/prefix.options @@ -0,0 +1,3 @@ +{ + "read_from_cache": false +} diff --git a/centaur/src/main/resources/standardTestCases/valid_labels.test b/centaur/src/main/resources/standardTestCases/valid_labels.test index 36010acd6de..1ab748f8a86 100644 --- a/centaur/src/main/resources/standardTestCases/valid_labels.test +++ b/centaur/src/main/resources/standardTestCases/valid_labels.test @@ -11,7 +11,7 @@ files { metadata { workflowName: wf_hello status: Succeeded - "submittedFiles.labels": "{\"label-key-1\":\"label-value-1\",\"label-key-2\":\"label-value-2\",\"only-key\":\"\",\"fc-id\":\"0123-abcd-4567-efgh\"}" + "submittedFiles.labels": "{\"fc-id\":\"0123-abcd-4567-efgh\",\"label-key-1\":\"label-value-1\",\"label-key-2\":\"label-value-2\",\"only-key\":\"\"}" "labels.label-key-1": "label-value-1" "labels.label-key-2": "label-value-2" "labels.only-key": "" diff --git a/centaurCwlRunner/src/main/resources/application.conf b/centaurCwlRunner/src/main/resources/application.conf index c6a5039833e..fcdd3da9c57 100644 --- a/centaurCwlRunner/src/main/resources/application.conf +++ b/centaurCwlRunner/src/main/resources/application.conf @@ -1,6 +1,15 @@ akka { loggers = ["akka.event.slf4j.Slf4jLogger"] logging-filter = "akka.event.slf4j.Slf4jLoggingFilter" + + # Silence Akka-Http warning logging of: + # [WARN] [01/28/2019 06:50:04.293] [centaur-acting-like-a-system-akka.actor.default-dispatcher-3] [centaur-acting-like-a-system/Pool(shared->http://localhost:8000)] Connection attempt failed. Backing off new connection attempts for at least 100 milliseconds. + # + # via: + # https://github.com/akka/akka/blob/v2.5.19/akka-actor/src/main/resources/reference.conf#L41-L44 + # https://github.com/akka/akka/blob/v2.5.19/akka-actor/src/main/scala/akka/event/Logging.scala#L377 + # https://github.com/akka/akka-http/blob/v10.1.7/akka-http-core/src/main/scala/akka/http/impl/engine/client/pool/NewHostConnectionPool.scala#L134 + stdout-loglevel = "ERROR" } centaur { diff --git a/centaurCwlRunner/src/main/resources/logback.xml b/centaurCwlRunner/src/main/resources/logback.xml index c21755aa774..d92bac61949 100644 --- a/centaurCwlRunner/src/main/resources/logback.xml +++ b/centaurCwlRunner/src/main/resources/logback.xml @@ -27,4 +27,22 @@ https://github.com/owlcs/owlapi/blob/owlapi-parent-5.1.7/api/src/main/java/org/semanticweb/owlapi/utilities/Injector.java#L383 --> + + + + + + diff --git a/cloud-nio/cloud-nio-impl-drs/src/main/scala/cloud/nio/impl/drs/DrsPathResolver.scala b/cloud-nio/cloud-nio-impl-drs/src/main/scala/cloud/nio/impl/drs/DrsPathResolver.scala index e56ecdf85cf..cc8bbf7dd26 100644 --- a/cloud-nio/cloud-nio-impl-drs/src/main/scala/cloud/nio/impl/drs/DrsPathResolver.scala +++ b/cloud-nio/cloud-nio-impl-drs/src/main/scala/cloud/nio/impl/drs/DrsPathResolver.scala @@ -15,10 +15,6 @@ import org.apache.http.impl.client.HttpClientBuilder import org.apache.http.util.EntityUtils import org.apache.http.{HttpResponse, HttpStatus} -//Do not remove this import. This import is required to compile, but IntelliJ doesn't correctly recognize that. -import cats.syntax.functor._ - - case class DrsPathResolver(drsConfig: DrsConfig, httpClientBuilder: HttpClientBuilder) { implicit lazy val urlDecoder: Decoder[Url] = deriveDecoder diff --git a/cloudSupport/src/main/scala/cromwell/cloudsupport/aws/AwsConfiguration.scala b/cloudSupport/src/main/scala/cromwell/cloudsupport/aws/AwsConfiguration.scala index 1e332eaf07d..396504afeff 100644 --- a/cloudSupport/src/main/scala/cromwell/cloudsupport/aws/AwsConfiguration.scala +++ b/cloudSupport/src/main/scala/cromwell/cloudsupport/aws/AwsConfiguration.scala @@ -42,7 +42,7 @@ import common.validation.Validation._ import cromwell.cloudsupport.aws.auth.{AssumeRoleMode, AwsAuthMode, CustomKeyMode, DefaultMode} import net.ceedubs.ficus.Ficus._ import org.slf4j.LoggerFactory -import software.amazon.awssdk.core.regions.Region +import software.amazon.awssdk.regions.Region final case class AwsConfiguration private (applicationName: String, authsByName: Map[String, AwsAuthMode], @@ -99,10 +99,7 @@ object AwsConfiguration { } def assumeRoleAuth(authConfig: Config, name: String, region: Option[String]): ErrorOr[AwsAuthMode] = validate { - val externalId = authConfig.hasPath("external-id") match { - case true => authConfig.getString("external-id") - case _ => "" - } + val externalId = authConfig.getOrElse("external-id", "") AssumeRoleMode( name, // We won't do anything with this now, but it is required for diff --git a/cloudSupport/src/main/scala/cromwell/cloudsupport/aws/auth/AwsAuthMode.scala b/cloudSupport/src/main/scala/cromwell/cloudsupport/aws/auth/AwsAuthMode.scala index f2338e98a00..43fb8844eb1 100644 --- a/cloudSupport/src/main/scala/cromwell/cloudsupport/aws/auth/AwsAuthMode.scala +++ b/cloudSupport/src/main/scala/cromwell/cloudsupport/aws/auth/AwsAuthMode.scala @@ -30,20 +30,14 @@ */ package cromwell.cloudsupport.aws.auth -import software.amazon.awssdk.core.auth.{AwsCredentials, - AwsSessionCredentials, - AnonymousCredentialsProvider, - DefaultCredentialsProvider, - StaticCredentialsProvider} -import software.amazon.awssdk.core.regions.Region -import software.amazon.awssdk.services.sts.{STSClient} -import software.amazon.awssdk.services.sts.model.{GetCallerIdentityRequest, - AssumeRoleRequest} - +import com.google.api.client.json.jackson2.JacksonFactory import cromwell.cloudsupport.aws.auth.AwsAuthMode.OptionLookup - import org.slf4j.LoggerFactory -import com.google.api.client.json.jackson2.JacksonFactory +import software.amazon.awssdk.auth.credentials._ +import software.amazon.awssdk.regions.Region +import software.amazon.awssdk.services.sts.StsClient +import software.amazon.awssdk.services.sts.model.{AssumeRoleRequest, GetCallerIdentityRequest} + import scala.util.{Failure, Success, Try} object AwsAuthMode { @@ -71,9 +65,9 @@ sealed trait AwsAuthMode { * All traits in this file are sealed, all classes final, meaning things * like Mockito or other java/scala overrides cannot work. */ - private[auth] var credentialValidation: ((AwsCredentials, Option[String]) => Unit) = + private[auth] var credentialValidation: (AwsCredentials, Option[String]) => Unit = (credentials: AwsCredentials, region: Option[String]) => { - val builder = STSClient.builder + val builder = StsClient.builder //If the region argument exists in config, set it in the builder. //Otherwise it is left unset and the AwsCredential builder will look in various places to supply, @@ -97,7 +91,7 @@ sealed trait AwsAuthMode { case object MockAuthMode extends AwsAuthMode { override val name = "no_auth" - lazy val _credential = AnonymousCredentialsProvider.create.getCredentials + lazy val _credential = AnonymousCredentialsProvider.create.resolveCredentials() override def credential(options: OptionLookup): AwsCredentials = _credential } @@ -113,7 +107,7 @@ final case class CustomKeyMode(override val name: String, // Validate credentials synchronously here, without retry. // It's very unlikely to fail as it should not happen more than a few times // (one for the engine and for each backend using it) per Cromwell instance. - validateCredential(AwsCredentials.create(accessKey, secretKey), region) + validateCredential(AwsBasicCredentials.create(accessKey, secretKey), region) } override def credential(options: OptionLookup): AwsCredentials = _credential @@ -130,7 +124,7 @@ final case class DefaultMode(override val name: String, region: Option[String]) // Validate credentials synchronously here, without retry. // It's very unlikely to fail as it should not happen more than a few times // (one for the engine and for each backend using it) per Cromwell instance. - validateCredential(DefaultCredentialsProvider.create.getCredentials, region) + validateCredential(DefaultCredentialsProvider.create.resolveCredentials(), region) } override def credential(options: OptionLookup): AwsCredentials = _credential @@ -157,7 +151,7 @@ final case class AssumeRoleMode(override val name: String, if (! externalId.isEmpty) requestBuilder.externalId(externalId) val request = requestBuilder.build - val builder = STSClient.builder + val builder = StsClient.builder region.foreach(str => builder.region(Region.of(str))) // See comment above regarding builder baseAuthObj match{ diff --git a/cloudSupport/src/main/scala/cromwell/cloudsupport/aws/s3/S3Storage.scala b/cloudSupport/src/main/scala/cromwell/cloudsupport/aws/s3/S3Storage.scala index ab0db7b1517..6a16a0c557c 100644 --- a/cloudSupport/src/main/scala/cromwell/cloudsupport/aws/s3/S3Storage.scala +++ b/cloudSupport/src/main/scala/cromwell/cloudsupport/aws/s3/S3Storage.scala @@ -30,12 +30,11 @@ */ package cromwell.cloudsupport.aws.s3 -import software.amazon.awssdk.services.s3.S3AdvancedConfiguration -import software.amazon.awssdk.services.s3.S3Client -import software.amazon.awssdk.core.auth.{AwsCredentials, StaticCredentialsProvider} import com.typesafe.config.ConfigFactory import net.ceedubs.ficus.Ficus._ -import software.amazon.awssdk.core.regions.Region +import software.amazon.awssdk.auth.credentials.{AwsCredentials, StaticCredentialsProvider} +import software.amazon.awssdk.regions.Region +import software.amazon.awssdk.services.s3.{S3Client, S3Configuration} object S3Storage { val DefaultConfiguration = { @@ -43,30 +42,30 @@ object S3Storage { val dualstackEnabled = ConfigFactory.load().as[Option[Boolean]]("s3.dual-stack").getOrElse(false) val pathStyleAccessEnabled = ConfigFactory.load().as[Option[Boolean]]("s3.path-style-access").getOrElse(false) - S3AdvancedConfiguration.builder + S3Configuration.builder .accelerateModeEnabled(accelerateModeEnabled) .dualstackEnabled(dualstackEnabled) .pathStyleAccessEnabled(pathStyleAccessEnabled) .build } - def s3Client(configuration: S3AdvancedConfiguration, credentials: AwsCredentials, region: Option[Region]): S3Client = { + def s3Client(configuration: S3Configuration, credentials: AwsCredentials, region: Option[Region]): S3Client = { val builder = S3Client.builder - .advancedConfiguration(configuration) + .serviceConfiguration(configuration) .credentialsProvider(StaticCredentialsProvider.create(credentials)) - region.foreach(builder.region) - builder.build + region.foreach(builder.region) + builder.build } def s3Client(credentials: AwsCredentials, region: Option[Region]): S3Client = { - s3Client(s3AdvancedConfiguration(), credentials, region) + s3Client(s3Configuration(), credentials, region) } - def s3AdvancedConfiguration(accelerateModeEnabled: Boolean = false, - dualstackEnabled: Boolean = false, - pathStyleAccessEnabled: Boolean = false): S3AdvancedConfiguration = { + def s3Configuration(accelerateModeEnabled: Boolean = false, + dualstackEnabled: Boolean = false, + pathStyleAccessEnabled: Boolean = false): S3Configuration = { - S3AdvancedConfiguration.builder + S3Configuration.builder .accelerateModeEnabled(accelerateModeEnabled) .dualstackEnabled(dualstackEnabled) .pathStyleAccessEnabled(pathStyleAccessEnabled) diff --git a/cloudSupport/src/test/scala/cromwell/cloudsupport/aws/s3/S3StorageSpec.scala b/cloudSupport/src/test/scala/cromwell/cloudsupport/aws/s3/S3StorageSpec.scala index 87b3091bc2d..4b23916e09d 100644 --- a/cloudSupport/src/test/scala/cromwell/cloudsupport/aws/s3/S3StorageSpec.scala +++ b/cloudSupport/src/test/scala/cromwell/cloudsupport/aws/s3/S3StorageSpec.scala @@ -29,9 +29,9 @@ * POSSIBILITY OF SUCH DAMAGE. */ package cromwell.cloudsupport.aws.s3 -import software.amazon.awssdk.core.auth.AnonymousCredentialsProvider +import software.amazon.awssdk.auth.credentials.AnonymousCredentialsProvider import org.scalatest.{FlatSpec, Matchers, Tag} -import software.amazon.awssdk.core.regions.Region +import software.amazon.awssdk.regions.Region class S3StorageSpec extends FlatSpec with Matchers { @@ -45,14 +45,17 @@ class S3StorageSpec extends FlatSpec with Matchers { } it should "build s3 storage" taggedAs S3StorageSpecUtils.AwsTest in { - val configuration = S3Storage.s3AdvancedConfiguration(false, true) + val configuration = S3Storage.s3Configuration(dualstackEnabled = true) configuration.accelerateModeEnabled should be(false) configuration.dualstackEnabled should be(true) configuration.pathStyleAccessEnabled should be(false) } it should "build s3 client with credentials" taggedAs S3StorageSpecUtils.AwsTest in { - S3Storage.s3Client(AnonymousCredentialsProvider.create.getCredentials, Option(Region.US_EAST_1)) + S3Storage.s3Client( + AnonymousCredentialsProvider.create.resolveCredentials(), + Option(Region.US_EAST_1) + ) } } diff --git a/dockerHashing/src/main/scala/cromwell/docker/DockerInfoActor.scala b/dockerHashing/src/main/scala/cromwell/docker/DockerInfoActor.scala index 60745e8c460..fda13b4a135 100644 --- a/dockerHashing/src/main/scala/cromwell/docker/DockerInfoActor.scala +++ b/dockerHashing/src/main/scala/cromwell/docker/DockerInfoActor.scala @@ -9,14 +9,15 @@ import com.google.common.cache.CacheBuilder import com.typesafe.config.Config import common.validation.ErrorOr.ErrorOr import common.validation.Validation._ -import cromwell.core.{Dispatcher, DockerConfiguration} import cromwell.core.actor.StreamIntegration.{BackPressure, StreamContext} +import cromwell.core.{Dispatcher, DockerConfiguration} import cromwell.docker.DockerInfoActor._ import cromwell.docker.registryv2.DockerRegistryV2Abstract import cromwell.docker.registryv2.flows.dockerhub.DockerHubRegistry import cromwell.docker.registryv2.flows.gcr.GcrRegistry import cromwell.docker.registryv2.flows.quay.QuayRegistry import cromwell.util.GracefulShutdownHelper.ShutdownCommand +import fs2.Pipe import fs2.concurrent.{NoneTerminatedQueue, Queue} import net.ceedubs.ficus.Ficus._ import org.http4s.client.blaze.BlazeClientBuilder @@ -112,7 +113,7 @@ final class DockerInfoActor( /* * Sends back responses and adds to cache in case of success. Used as the sink for each registry stream. */ - val streamSink = fs2.Sink[IO, (DockerInfoResponse, DockerInfoContext)] { + val streamSink: Pipe[IO, (DockerInfoResponse, DockerInfoContext), Unit] = _ evalMap { case (response: DockerInfoSuccessResponse, dockerInfoContext) => dockerInfoContext.replyTo ! response IO.pure(cache.put(dockerInfoContext.dockerImageID, response.dockerInformation)) @@ -148,7 +149,7 @@ final class DockerInfoActor( // Run requests in parallel - allow nbThreads max concurrent requests - order doesn't matter .parEvalMapUnordered(registry.config.nbThreads)({ request => registry.run(request)(client) }) // Send to the sink for finalization of the result - .to(streamSink) + .through(streamSink) }) // Start the stream now asynchronously. It will keep running until we terminate the queue by sending None to it diff --git a/docs/api/RESTAPI.md b/docs/api/RESTAPI.md index ff40bf28615..45879915e31 100644 --- a/docs/api/RESTAPI.md +++ b/docs/api/RESTAPI.md @@ -1,5 +1,5 @@