From 280003fb5ea82f5749133f786d2658eff8eee442 Mon Sep 17 00:00:00 2001 From: Khalid Shakir Date: Mon, 28 Jan 2019 21:06:06 -0500 Subject: [PATCH] Bump libraries and run centaur bcs. --- .gitignore | 1 + .../labels_bad_workflow.test | 2 +- .../resources/standardTestCases/prefix.test | 2 + .../standardTestCases/prefix/prefix.options | 3 + .../standardTestCases/valid_labels.test | 2 +- .../src/main/resources/application.conf | 9 + .../src/main/resources/logback.xml | 18 + .../cloud/nio/impl/drs/DrsPathResolver.scala | 4 - .../cloudsupport/aws/AwsConfiguration.scala | 7 +- .../cloudsupport/aws/auth/AwsAuthMode.scala | 30 +- .../cloudsupport/aws/s3/S3Storage.scala | 27 +- .../cloudsupport/aws/s3/S3StorageSpec.scala | 11 +- .../cromwell/docker/DockerInfoActor.scala | 7 +- docs/api/RESTAPI.md | 98 +++-- .../src/main/resources/swagger/cromwell.yaml | 8 +- .../cromwell/jobstore/JobResultSpec.scala | 6 +- .../webservice/SwaggerServiceSpec.scala | 15 +- .../routes/CromwellApiServiceSpec.scala | 16 +- .../java/org/lerch/s3fs/AmazonS3Factory.java | 35 +- .../s3/src/main/java/org/lerch/s3fs/LICENSE | 22 ++ .../s3/src/main/java/org/lerch/s3fs/README.md | 9 + .../java/org/lerch/s3fs/S3FileChannel.java | 6 +- .../org/lerch/s3fs/S3FileSystemProvider.java | 8 +- .../org/lerch/s3fs/S3SeekableByteChannel.java | 4 +- .../java/org/lerch/s3fs/util/S3Utils.java | 6 +- .../filesystems/s3/S3PathBuilder.scala | 20 +- .../filesystems/s3/S3PathBuilderFactory.scala | 2 +- .../filesystems/s3/S3PathBuilderSpec.scala | 12 +- project/Dependencies.scala | 371 ++++++++++-------- project/Merging.scala | 2 +- .../main/scala/statsdproxy/StatsDProxy.scala | 2 +- ...terializeWorkflowDescriptorActorSpec.scala | 3 +- src/ci/bin/test.inc.sh | 156 +++++--- src/ci/bin/testCentaurAws.sh | 2 +- src/ci/bin/testCentaurBcs.sh | 166 +++++--- src/ci/bin/testCentaurTes.sh | 6 +- src/ci/bin/testConformanceLocal.sh | 4 +- src/ci/bin/testConformancePapiV2.sh | 4 +- src/ci/bin/testConformanceTesk.sh | 4 +- src/ci/bin/test_bcs.inc.sh | 130 ++++++ src/ci/resources/bcs.jq | 31 ++ src/ci/resources/bcs_application.conf.ctmpl | 13 +- src/ci/resources/bcs_login.inc.sh.ctmpl | 17 + src/ci/resources/dockerhub_auth.inc.sh.ctmpl | 3 - .../AwsBatchBackendInitializationData.scala | 2 +- .../aws/AwsBatchInitializationActor.scala | 2 +- .../backend/impl/aws/AwsBatchJob.scala | 22 +- .../impl/aws/AwsBatchWorkflowPaths.scala | 2 +- ...tchAsyncBackendJobExecutionActorSpec.scala | 18 +- .../impl/aws/AwsBatchCallPathsSpec.scala | 20 +- .../backend/impl/aws/AwsBatchJobSpec.scala | 34 +- .../impl/aws/AwsBatchWorkflowPathsSpec.scala | 8 +- .../PipelinesApiInitializationActorSpec.scala | 4 +- 53 files changed, 928 insertions(+), 488 deletions(-) create mode 100644 centaur/src/main/resources/standardTestCases/prefix/prefix.options create mode 100644 filesystems/s3/src/main/java/org/lerch/s3fs/LICENSE create mode 100644 filesystems/s3/src/main/java/org/lerch/s3fs/README.md create mode 100644 src/ci/bin/test_bcs.inc.sh create mode 100644 src/ci/resources/bcs.jq create mode 100644 src/ci/resources/bcs_login.inc.sh.ctmpl diff --git a/.gitignore b/.gitignore index 1f20691f99e..42680886bf9 100644 --- a/.gitignore +++ b/.gitignore @@ -29,6 +29,7 @@ scripts/docker-compose-mysql/compose/mysql/data artifactory_credentials.properties aws_credentials bcs_application.conf +bcs_login.inc.sh centaur_secure.inc.conf cromwell-centaur-requester-pays-service-account.json cromwell-centaur-service-account.json diff --git a/centaur/src/main/resources/standardTestCases/labels_bad_workflow.test b/centaur/src/main/resources/standardTestCases/labels_bad_workflow.test index 426be8c4e73..9413985f25f 100644 --- a/centaur/src/main/resources/standardTestCases/labels_bad_workflow.test +++ b/centaur/src/main/resources/standardTestCases/labels_bad_workflow.test @@ -12,7 +12,7 @@ files { metadata { status: Failed - "submittedFiles.labels": "{\"label-key-1\":\"label-value-1\",\"label-key-2\":\"label-value-2\",\"only-key\":\"\",\"fc-id\":\"0123-abcd-4567-efgh\"}" + "submittedFiles.labels": "{\"fc-id\":\"0123-abcd-4567-efgh\",\"label-key-1\":\"label-value-1\",\"label-key-2\":\"label-value-2\",\"only-key\":\"\"}" "labels.label-key-1": "label-value-1" "labels.label-key-2": "label-value-2" "labels.only-key": "" diff --git a/centaur/src/main/resources/standardTestCases/prefix.test b/centaur/src/main/resources/standardTestCases/prefix.test index 457e01b8b35..a836843066e 100644 --- a/centaur/src/main/resources/standardTestCases/prefix.test +++ b/centaur/src/main/resources/standardTestCases/prefix.test @@ -3,6 +3,8 @@ testFormat: workflowsuccess files { workflow: prefix/prefix.wdl + # https://github.com/broadinstitute/cromwell/issues/4590 + options: prefix/prefix.options } metadata { diff --git a/centaur/src/main/resources/standardTestCases/prefix/prefix.options b/centaur/src/main/resources/standardTestCases/prefix/prefix.options new file mode 100644 index 00000000000..1840c33cd39 --- /dev/null +++ b/centaur/src/main/resources/standardTestCases/prefix/prefix.options @@ -0,0 +1,3 @@ +{ + "read_from_cache": false +} diff --git a/centaur/src/main/resources/standardTestCases/valid_labels.test b/centaur/src/main/resources/standardTestCases/valid_labels.test index 36010acd6de..1ab748f8a86 100644 --- a/centaur/src/main/resources/standardTestCases/valid_labels.test +++ b/centaur/src/main/resources/standardTestCases/valid_labels.test @@ -11,7 +11,7 @@ files { metadata { workflowName: wf_hello status: Succeeded - "submittedFiles.labels": "{\"label-key-1\":\"label-value-1\",\"label-key-2\":\"label-value-2\",\"only-key\":\"\",\"fc-id\":\"0123-abcd-4567-efgh\"}" + "submittedFiles.labels": "{\"fc-id\":\"0123-abcd-4567-efgh\",\"label-key-1\":\"label-value-1\",\"label-key-2\":\"label-value-2\",\"only-key\":\"\"}" "labels.label-key-1": "label-value-1" "labels.label-key-2": "label-value-2" "labels.only-key": "" diff --git a/centaurCwlRunner/src/main/resources/application.conf b/centaurCwlRunner/src/main/resources/application.conf index c6a5039833e..fcdd3da9c57 100644 --- a/centaurCwlRunner/src/main/resources/application.conf +++ b/centaurCwlRunner/src/main/resources/application.conf @@ -1,6 +1,15 @@ akka { loggers = ["akka.event.slf4j.Slf4jLogger"] logging-filter = "akka.event.slf4j.Slf4jLoggingFilter" + + # Silence Akka-Http warning logging of: + # [WARN] [01/28/2019 06:50:04.293] [centaur-acting-like-a-system-akka.actor.default-dispatcher-3] [centaur-acting-like-a-system/Pool(shared->http://localhost:8000)] Connection attempt failed. Backing off new connection attempts for at least 100 milliseconds. + # + # via: + # https://github.com/akka/akka/blob/v2.5.19/akka-actor/src/main/resources/reference.conf#L41-L44 + # https://github.com/akka/akka/blob/v2.5.19/akka-actor/src/main/scala/akka/event/Logging.scala#L377 + # https://github.com/akka/akka-http/blob/v10.1.7/akka-http-core/src/main/scala/akka/http/impl/engine/client/pool/NewHostConnectionPool.scala#L134 + stdout-loglevel = "ERROR" } centaur { diff --git a/centaurCwlRunner/src/main/resources/logback.xml b/centaurCwlRunner/src/main/resources/logback.xml index c21755aa774..d92bac61949 100644 --- a/centaurCwlRunner/src/main/resources/logback.xml +++ b/centaurCwlRunner/src/main/resources/logback.xml @@ -27,4 +27,22 @@ https://github.com/owlcs/owlapi/blob/owlapi-parent-5.1.7/api/src/main/java/org/semanticweb/owlapi/utilities/Injector.java#L383 --> + + + + + + diff --git a/cloud-nio/cloud-nio-impl-drs/src/main/scala/cloud/nio/impl/drs/DrsPathResolver.scala b/cloud-nio/cloud-nio-impl-drs/src/main/scala/cloud/nio/impl/drs/DrsPathResolver.scala index e56ecdf85cf..cc8bbf7dd26 100644 --- a/cloud-nio/cloud-nio-impl-drs/src/main/scala/cloud/nio/impl/drs/DrsPathResolver.scala +++ b/cloud-nio/cloud-nio-impl-drs/src/main/scala/cloud/nio/impl/drs/DrsPathResolver.scala @@ -15,10 +15,6 @@ import org.apache.http.impl.client.HttpClientBuilder import org.apache.http.util.EntityUtils import org.apache.http.{HttpResponse, HttpStatus} -//Do not remove this import. This import is required to compile, but IntelliJ doesn't correctly recognize that. -import cats.syntax.functor._ - - case class DrsPathResolver(drsConfig: DrsConfig, httpClientBuilder: HttpClientBuilder) { implicit lazy val urlDecoder: Decoder[Url] = deriveDecoder diff --git a/cloudSupport/src/main/scala/cromwell/cloudsupport/aws/AwsConfiguration.scala b/cloudSupport/src/main/scala/cromwell/cloudsupport/aws/AwsConfiguration.scala index 1e332eaf07d..396504afeff 100644 --- a/cloudSupport/src/main/scala/cromwell/cloudsupport/aws/AwsConfiguration.scala +++ b/cloudSupport/src/main/scala/cromwell/cloudsupport/aws/AwsConfiguration.scala @@ -42,7 +42,7 @@ import common.validation.Validation._ import cromwell.cloudsupport.aws.auth.{AssumeRoleMode, AwsAuthMode, CustomKeyMode, DefaultMode} import net.ceedubs.ficus.Ficus._ import org.slf4j.LoggerFactory -import software.amazon.awssdk.core.regions.Region +import software.amazon.awssdk.regions.Region final case class AwsConfiguration private (applicationName: String, authsByName: Map[String, AwsAuthMode], @@ -99,10 +99,7 @@ object AwsConfiguration { } def assumeRoleAuth(authConfig: Config, name: String, region: Option[String]): ErrorOr[AwsAuthMode] = validate { - val externalId = authConfig.hasPath("external-id") match { - case true => authConfig.getString("external-id") - case _ => "" - } + val externalId = authConfig.getOrElse("external-id", "") AssumeRoleMode( name, // We won't do anything with this now, but it is required for diff --git a/cloudSupport/src/main/scala/cromwell/cloudsupport/aws/auth/AwsAuthMode.scala b/cloudSupport/src/main/scala/cromwell/cloudsupport/aws/auth/AwsAuthMode.scala index f2338e98a00..43fb8844eb1 100644 --- a/cloudSupport/src/main/scala/cromwell/cloudsupport/aws/auth/AwsAuthMode.scala +++ b/cloudSupport/src/main/scala/cromwell/cloudsupport/aws/auth/AwsAuthMode.scala @@ -30,20 +30,14 @@ */ package cromwell.cloudsupport.aws.auth -import software.amazon.awssdk.core.auth.{AwsCredentials, - AwsSessionCredentials, - AnonymousCredentialsProvider, - DefaultCredentialsProvider, - StaticCredentialsProvider} -import software.amazon.awssdk.core.regions.Region -import software.amazon.awssdk.services.sts.{STSClient} -import software.amazon.awssdk.services.sts.model.{GetCallerIdentityRequest, - AssumeRoleRequest} - +import com.google.api.client.json.jackson2.JacksonFactory import cromwell.cloudsupport.aws.auth.AwsAuthMode.OptionLookup - import org.slf4j.LoggerFactory -import com.google.api.client.json.jackson2.JacksonFactory +import software.amazon.awssdk.auth.credentials._ +import software.amazon.awssdk.regions.Region +import software.amazon.awssdk.services.sts.StsClient +import software.amazon.awssdk.services.sts.model.{AssumeRoleRequest, GetCallerIdentityRequest} + import scala.util.{Failure, Success, Try} object AwsAuthMode { @@ -71,9 +65,9 @@ sealed trait AwsAuthMode { * All traits in this file are sealed, all classes final, meaning things * like Mockito or other java/scala overrides cannot work. */ - private[auth] var credentialValidation: ((AwsCredentials, Option[String]) => Unit) = + private[auth] var credentialValidation: (AwsCredentials, Option[String]) => Unit = (credentials: AwsCredentials, region: Option[String]) => { - val builder = STSClient.builder + val builder = StsClient.builder //If the region argument exists in config, set it in the builder. //Otherwise it is left unset and the AwsCredential builder will look in various places to supply, @@ -97,7 +91,7 @@ sealed trait AwsAuthMode { case object MockAuthMode extends AwsAuthMode { override val name = "no_auth" - lazy val _credential = AnonymousCredentialsProvider.create.getCredentials + lazy val _credential = AnonymousCredentialsProvider.create.resolveCredentials() override def credential(options: OptionLookup): AwsCredentials = _credential } @@ -113,7 +107,7 @@ final case class CustomKeyMode(override val name: String, // Validate credentials synchronously here, without retry. // It's very unlikely to fail as it should not happen more than a few times // (one for the engine and for each backend using it) per Cromwell instance. - validateCredential(AwsCredentials.create(accessKey, secretKey), region) + validateCredential(AwsBasicCredentials.create(accessKey, secretKey), region) } override def credential(options: OptionLookup): AwsCredentials = _credential @@ -130,7 +124,7 @@ final case class DefaultMode(override val name: String, region: Option[String]) // Validate credentials synchronously here, without retry. // It's very unlikely to fail as it should not happen more than a few times // (one for the engine and for each backend using it) per Cromwell instance. - validateCredential(DefaultCredentialsProvider.create.getCredentials, region) + validateCredential(DefaultCredentialsProvider.create.resolveCredentials(), region) } override def credential(options: OptionLookup): AwsCredentials = _credential @@ -157,7 +151,7 @@ final case class AssumeRoleMode(override val name: String, if (! externalId.isEmpty) requestBuilder.externalId(externalId) val request = requestBuilder.build - val builder = STSClient.builder + val builder = StsClient.builder region.foreach(str => builder.region(Region.of(str))) // See comment above regarding builder baseAuthObj match{ diff --git a/cloudSupport/src/main/scala/cromwell/cloudsupport/aws/s3/S3Storage.scala b/cloudSupport/src/main/scala/cromwell/cloudsupport/aws/s3/S3Storage.scala index ab0db7b1517..6a16a0c557c 100644 --- a/cloudSupport/src/main/scala/cromwell/cloudsupport/aws/s3/S3Storage.scala +++ b/cloudSupport/src/main/scala/cromwell/cloudsupport/aws/s3/S3Storage.scala @@ -30,12 +30,11 @@ */ package cromwell.cloudsupport.aws.s3 -import software.amazon.awssdk.services.s3.S3AdvancedConfiguration -import software.amazon.awssdk.services.s3.S3Client -import software.amazon.awssdk.core.auth.{AwsCredentials, StaticCredentialsProvider} import com.typesafe.config.ConfigFactory import net.ceedubs.ficus.Ficus._ -import software.amazon.awssdk.core.regions.Region +import software.amazon.awssdk.auth.credentials.{AwsCredentials, StaticCredentialsProvider} +import software.amazon.awssdk.regions.Region +import software.amazon.awssdk.services.s3.{S3Client, S3Configuration} object S3Storage { val DefaultConfiguration = { @@ -43,30 +42,30 @@ object S3Storage { val dualstackEnabled = ConfigFactory.load().as[Option[Boolean]]("s3.dual-stack").getOrElse(false) val pathStyleAccessEnabled = ConfigFactory.load().as[Option[Boolean]]("s3.path-style-access").getOrElse(false) - S3AdvancedConfiguration.builder + S3Configuration.builder .accelerateModeEnabled(accelerateModeEnabled) .dualstackEnabled(dualstackEnabled) .pathStyleAccessEnabled(pathStyleAccessEnabled) .build } - def s3Client(configuration: S3AdvancedConfiguration, credentials: AwsCredentials, region: Option[Region]): S3Client = { + def s3Client(configuration: S3Configuration, credentials: AwsCredentials, region: Option[Region]): S3Client = { val builder = S3Client.builder - .advancedConfiguration(configuration) + .serviceConfiguration(configuration) .credentialsProvider(StaticCredentialsProvider.create(credentials)) - region.foreach(builder.region) - builder.build + region.foreach(builder.region) + builder.build } def s3Client(credentials: AwsCredentials, region: Option[Region]): S3Client = { - s3Client(s3AdvancedConfiguration(), credentials, region) + s3Client(s3Configuration(), credentials, region) } - def s3AdvancedConfiguration(accelerateModeEnabled: Boolean = false, - dualstackEnabled: Boolean = false, - pathStyleAccessEnabled: Boolean = false): S3AdvancedConfiguration = { + def s3Configuration(accelerateModeEnabled: Boolean = false, + dualstackEnabled: Boolean = false, + pathStyleAccessEnabled: Boolean = false): S3Configuration = { - S3AdvancedConfiguration.builder + S3Configuration.builder .accelerateModeEnabled(accelerateModeEnabled) .dualstackEnabled(dualstackEnabled) .pathStyleAccessEnabled(pathStyleAccessEnabled) diff --git a/cloudSupport/src/test/scala/cromwell/cloudsupport/aws/s3/S3StorageSpec.scala b/cloudSupport/src/test/scala/cromwell/cloudsupport/aws/s3/S3StorageSpec.scala index 87b3091bc2d..4b23916e09d 100644 --- a/cloudSupport/src/test/scala/cromwell/cloudsupport/aws/s3/S3StorageSpec.scala +++ b/cloudSupport/src/test/scala/cromwell/cloudsupport/aws/s3/S3StorageSpec.scala @@ -29,9 +29,9 @@ * POSSIBILITY OF SUCH DAMAGE. */ package cromwell.cloudsupport.aws.s3 -import software.amazon.awssdk.core.auth.AnonymousCredentialsProvider +import software.amazon.awssdk.auth.credentials.AnonymousCredentialsProvider import org.scalatest.{FlatSpec, Matchers, Tag} -import software.amazon.awssdk.core.regions.Region +import software.amazon.awssdk.regions.Region class S3StorageSpec extends FlatSpec with Matchers { @@ -45,14 +45,17 @@ class S3StorageSpec extends FlatSpec with Matchers { } it should "build s3 storage" taggedAs S3StorageSpecUtils.AwsTest in { - val configuration = S3Storage.s3AdvancedConfiguration(false, true) + val configuration = S3Storage.s3Configuration(dualstackEnabled = true) configuration.accelerateModeEnabled should be(false) configuration.dualstackEnabled should be(true) configuration.pathStyleAccessEnabled should be(false) } it should "build s3 client with credentials" taggedAs S3StorageSpecUtils.AwsTest in { - S3Storage.s3Client(AnonymousCredentialsProvider.create.getCredentials, Option(Region.US_EAST_1)) + S3Storage.s3Client( + AnonymousCredentialsProvider.create.resolveCredentials(), + Option(Region.US_EAST_1) + ) } } diff --git a/dockerHashing/src/main/scala/cromwell/docker/DockerInfoActor.scala b/dockerHashing/src/main/scala/cromwell/docker/DockerInfoActor.scala index 60745e8c460..fda13b4a135 100644 --- a/dockerHashing/src/main/scala/cromwell/docker/DockerInfoActor.scala +++ b/dockerHashing/src/main/scala/cromwell/docker/DockerInfoActor.scala @@ -9,14 +9,15 @@ import com.google.common.cache.CacheBuilder import com.typesafe.config.Config import common.validation.ErrorOr.ErrorOr import common.validation.Validation._ -import cromwell.core.{Dispatcher, DockerConfiguration} import cromwell.core.actor.StreamIntegration.{BackPressure, StreamContext} +import cromwell.core.{Dispatcher, DockerConfiguration} import cromwell.docker.DockerInfoActor._ import cromwell.docker.registryv2.DockerRegistryV2Abstract import cromwell.docker.registryv2.flows.dockerhub.DockerHubRegistry import cromwell.docker.registryv2.flows.gcr.GcrRegistry import cromwell.docker.registryv2.flows.quay.QuayRegistry import cromwell.util.GracefulShutdownHelper.ShutdownCommand +import fs2.Pipe import fs2.concurrent.{NoneTerminatedQueue, Queue} import net.ceedubs.ficus.Ficus._ import org.http4s.client.blaze.BlazeClientBuilder @@ -112,7 +113,7 @@ final class DockerInfoActor( /* * Sends back responses and adds to cache in case of success. Used as the sink for each registry stream. */ - val streamSink = fs2.Sink[IO, (DockerInfoResponse, DockerInfoContext)] { + val streamSink: Pipe[IO, (DockerInfoResponse, DockerInfoContext), Unit] = _ evalMap { case (response: DockerInfoSuccessResponse, dockerInfoContext) => dockerInfoContext.replyTo ! response IO.pure(cache.put(dockerInfoContext.dockerImageID, response.dockerInformation)) @@ -148,7 +149,7 @@ final class DockerInfoActor( // Run requests in parallel - allow nbThreads max concurrent requests - order doesn't matter .parEvalMapUnordered(registry.config.nbThreads)({ request => registry.run(request)(client) }) // Send to the sink for finalization of the result - .to(streamSink) + .through(streamSink) }) // Start the stream now asynchronously. It will keep running until we terminate the queue by sending None to it diff --git a/docs/api/RESTAPI.md b/docs/api/RESTAPI.md index ff40bf28615..45879915e31 100644 --- a/docs/api/RESTAPI.md +++ b/docs/api/RESTAPI.md @@ -1,5 +1,5 @@