Skip to content

Commit

Permalink
Google Auth Updates
Browse files Browse the repository at this point in the history 
- Replaced to-be-deprecated Credential (no 's') with Adapter around Credentials
- Removed dupe credentials adapting from PipelinesApiFactoryInterface
- Move service specific scopes (KMS, Genomics) out of GoogleAuthMode
- Changed credential creation methods to take scala collections
  • Loading branch information
@kshakir
kshakir committed Jun 1, 2019
1 parent c834828 commit 1b2faaf
Show file tree
Hide file tree
Showing 9 changed files with 98 additions and 128 deletions.
6 changes: 4 additions & 2 deletions centaur/src/main/scala/centaur/test/Test.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,8 @@ import centaur.test.metadata.WorkflowFlatMetadata
import centaur.test.metadata.WorkflowFlatMetadata._
import centaur.test.submit.SubmitHttpResponse
import centaur.test.workflow.Workflow
import com.google.api.services.genomics.Genomics
import com.google.api.services.genomics.{Genomics, GenomicsScopes}
import com.google.api.services.storage.StorageScopes
import com.google.auth.Credentials
import com.google.auth.http.HttpCredentialsAdapter
import com.google.auth.oauth2.ServiceAccountCredentials
Expand Down Expand Up @@ -86,9 +87,10 @@ object Operations {
lazy val googleConf: Config = CentaurConfig.conf.getConfig("google")
lazy val authName: String = googleConf.getString("auth")
lazy val genomicsEndpointUrl: String = googleConf.getString("genomics.endpoint-url")
lazy val genomicsAndStorageScopes = List(StorageScopes.CLOUD_PLATFORM_READ_ONLY, GenomicsScopes.GENOMICS)
lazy val credentials: Credentials = configuration.auth(authName)
.unsafe
.pipelinesApiCredentials(GoogleAuthMode.NoOptionLookup)
.credentials(genomicsAndStorageScopes)
lazy val credentialsProjectOption: Option[String] = {
Option(credentials) collect {
case serviceAccountCredentials: ServiceAccountCredentials => serviceAccountCredentials.getProjectId
Expand Down
98 changes: 17 additions & 81 deletions cloudSupport/src/main/scala/cromwell/cloudsupport/gcp/auth/GoogleAuthMode.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,15 +4,9 @@ import java.io.{ByteArrayInputStream, FileNotFoundException, InputStream}
import java.net.HttpURLConnection._

import better.files.File
import com.google.api.client.googleapis.auth.oauth2.GoogleCredential
import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport
import com.google.api.client.googleapis.testing.auth.oauth2.MockGoogleCredential
import com.google.api.client.http.HttpResponseException
import com.google.api.client.json.jackson2.JacksonFactory
import com.google.api.services.cloudkms.v1.CloudKMS
import com.google.api.services.compute.ComputeScopes
import com.google.api.services.genomics.v2alpha1.GenomicsScopes
import com.google.api.services.storage.StorageScopes
import com.google.auth.Credentials
import com.google.auth.http.HttpTransportFactory
import com.google.auth.oauth2.{GoogleCredentials, OAuth2Credentials, ServiceAccountCredentials, UserCredentials}
Expand Down Expand Up @@ -55,13 +49,6 @@ object GoogleAuthMode {
val DockerCredentialsEncryptionKeyNameKey = "docker_credentials_key_name"
val DockerCredentialsTokenKey = "docker_credentials_token"

private val PipelinesApiScopes = List(
StorageScopes.DEVSTORAGE_FULL_CONTROL,
StorageScopes.DEVSTORAGE_READ_WRITE,
GenomicsScopes.GENOMICS,
ComputeScopes.COMPUTE
)

def checkReadable(file: File) = {
if (!file.isReadable) throw new FileNotFoundException(s"File $file does not exist or is not readable")
}
Expand All @@ -85,24 +72,6 @@ object GoogleAuthMode {
}
}

def encryptKms(keyName: String, credential: GoogleCredential, plainText: String) = {
import com.google.api.services.cloudkms.v1.CloudKMSScopes

// Depending on the environment that provides the default credentials (e.g. Compute Engine, App
// Engine), the credentials may require us to specify the scopes we need explicitly.
// Check for this case, and inject the scope if required.
val scopedCredential = if (credential.createScopedRequired) credential.createScoped(CloudKMSScopes.all) else credential

val kms = new CloudKMS.Builder(httpTransport, jsonFactory, scopedCredential)
.setApplicationName("cromwell")
.build()

import com.google.api.services.cloudkms.v1.model.EncryptRequest
val request = new EncryptRequest().encodePlaintext(plainText.toCharArray.map(_.toByte))
val response = kms.projects.locations.keyRings.cryptoKeys.encrypt(keyName, request).execute
response.getCiphertext
}

/** Used for both checking that the credential is valid and creating a fresh credential. */
private def refreshCredentials(credentials: Credentials): Unit = {
credentials.refresh()
Expand All @@ -115,49 +84,33 @@ sealed trait GoogleAuthMode {

def name: String

// Create a Credential object from the google.api.client.auth library (https://github.com/google/google-api-java-client)
def credentials(options: OptionLookup, scopes: java.util.Collection[String]): OAuth2Credentials

/**
* Create a credential object suitable for use with Pipelines API.
*
* @param options A lookup for external credential information.
* @return Credentials with scopes compatible with the Genomics API compute and storage.
* Creates OAuth credentials with the specified scopes.
*/
def pipelinesApiCredentials(options: OptionLookup): OAuth2Credentials = {
credentials(options, PipelinesApiScopes.asJavaCollection)
}
def credentials(options: OptionLookup, scopes: Iterable[String]): OAuth2Credentials

/**
* Alias for credentials(GoogleAuthMode.NoOptionLookup, scopes).
* Only valid for credentials that are NOT externally provided, such as ApplicationDefault.
*/
def credentials(scopes: Iterable[String]): OAuth2Credentials = {
credentials(GoogleAuthMode.NoOptionLookup, scopes.asJavaCollection)
}

/**
* Alias for credentials(GoogleAuthMode.NoOptionLookup, scopes).
* Only valid for credentials that are NOT externally provided, such as ApplicationDefault.
*/
def credentials(scopes: java.util.Collection[String]): OAuth2Credentials = {
credentials(GoogleAuthMode.NoOptionLookup, scopes)
}

/**
* Alias for credentials(GoogleAuthMode.NoOptionLookup, Set.empty).
* Alias for credentials(GoogleAuthMode.NoOptionLookup, Nil).
* Only valid for credentials that are NOT externally provided and do not need scopes, such as ApplicationDefault.
*/
private[auth] def credentials(): OAuth2Credentials = {
credentials(GoogleAuthMode.NoOptionLookup, java.util.Collections.emptySet[String])
credentials(GoogleAuthMode.NoOptionLookup, Nil)
}

/**
* Alias for credentials(options, Set.empty).
* Alias for credentials(options, Nil).
* Only valid for credentials that are NOT externally provided and do not need scopes, such as ApplicationDefault.
*/
private[auth] def credentials(options: OptionLookup): OAuth2Credentials = {
credentials(options, java.util.Collections.emptySet[String])
credentials(options, Nil)
}

def requiresAuthFile: Boolean = false
Expand All @@ -176,20 +129,14 @@ sealed trait GoogleAuthMode {
case Success(_) => credential
}
}

def apiClientGoogleCredential(options: OptionLookup): Option[GoogleCredential] = None
}

case object MockAuthMode extends GoogleAuthMode {
override val name = "no_auth"

override def credentials(unusedOptions: OptionLookup, unusedScopes: java.util.Collection[String]): NoCredentials = {
override def credentials(unusedOptions: OptionLookup, unusedScopes: Iterable[String]): NoCredentials = {
NoCredentials.getInstance
}

override def apiClientGoogleCredential(options: OptionLookup): Option[MockGoogleCredential] = {
Option(new MockGoogleCredential.Builder().build())
}
}

object ServiceAccountMode {
Expand All @@ -204,20 +151,12 @@ object ServiceAccountMode {

}

trait HasApiClientGoogleCredentialStream { self: GoogleAuthMode =>
protected def credentialStream(options: OptionLookup): InputStream

override def apiClientGoogleCredential(options: OptionLookup): Option[GoogleCredential] = Option(GoogleCredential.fromStream(credentialStream(options)))
}

final case class ServiceAccountMode(override val name: String,
fileFormat: CredentialFileFormat)
extends GoogleAuthMode with HasApiClientGoogleCredentialStream {
extends GoogleAuthMode {
private val credentialsFile = File(fileFormat.file)
checkReadable(credentialsFile)

override protected def credentialStream(options: OptionLookup): InputStream = credentialsFile.newInputStream

private lazy val serviceAccountCredentials: ServiceAccountCredentials = {
fileFormat match {
case PemFileFormat(accountId, _) =>
Expand All @@ -228,25 +167,24 @@ final case class ServiceAccountMode(override val name: String,
}

override def credentials(unusedOptions: OptionLookup,
scopes: java.util.Collection[String]): GoogleCredentials = {
val scopedCredentials = serviceAccountCredentials.createScoped(scopes)
scopes: Iterable[String]): GoogleCredentials = {
val scopedCredentials = serviceAccountCredentials.createScoped(scopes.asJavaCollection)
validateCredentials(scopedCredentials)
}
}

final case class UserServiceAccountMode(override val name: String)
extends GoogleAuthMode with HasApiClientGoogleCredentialStream {
final case class UserServiceAccountMode(override val name: String) extends GoogleAuthMode {
private def extractServiceAccount(options: OptionLookup): String = {
extract(options, UserServiceAccountKey)
}

override protected def credentialStream(options: OptionLookup): InputStream = {
private def credentialStream(options: OptionLookup): InputStream = {
new ByteArrayInputStream(extractServiceAccount(options).getBytes("UTF-8"))
}

override def credentials(options: OptionLookup, scopes: java.util.Collection[String]): GoogleCredentials = {
override def credentials(options: OptionLookup, scopes: Iterable[String]): GoogleCredentials = {
val newCredentials = ServiceAccountCredentials.fromStream(credentialStream(options))
val scopedCredentials: GoogleCredentials = newCredentials.createScoped(scopes)
val scopedCredentials: GoogleCredentials = newCredentials.createScoped(scopes.asJavaCollection)
validateCredentials(scopedCredentials)
}
}
Expand All @@ -267,7 +205,7 @@ final case class UserMode(override val name: String,
validateCredentials(UserCredentials.fromStream(secretsStream))
}

override def credentials(unusedOptions: OptionLookup, unusedScopes: java.util.Collection[String]): OAuth2Credentials = {
override def credentials(unusedOptions: OptionLookup, unusedScopes: Iterable[String]): OAuth2Credentials = {
userCredentials
}
}
Expand All @@ -278,11 +216,9 @@ object ApplicationDefaultMode {

final case class ApplicationDefaultMode(name: String) extends GoogleAuthMode {
override def credentials(unusedOptions: OptionLookup,
unusedScopes: java.util.Collection[String]): GoogleCredentials = {
unusedScopes: Iterable[String]): GoogleCredentials = {
ApplicationDefaultMode.applicationDefaultCredentials
}

override def apiClientGoogleCredential(unused: OptionLookup): Option[GoogleCredential] = Option(GoogleCredential.getApplicationDefault(httpTransport, jsonFactory))
}

final case class RefreshTokenMode(name: String,
Expand All @@ -297,7 +233,7 @@ final case class RefreshTokenMode(name: String,
extract(options, RefreshTokenOptionKey)
}

override def credentials(options: OptionLookup, unusedScopes: java.util.Collection[String]): UserCredentials = {
override def credentials(options: OptionLookup, unusedScopes: Iterable[String]): UserCredentials = {
val refreshToken = extractRefreshToken(options)
val newCredentials: UserCredentials = UserCredentials
.newBuilder()
Expand Down
9 changes: 3 additions & 6 deletions filesystems/drs/src/main/scala/cromwell/filesystems/drs/DrsPathBuilderFactory.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,10 +16,8 @@ import cromwell.core.WorkflowOptions
import cromwell.core.path.{PathBuilder, PathBuilderFactory}
import org.apache.http.impl.client.HttpClientBuilder

import scala.collection.JavaConverters._
import scala.concurrent.{ExecutionContext, Future}


/**
* Cromwell Wrapper around DrsFileSystems to load the configuration.
* This class is used as the global configuration class in the drs filesystem
Expand Down Expand Up @@ -54,10 +52,9 @@ class DrsPathBuilderFactory(globalConfig: Config, instanceConfig: Config, single

private def inputReadChannel(url: String, urlScheme: String, serviceAccount: String): IO[ReadableByteChannel] = {
urlScheme match {
case GcsScheme => {
case GcsScheme =>
val Array(bucket, fileToBeLocalized) = url.replace(s"$GcsScheme://", "").split("/", 2)
gcsInputStream(GcsFilePath(bucket, fileToBeLocalized), serviceAccount)
}
case otherScheme => IO.raiseError(new UnsupportedOperationException(s"DRS currently doesn't support reading files for $otherScheme."))
}
}
Expand All @@ -82,8 +79,8 @@ class DrsPathBuilderFactory(globalConfig: Config, instanceConfig: Config, single
// Profile and Email scopes are requirements for interacting with Martha v2
Oauth2Scopes.USERINFO_EMAIL,
Oauth2Scopes.USERINFO_PROFILE
).asJavaCollection
val authCredentials = googleAuthMode.credentials((key: String) => options.get(key).get, marthaScopes)
)
val authCredentials = googleAuthMode.credentials(options.get(_).get, marthaScopes)

Future.successful(DrsPathBuilder(new DrsCloudNioFileSystemProvider(singletonConfig.config, authCredentials, httpClientBuilder, drsReadInterpreter)))
}
Expand Down
3 changes: 2 additions & 1 deletion filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/GcsPathBuilder.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import akka.http.scaladsl.model.ContentTypes
import better.files.File.OpenOptions
import cats.effect.IO
import com.google.api.gax.retrying.RetrySettings
import com.google.api.services.storage.StorageScopes
import com.google.auth.Credentials
import com.google.cloud.storage.Storage.BlobTargetOption
import com.google.cloud.storage.contrib.nio.{CloudStorageConfiguration, CloudStorageFileSystem, CloudStoragePath}
Expand Down Expand Up @@ -100,7 +101,7 @@ object GcsPathBuilder {
cloudStorageConfiguration: CloudStorageConfiguration,
options: WorkflowOptions,
defaultProject: Option[String])(implicit as: ActorSystem, ec: ExecutionContext): Future[GcsPathBuilder] = {
authMode.retryPipelinesApiCredentials(options) map { credentials =>
authMode.retryCredentials(options, List(StorageScopes.DEVSTORAGE_FULL_CONTROL)) map { credentials =>
fromCredentials(credentials,
applicationName,
retrySettings,
Expand Down
11 changes: 7 additions & 4 deletions filesystems/gcs/src/main/scala/cromwell/filesystems/gcs/GoogleUtil.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,13 +24,16 @@ object GoogleUtil {

implicit class EnhancedGoogleAuthMode(val googleAuthMode: GoogleAuthMode) extends AnyVal {
/**
* Retries getting the pipelines API credentials three times.
* Retries getting the credentials three times.
*
* There is nothing GCS specific about this method. This package just happens to be the lowest level with access
* to core's version of Retry + cloudSupport's implementation of GoogleAuthMode.
*/
def retryPipelinesApiCredentials(options: WorkflowOptions)
(implicit as: ActorSystem, ec: ExecutionContext): Future[Credentials] = {
def retryCredentials(options: WorkflowOptions, scopes: Iterable[String])
(implicit actorSystem: ActorSystem, executionContext: ExecutionContext): Future[Credentials] = {
def credential(): Credentials = {
try {
googleAuthMode.pipelinesApiCredentials((key: String) => options.get(key).get)
googleAuthMode.credentials(options.get(_).get, scopes)
} catch {
case exception: OptionLookupException =>
throw new IllegalArgumentException(s"Missing parameters in workflow options: ${exception.key}", exception)
Expand Down
7 changes: 5 additions & 2 deletions ...a/cromwell/services/healthmonitor/impl/workbench/WorkbenchHealthMonitorServiceActor.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@ import cats.instances.future._
import cats.syntax.functor._
import com.google.api.client.http.{HttpRequest, HttpRequestInitializer}
import com.google.api.gax.retrying.RetrySettings
import com.google.api.services.genomics.v2alpha1.GenomicsScopes
import com.google.api.services.storage.StorageScopes
import com.google.auth.Credentials
import com.google.auth.http.HttpCredentialsAdapter
import com.typesafe.config.Config
Expand Down Expand Up @@ -57,7 +59,8 @@ abstract class WorkbenchHealthMonitorServiceActor(val serviceConfig: Config, glo
private def checkGcs(): Future[SubsystemStatus] = {
// For any expected production usage of this check, the GCS bucket should be public read */
val gcsBucketToCheck = serviceConfig.as[String]("gcs-bucket-to-check")
val storage = Future(googleAuth.pipelinesApiCredentials(GoogleAuthMode.NoOptionLookup)) map { credentials =>
val storageScopes = List(StorageScopes.DEVSTORAGE_READ_ONLY)
val storage = Future(googleAuth.credentials(storageScopes)) map { credentials =>
GcsStorage.gcsStorage(googleConfig.applicationName, credentials, RetrySettings.newBuilder().build())
}
storage map { _.buckets.get(gcsBucketToCheck).execute() } as OkStatus
Expand All @@ -71,7 +74,7 @@ abstract class WorkbenchHealthMonitorServiceActor(val serviceConfig: Config, glo
val papiProjectId = papiConfig.as[String]("project")

val check = for {
credentials <- Future(googleAuth.pipelinesApiCredentials(GoogleAuthMode.NoOptionLookup))
credentials <- Future(googleAuth.credentials(List(GenomicsScopes.GENOMICS)))
genomicsChecker = if (papiProviderConfig.as[String]("actor-factory").contains("v2alpha1"))
GenomicsCheckerV2(googleConfig.applicationName, googleAuth, endpointUrl, credentials, papiProjectId)
else
Expand Down
Loading

0 comments on commit 1b2faaf

Please sign in to comment.