fix: Refactoring terraform

ai-ml/bionemo/eks.tf

@@ -1,3 +1,13 @@
+#---------------------------------------------------------------
+# Data Sources
+#---------------------------------------------------------------
+data "aws_availability_zones" "available" {}
+
+data "aws_eks_cluster_auth" "this" {
+  name = module.eks.cluster_name
+}
+
+
 #---------------------------------------------------------------
 # EKS Cluster
 #---------------------------------------------------------------

ai-ml/bionemo/locals.tf

@@ -0,0 +1,27 @@
+#---------------------------------------------------------------
+# Local Variables
+#---------------------------------------------------------------
+locals {
+  name     = var.name
+  region   = var.region
+
+  # Routable Private subnets only for Private NAT Gateway -> Transit Gateway -> Second VPC for overlapping CIDRs
+  # e.g., var.vpc_cidr = "10.1.0.0/21" => output: ["10.1.0.0/24", "10.1.1.0/24"] => 256-2 = 254 usable IPs per subnet/AZ
+  private_subnets = [for k, v in local.azs : cidrsubnet(var.vpc_cidr, 3, k)]
+  # Routable Public subnets with NAT Gateway and Internet Gateway
+  # e.g., var.vpc_cidr = "10.1.0.0/21" => output: ["10.1.2.0/26", "10.1.2.64/26"] => 64-2 = 62 usable IPs per subnet/AZ
+  public_subnets = [for k, v in local.azs : cidrsubnet(var.vpc_cidr, 5, k + 8)]
+
+  database_private_subnets = [for k, v in local.azs : cidrsubnet(var.vpc_cidr, 3, k + 5)]
+  # RFC6598 range 100.64.0.0/16 for EKS Data Plane for two subnets(32768 IPs per Subnet) across two AZs for EKS Control Plane ENI + Nodes + Pods
+  # e.g., var.secondary_cidr_blocks = "100.64.0.0/16" => output: ["100.64.0.0/17", "100.64.128.0/17"] => 32768-2 = 32766 usable IPs per subnet/AZ
+  secondary_ip_range_private_subnets = [for k, v in local.azs : cidrsubnet(element(var.secondary_cidr_blocks, 0), 1, k)]
+
+  vpc_cidr = var.vpc_cidr
+  azs      = slice(data.aws_availability_zones.available.names, 0, 2)
+
+  tags = {
+    Blueprint  = local.name
+    GithubRepo = "github.com/awslabs/data-on-eks"
+  }
+}

ai-ml/bionemo/main.tf → ai-ml/bionemo/providers.tf

@@ -1,3 +1,6 @@
+#---------------------------------------------------------------
+# Providers
+#---------------------------------------------------------------
 provider "aws" {
   region = local.region
 }
@@ -30,24 +33,3 @@ provider "kubectl" {
   load_config_file       = false
   token                  = data.aws_eks_cluster_auth.this.token
 }
-
-data "aws_availability_zones" "available" {}
-
-data "aws_eks_cluster_auth" "this" {
-  name = module.eks.cluster_name
-}
-
-#---------------------------------------------------------------
-# Local variables
-#---------------------------------------------------------------
-locals {
-  name     = var.name
-  region   = var.region
-  vpc_cidr = var.vpc_cidr
-  azs      = slice(data.aws_availability_zones.available.names, 0, 2)
-
-  tags = {
-    Blueprint  = local.name
-    GithubRepo = "github.com/awslabs/data-on-eks"
-  }
-}

ai-ml/bionemo/vpc.tf

@@ -1,17 +1,3 @@
-locals {
-  # Routable Private subnets only for Private NAT Gateway -> Transit Gateway -> Second VPC for overlapping CIDRs
-  # e.g., var.vpc_cidr = "10.1.0.0/21" => output: ["10.1.0.0/24", "10.1.1.0/24"] => 256-2 = 254 usable IPs per subnet/AZ
-  private_subnets = [for k, v in local.azs : cidrsubnet(var.vpc_cidr, 3, k)]
-  # Routable Public subnets with NAT Gateway and Internet Gateway
-  # e.g., var.vpc_cidr = "10.1.0.0/21" => output: ["10.1.2.0/26", "10.1.2.64/26"] => 64-2 = 62 usable IPs per subnet/AZ
-  public_subnets = [for k, v in local.azs : cidrsubnet(var.vpc_cidr, 5, k + 8)]
-
-  database_private_subnets = [for k, v in local.azs : cidrsubnet(var.vpc_cidr, 3, k + 5)]
-  # RFC6598 range 100.64.0.0/16 for EKS Data Plane for two subnets(32768 IPs per Subnet) across two AZs for EKS Control Plane ENI + Nodes + Pods
-  # e.g., var.secondary_cidr_blocks = "100.64.0.0/16" => output: ["100.64.0.0/17", "100.64.128.0/17"] => 32768-2 = 32766 usable IPs per subnet/AZ
-  secondary_ip_range_private_subnets = [for k, v in local.azs : cidrsubnet(element(var.secondary_cidr_blocks, 0), 1, k)]
-}
-
 #---------------------------------------------------------------
 # VPC
 #---------------------------------------------------------------

ai-ml/emr-spark-rapids/amp.tf

@@ -99,15 +99,8 @@ data "aws_iam_policy_document" "grafana" {
   }
 }
 
-#------------------------------------------
-# Amazon Prometheus
-#------------------------------------------
-locals {
-  amp_ingest_service_account = "amp-iamproxy-ingest-service-account"
-  amp_namespace              = "kube-prometheus-stack"
-}
-
 resource "aws_prometheus_workspace" "amp" {
+
   count = var.enable_amazon_prometheus ? 1 : 0
 
   alias = format("%s-%s", "amp-ws", local.name)

ai-ml/emr-spark-rapids/eks.tf

@@ -1,7 +1,22 @@
 #---------------------------------------------------------------
-# EKS Cluster
+# Data Sources
 #---------------------------------------------------------------
+data "aws_eks_cluster_auth" "this" {
+  name = module.eks.cluster_name
+}
+
+data "aws_ecrpublic_authorization_token" "token" {
+  provider = aws.ecr
+}
+
+data "aws_availability_zones" "available" {}
 
+data "aws_caller_identity" "current" {}
+data "aws_partition" "current" {}
+
+#---------------------------------------------------------------
+# EKS Cluster
+#---------------------------------------------------------------
 module "eks" {
   source  = "terraform-aws-modules/eks/aws"
   version = "~> 19.21"

ai-ml/emr-spark-rapids/locals.tf

@@ -0,0 +1,37 @@
+#---------------------------------------------------------------
+# Local Variables
+#---------------------------------------------------------------
+locals {
+  name   = var.name
+  region = var.region
+
+  # Only two AZs for this example
+  azs = slice(data.aws_availability_zones.available.names, 0, 2)
+
+  account_id = data.aws_caller_identity.current.account_id
+  partition  = data.aws_partition.current.partition
+
+  #------------------------------------------
+  # Amazon Prometheus
+  #------------------------------------------
+  amp_ingest_service_account = "amp-iamproxy-ingest-service-account"
+  amp_namespace              = "kube-prometheus-stack"
+
+  #------------------------------------------
+  # VPC
+  #------------------------------------------
+  # Routable Private subnets only for Private NAT Gateway -> Transit Gateway -> Second VPC for overlapping CIDRs
+  # e.g., var.vpc_cidr = "10.1.0.0/21" => output: ["10.1.0.0/24", "10.1.1.0/24"] => 256-2 = 254 usable IPs per subnet/AZ
+  private_subnets = [for k, v in local.azs : cidrsubnet(var.vpc_cidr, 3, k)]
+  # Routable Public subnets with NAT Gateway and Internet Gateway
+  # e.g., var.vpc_cidr = "10.1.0.0/21" => output: ["10.1.2.0/26", "10.1.2.64/26"] => 64-2 = 62 usable IPs per subnet/AZ
+  public_subnets = [for k, v in local.azs : cidrsubnet(var.vpc_cidr, 5, k + 8)]
+  # RFC6598 range 100.64.0.0/16 for EKS Data Plane for two subnets(32768 IPs per Subnet) across two AZs for EKS Control Plane ENI + Nodes + Pods
+  # e.g., var.secondary_cidr_blocks = "100.64.0.0/16" => output: ["100.64.0.0/17", "100.64.128.0/17"] => 32768-2 = 32766 usable IPs per subnet/AZ
+  secondary_ip_range_private_subnets = [for k, v in local.azs : cidrsubnet(element(var.secondary_cidr_blocks, 0), 1, k)]
+
+  tags = merge(var.tags, {
+    Blueprint  = local.name
+    GithubRepo = "github.com/awslabs/data-on-eks"
+  })
+}

ai-ml/emr-spark-rapids/outputs.tf

@@ -1,7 +1,6 @@
-################################################################################
-# Cluster
-################################################################################
-
+#---------------------------------------------------------------
+# EKS Cluster
+#---------------------------------------------------------------
 output "cluster_arn" {
   description = "The Amazon Resource Name (ARN) of the cluster"
   value       = module.eks.cluster_arn
@@ -17,10 +16,9 @@ output "oidc_provider_arn" {
   value       = module.eks.oidc_provider_arn
 }
 
-################################################################################
+#---------------------------------------------------------------
 # EKS Managed Node Group
-################################################################################
-
+#---------------------------------------------------------------
 output "configure_kubectl" {
   description = "Configure kubectl: make sure you're logged in with the correct AWS profile and run the following command to update your kubeconfig"
   value       = "aws eks --region ${local.region} update-kubeconfig --name ${module.eks.cluster_name}"
@@ -31,10 +29,9 @@ output "emr_on_eks" {
   value       = module.emr_containers
 }
 
-################################################################################
+#---------------------------------------------------------------
 # AMP
-################################################################################
-
+#---------------------------------------------------------------
 output "amp_workspace_id" {
   description = "The id of amp"
   value       = aws_prometheus_workspace.amp[0].id

ai-ml/emr-spark-rapids/providers.tf

@@ -0,0 +1,35 @@
+#---------------------------------------------------------------
+# Providers
+#---------------------------------------------------------------
+provider "aws" {
+  region = local.region
+}
+
+# ECR always authenticates with `us-east-1` region
+# Docs -> https://docs.aws.amazon.com/AmazonECR/latest/public/public-registries.html
+provider "aws" {
+  alias  = "ecr"
+  region = "us-east-1"
+}
+
+provider "kubernetes" {
+  host                   = module.eks.cluster_endpoint
+  cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data)
+  token                  = data.aws_eks_cluster_auth.this.token
+}
+
+provider "helm" {
+  kubernetes {
+    host                   = module.eks.cluster_endpoint
+    cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data)
+    token                  = data.aws_eks_cluster_auth.this.token
+  }
+}
+
+provider "kubectl" {
+  apply_retry_count      = 30
+  host                   = module.eks.cluster_endpoint
+  cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data)
+  load_config_file       = false
+  token                  = data.aws_eks_cluster_auth.this.token
+}

ai-ml/emr-spark-rapids/vpc.tf

@@ -1,15 +1,3 @@
-locals {
-  # Routable Private subnets only for Private NAT Gateway -> Transit Gateway -> Second VPC for overlapping CIDRs
-  # e.g., var.vpc_cidr = "10.1.0.0/21" => output: ["10.1.0.0/24", "10.1.1.0/24"] => 256-2 = 254 usable IPs per subnet/AZ
-  private_subnets = [for k, v in local.azs : cidrsubnet(var.vpc_cidr, 3, k)]
-  # Routable Public subnets with NAT Gateway and Internet Gateway
-  # e.g., var.vpc_cidr = "10.1.0.0/21" => output: ["10.1.2.0/26", "10.1.2.64/26"] => 64-2 = 62 usable IPs per subnet/AZ
-  public_subnets = [for k, v in local.azs : cidrsubnet(var.vpc_cidr, 5, k + 8)]
-  # RFC6598 range 100.64.0.0/16 for EKS Data Plane for two subnets(32768 IPs per Subnet) across two AZs for EKS Control Plane ENI + Nodes + Pods
-  # e.g., var.secondary_cidr_blocks = "100.64.0.0/16" => output: ["100.64.0.0/17", "100.64.128.0/17"] => 32768-2 = 32766 usable IPs per subnet/AZ
-  secondary_ip_range_private_subnets = [for k, v in local.azs : cidrsubnet(element(var.secondary_cidr_blocks, 0), 1, k)]
-}
-
 #---------------------------------------------------------------
 # VPC
 #---------------------------------------------------------------

ai-ml/jark-stack/terraform/eks.tf

@@ -1,3 +1,29 @@
+#---------------------------------------------------------------
+# Data Sources
+#---------------------------------------------------------------
+data "aws_eks_cluster_auth" "this" {
+  name = module.eks.cluster_name
+}
+
+data "aws_availability_zones" "available" {}
+
+data "aws_ecrpublic_authorization_token" "token" {
+  provider = aws.ecr
+}
+
+#---------------------------------------------------------------
+# EKS Cluster
+#---------------------------------------------------------------
+data "aws_eks_cluster_auth" "this" {
+  name = module.eks.cluster_name
+}
+
+data "aws_availability_zones" "available" {}
+
+data "aws_ecrpublic_authorization_token" "token" {
+  provider = aws.ecr
+}
+
 #---------------------------------------------------------------
 # EKS Cluster
 #---------------------------------------------------------------

ai-ml/jark-stack/terraform/locals.tf

@@ -0,0 +1,20 @@
+locals {
+  name   = var.name
+  region = var.region
+  azs    = slice(data.aws_availability_zones.available.names, 0, 2)
+
+  # Routable Private subnets only for Private NAT Gateway -> Transit Gateway -> Second VPC for overlapping CIDRs
+  # e.g., var.vpc_cidr = "10.1.0.0/21" => output: ["10.1.0.0/24", "10.1.1.0/24"] => 256-2 = 254 usable IPs per subnet/AZ
+  private_subnets = [for k, v in local.azs : cidrsubnet(var.vpc_cidr, 3, k)]
+  # Routable Public subnets with NAT Gateway and Internet Gateway
+  # e.g., var.vpc_cidr = "10.1.0.0/21" => output: ["10.1.2.0/26", "10.1.2.64/26"] => 64-2 = 62 usable IPs per subnet/AZ
+  public_subnets = [for k, v in local.azs : cidrsubnet(var.vpc_cidr, 5, k + 8)]
+  # RFC6598 range 100.64.0.0/16 for EKS Data Plane for two subnets(32768 IPs per Subnet) across two AZs for EKS Control Plane ENI + Nodes + Pods
+  # e.g., var.secondary_cidr_blocks = "100.64.0.0/16" => output: ["100.64.0.0/17", "100.64.128.0/17"] => 32768-2 = 32766 usable IPs per subnet/AZ
+  secondary_ip_range_private_subnets = [for k, v in local.azs : cidrsubnet(element(var.secondary_cidr_blocks, 0), 1, k)]
+
+  tags = {
+    Blueprint  = local.name
+    GithubRepo = "github.com/awslabs/data-on-eks"
+  }
+}

ai-ml/jark-stack/terraform/main.tf → ai-ml/jark-stack/terraform/providers.tf

@@ -29,23 +29,3 @@ provider "kubectl" {
   token                  = data.aws_eks_cluster_auth.this.token
   load_config_file       = false
 }
-
-data "aws_eks_cluster_auth" "this" {
-  name = module.eks.cluster_name
-}
-
-data "aws_availability_zones" "available" {}
-
-data "aws_ecrpublic_authorization_token" "token" {
-  provider = aws.ecr
-}
-
-locals {
-  name   = var.name
-  region = var.region
-  azs    = slice(data.aws_availability_zones.available.names, 0, 2)
-  tags = {
-    Blueprint  = local.name
-    GithubRepo = "github.com/awslabs/data-on-eks"
-  }
-}