Tag ECS services, pull in subset of upstream changes

Author: Limess

README.md

@@ -73,7 +73,7 @@ pre-commit run --all-files
 | <a name="input_ui_allow_list"></a> [ui\_allow\_list](#input\_ui\_allow\_list) | List of CIDRs we want to grant access to our Metaflow UI Service. Usually this is our VPN's CIDR blocks. | `list(string)` | `[]` | no |
 | <a name="input_ui_certificate_arn"></a> [ui\_certificate\_arn](#input\_ui\_certificate\_arn) | SSL certificate for UI. If no certificate arn is provided, HTTP will be used. | `string` | `null` | no |
 | <a name="input_ui_static_container_image"></a> [ui\_static\_container\_image](#input\_ui\_static\_container\_image) | Container image for the UI frontend app | `string` | `""` | no |
-| <a name="input_vpc_cidr_block"></a> [vpc\_cidr\_block](#input\_vpc\_cidr\_block) | The VPC CIDR block that we'll access list on our Metadata Service API to allow all internal communications | `string` | n/a | yes |
+| <a name="input_vpc_cidr_blocks"></a> [vpc\_cidr\_blocks](#input\_vpc\_cidr\_blocks) | The VPC CIDR blocks that we'll access list on our Metadata Service API to allow all internal communications | `list(string)` | n/a | yes |
 | <a name="input_vpc_id"></a> [vpc\_id](#input\_vpc\_id) | The id of the single VPC we stood up for all Metaflow resources to exist in. | `string` | n/a | yes |
 
 ## Outputs

examples/eks/metaflow.tf

@@ -59,7 +59,7 @@ module "metaflow-metadata-service" {
   s3_bucket_arn                        = module.metaflow-datastore.s3_bucket_arn
   subnet1_id                           = module.vpc.private_subnets[0]
   subnet2_id                           = module.vpc.private_subnets[1]
-  vpc_cidr_block                       = module.vpc.vpc_cidr_block
+  vpc_cidr_blocks                      = module.vpc.vpc_cidr_blocks
 
   standard_tags = local.tags
 }

examples/minimal/minimal_example.tf

@@ -38,15 +38,15 @@ module "vpc" {
 
 module "metaflow" {
   source  = "outerbounds/metaflow/aws"
-  version = "0.3.0"
+  version = "0.7.0"
 
   resource_prefix = local.resource_prefix
   resource_suffix = local.resource_suffix
 
   enable_step_functions = false
   subnet1_id            = module.vpc.public_subnets[0]
   subnet2_id            = module.vpc.public_subnets[1]
-  vpc_cidr_block        = module.vpc.vpc_cidr_block
+  vpc_cidr_blocks       = module.vpc.vpc_cidr_blocks
   vpc_id                = module.vpc.vpc_id
 
   tags = {

main.tf

@@ -34,7 +34,7 @@ module "metaflow-metadata-service" {
   s3_bucket_arn                        = module.metaflow-datastore.s3_bucket_arn
   subnet1_id                           = var.subnet1_id
   subnet2_id                           = var.subnet2_id
-  vpc_cidr_block                       = var.vpc_cidr_block
+  vpc_cidr_blocks                      = var.vpc_cidr_blocks
 
   standard_tags = var.tags
 }

modules/metadata-service/README.md

@@ -17,14 +17,17 @@ If the `access_list_cidr_blocks` variable is set, only traffic originating from
 |------|-------------|------|---------|:--------:|
 | <a name="input_access_list_cidr_blocks"></a> [access\_list\_cidr\_blocks](#input\_access\_list\_cidr\_blocks) | List of CIDRs we want to grant access to our Metaflow Metadata Service. Usually this is our VPN's CIDR blocks. | `list(string)` | n/a | yes |
 | <a name="input_api_basic_auth"></a> [api\_basic\_auth](#input\_api\_basic\_auth) | Enable basic auth for API Gateway? (requires key export) | `bool` | `true` | no |
+| <a name="input_database_name"></a> [database\_name](#input\_database\_name) | The database name | `string` | `"metaflow"` | no |
 | <a name="input_database_password_secret_manager_arn"></a> [database\_password\_secret\_manager\_arn](#input\_database\_password\_secret\_manager\_arn) | The arn of the database password stored in AWS secrets manager | `string` | n/a | yes |
 | <a name="input_database_username"></a> [database\_username](#input\_database\_username) | The database username | `string` | n/a | yes |
 | <a name="input_datastore_s3_bucket_kms_key_arn"></a> [datastore\_s3\_bucket\_kms\_key\_arn](#input\_datastore\_s3\_bucket\_kms\_key\_arn) | The ARN of the KMS key used to encrypt the Metaflow datastore S3 bucket | `string` | n/a | yes |
 | <a name="input_ecs_cluster_id"></a> [ecs\_cluster\_id](#input\_ecs\_cluster\_id) | The ID of an existing ECS cluster to run services on. If no cluster ID is specfied, a new cluster will be created. | `string` | `null` | no |
 | <a name="input_fargate_execution_role_arn"></a> [fargate\_execution\_role\_arn](#input\_fargate\_execution\_role\_arn) | The IAM role that grants access to ECS and Batch services which we'll use as our Metadata Service API's execution\_role for our Fargate instance | `string` | n/a | yes |
 | <a name="input_iam_partition"></a> [iam\_partition](#input\_iam\_partition) | IAM Partition (Select aws-us-gov for AWS GovCloud, otherwise leave as is) | `string` | `"aws"` | no |
 | <a name="input_is_gov"></a> [is\_gov](#input\_is\_gov) | Set to true if IAM partition is 'aws-us-gov' | `bool` | `false` | no |
-| <a name="input_metadata_service_container_image"></a> [metadata\_service\_container\_image](#input\_metadata\_service\_container\_image) | Container image for metadata service | `string` | `""` | no |
+| <a name="input_metadata_service_container_image"></a> [metadata\_service\_container\_image](#input\_metadata\_service\_container\_image) | Container image for metadata service | `string` | n/a | yes |
+| <a name="input_metadata_service_cpu"></a> [metadata\_service\_cpu](#input\_metadata\_service\_cpu) | ECS task CPU unit for metadata service | `number` | `512` | no |
+| <a name="input_metadata_service_memory"></a> [metadata\_service\_memory](#input\_metadata\_service\_memory) | ECS task memory in MiB for metadata service | `number` | `1024` | no |
 | <a name="input_metaflow_vpc_id"></a> [metaflow\_vpc\_id](#input\_metaflow\_vpc\_id) | ID of the Metaflow VPC this SageMaker notebook instance is to be deployed in | `string` | n/a | yes |
 | <a name="input_rds_master_instance_endpoint"></a> [rds\_master\_instance\_endpoint](#input\_rds\_master\_instance\_endpoint) | The database connection endpoint in address:port format | `string` | n/a | yes |
 | <a name="input_resource_prefix"></a> [resource\_prefix](#input\_resource\_prefix) | Prefix given to all AWS resources to differentiate between applications | `string` | n/a | yes |
@@ -33,7 +36,8 @@ If the `access_list_cidr_blocks` variable is set, only traffic originating from
 | <a name="input_standard_tags"></a> [standard\_tags](#input\_standard\_tags) | The standard tags to apply to every AWS resource. | `map(string)` | n/a | yes |
 | <a name="input_subnet1_id"></a> [subnet1\_id](#input\_subnet1\_id) | First private subnet used for availability zone redundancy | `string` | n/a | yes |
 | <a name="input_subnet2_id"></a> [subnet2\_id](#input\_subnet2\_id) | Second private subnet used for availability zone redundancy | `string` | n/a | yes |
-| <a name="input_vpc_cidr_block"></a> [vpc\_cidr\_block](#input\_vpc\_cidr\_block) | The VPC CIDR block that we'll access list on our Metadata Service API to allow all internal communications | `string` | n/a | yes |
+| <a name="input_vpc_cidr_blocks"></a> [vpc\_cidr\_blocks](#input\_vpc\_cidr\_blocks) | The VPC CIDR blocks that we'll access list on our Metadata Service API to allow all internal communications | `list(string)` | n/a | yes |
+| <a name="input_with_public_ip"></a> [with\_public\_ip](#input\_with\_public\_ip) | Enable private IP by default | `bool` | `false` | no |
 
 ## Outputs
 

modules/metadata-service/ec2.tf

@@ -7,15 +7,15 @@ resource "aws_security_group" "metadata_service_security_group" {
     from_port   = 8080
     to_port     = 8080
     protocol    = "tcp"
-    cidr_blocks = [var.vpc_cidr_block]
+    cidr_blocks = var.vpc_cidr_blocks
     description = "Allow API calls internally"
   }
 
   ingress {
     from_port   = 8082
     to_port     = 8082
     protocol    = "tcp"
-    cidr_blocks = [var.vpc_cidr_block]
+    cidr_blocks = var.vpc_cidr_blocks
     description = "Allow API calls internally"
   }
 

modules/metadata-service/ecs.tf

@@ -19,8 +19,8 @@ resource "aws_ecs_task_definition" "this" {
       "name"      = "${var.resource_prefix}service${var.resource_suffix}",
       "image"     = "${var.metadata_service_container_image}",
       "essential" = true,
-      "cpu"       = 512,
-      "memory"    = 1024,
+      "cpu" : var.metadata_service_cpu,
+      "memory" : var.metadata_service_memory,
       "portMappings" = [
         {
           "containerPort" = 8080,
@@ -33,7 +33,7 @@ resource "aws_ecs_task_definition" "this" {
       ],
       "environment" = [
         { "name" = "MF_METADATA_DB_HOST", "value" = "${replace(var.rds_master_instance_endpoint, ":5432", "")}" },
-        { "name" = "MF_METADATA_DB_NAME", "value" = "metaflow" },
+        { "name" = "MF_METADATA_DB_NAME", "value" = "${var.database_name}" },
         { "name" = "MF_METADATA_DB_PORT", "value" = "5432" },
         { "name" = "MF_METADATA_DB_USER", "value" = "${var.database_username}" },
       ],
@@ -57,8 +57,8 @@ resource "aws_ecs_task_definition" "this" {
   requires_compatibilities = ["FARGATE"]
   task_role_arn            = aws_iam_role.metadata_svc_ecs_task_role.arn
   execution_role_arn       = var.fargate_execution_role_arn
-  cpu                      = 512
-  memory                   = 1024
+  cpu                      = var.metadata_service_cpu
+  memory                   = var.metadata_service_memory
 
   tags = merge(
     var.standard_tags,
@@ -93,9 +93,13 @@ resource "aws_ecs_service" "this" {
     container_port   = 8082
   }
 
+  enable_ecs_managed_tags = true
+  propagate_tags          = "TASK_DEFINITION"
+
+  tags = var.standard_tags
+
   lifecycle {
     ignore_changes = [desired_count]
   }
 
-  tags = var.standard_tags
 }

modules/metadata-service/variables.tf

@@ -9,6 +9,12 @@ variable "api_basic_auth" {
   description = "Enable basic auth for API Gateway? (requires key export)"
 }
 
+variable "database_name" {
+  type        = string
+  default     = "metaflow"
+  description = "The database name"
+}
+
 variable "ecs_cluster_id" {
   type        = string
   default     = null
@@ -50,10 +56,21 @@ variable "is_gov" {
 
 variable "metadata_service_container_image" {
   type        = string
-  default     = ""
   description = "Container image for metadata service"
 }
 
+variable "metadata_service_cpu" {
+  type        = number
+  default     = 512
+  description = "ECS task CPU unit for metadata service"
+}
+
+variable "metadata_service_memory" {
+  type        = number
+  default     = 1024
+  description = "ECS task memory in MiB for metadata service"
+}
+
 variable "metaflow_vpc_id" {
   type        = string
   description = "ID of the Metaflow VPC this SageMaker notebook instance is to be deployed in"
@@ -94,7 +111,13 @@ variable "subnet2_id" {
   description = "Second private subnet used for availability zone redundancy"
 }
 
-variable "vpc_cidr_block" {
-  type        = string
-  description = "The VPC CIDR block that we'll access list on our Metadata Service API to allow all internal communications"
+variable "vpc_cidr_blocks" {
+  type        = list(string)
+  description = "The VPC CIDR blocks that we'll access list on our Metadata Service API to allow all internal communications"
+}
+
+variable "with_public_ip" {
+  type        = bool
+  default     = false
+  description = "Enable private IP by default"
 }

modules/ui/ecs_ui_backend.tf

@@ -76,9 +76,11 @@ resource "aws_ecs_service" "ui_backend" {
     container_port   = 8083
   }
 
+  enable_ecs_managed_tags = true
+  propagate_tags          = "TASK_DEFINITION"
+
+  tags = var.standard_tags
   lifecycle {
     ignore_changes = [desired_count]
   }
-
-  tags = var.standard_tags
 }

modules/ui/ecs_ui_static.tf

@@ -62,9 +62,12 @@ resource "aws_ecs_service" "ui_static" {
     container_port   = 3000
   }
 
+  enable_ecs_managed_tags = true
+  propagate_tags          = "TASK_DEFINITION"
+
+  tags = var.standard_tags
+
   lifecycle {
     ignore_changes = [desired_count]
   }
-
-  tags = var.standard_tags
 }

variables.tf

@@ -149,9 +149,9 @@ variable "subnet2_id" {
   description = "Second subnet used for availability zone redundancy"
 }
 
-variable "vpc_cidr_block" {
-  type        = string
-  description = "The VPC CIDR block that we'll access list on our Metadata Service API to allow all internal communications"
+variable "vpc_cidr_blocks" {
+  type        = list(string)
+  description = "The VPC CIDR blocks that we'll access list on our Metadata Service API to allow all internal communications"
 }
 
 variable "vpc_id" {