Skip to content

Commit 5833e84

Browse files
LimessLimess
committed
Move DB password secrets into secret manager to prevent leaks in task definitions
1 parent 5117b57 commit 5833e84

17 files changed

+159
-91
lines changed

‎examples/eks/metaflow.tf

+13-13
Original file line numberDiff line numberDiff line change
@@ -47,19 +47,19 @@ module "metaflow-metadata-service" {
4747
resource_prefix = local.resource_prefix
4848
resource_suffix = local.resource_suffix
4949

50-
access_list_cidr_blocks = []
51-
api_basic_auth = true
52-
database_password = module.metaflow-datastore.database_password
53-
database_username = module.metaflow-datastore.database_username
54-
datastore_s3_bucket_kms_key_arn = module.metaflow-datastore.datastore_s3_bucket_kms_key_arn
55-
fargate_execution_role_arn = aws_iam_role.ecs_execution_role.arn
56-
metaflow_vpc_id = module.vpc.vpc_id
57-
metadata_service_container_image = module.metaflow-common.default_metadata_service_container_image
58-
rds_master_instance_endpoint = module.metaflow-datastore.rds_master_instance_endpoint
59-
s3_bucket_arn = module.metaflow-datastore.s3_bucket_arn
60-
subnet1_id = module.vpc.private_subnets[0]
61-
subnet2_id = module.vpc.private_subnets[1]
62-
vpc_cidr_block = module.vpc.vpc_cidr_block
50+
access_list_cidr_blocks = []
51+
api_basic_auth = true
52+
database_password_secret_manager_arn = module.metaflow-datastore.database_password_secret_manager_arn
53+
database_username = module.metaflow-datastore.database_username
54+
datastore_s3_bucket_kms_key_arn = module.metaflow-datastore.datastore_s3_bucket_kms_key_arn
55+
fargate_execution_role_arn = aws_iam_role.ecs_execution_role.arn
56+
metaflow_vpc_id = module.vpc.vpc_id
57+
metadata_service_container_image = module.metaflow-common.default_metadata_service_container_image
58+
rds_master_instance_endpoint = module.metaflow-datastore.rds_master_instance_endpoint
59+
s3_bucket_arn = module.metaflow-datastore.s3_bucket_arn
60+
subnet1_id = module.vpc.private_subnets[0]
61+
subnet2_id = module.vpc.private_subnets[1]
62+
vpc_cidr_block = module.vpc.vpc_cidr_block
6363

6464
standard_tags = local.tags
6565
}

‎main.tf

+32-30
Original file line numberDiff line numberDiff line change
@@ -20,21 +20,21 @@ module "metaflow-metadata-service" {
2020
resource_prefix = local.resource_prefix
2121
resource_suffix = local.resource_suffix
2222

23-
access_list_cidr_blocks = var.access_list_cidr_blocks
24-
api_basic_auth = var.api_basic_auth
25-
database_password = module.metaflow-datastore.database_password
26-
database_username = module.metaflow-datastore.database_username
27-
datastore_s3_bucket_kms_key_arn = module.metaflow-datastore.datastore_s3_bucket_kms_key_arn
28-
ecs_cluster_id = var.ecs_cluster_id
29-
fargate_execution_role_arn = module.metaflow-computation.ecs_execution_role_arn
30-
iam_partition = var.iam_partition
31-
metadata_service_container_image = local.metadata_service_container_image
32-
metaflow_vpc_id = var.vpc_id
33-
rds_master_instance_endpoint = module.metaflow-datastore.rds_master_instance_endpoint
34-
s3_bucket_arn = module.metaflow-datastore.s3_bucket_arn
35-
subnet1_id = var.subnet1_id
36-
subnet2_id = var.subnet2_id
37-
vpc_cidr_block = var.vpc_cidr_block
23+
access_list_cidr_blocks = var.access_list_cidr_blocks
24+
api_basic_auth = var.api_basic_auth
25+
database_password_secret_manager_arn = module.metaflow-datastore.database_password_secret_manager_arn
26+
database_username = module.metaflow-datastore.database_username
27+
datastore_s3_bucket_kms_key_arn = module.metaflow-datastore.datastore_s3_bucket_kms_key_arn
28+
ecs_cluster_id = var.ecs_cluster_id
29+
fargate_execution_role_arn = module.metaflow-computation.ecs_execution_role_arn
30+
iam_partition = var.iam_partition
31+
metadata_service_container_image = local.metadata_service_container_image
32+
metaflow_vpc_id = var.vpc_id
33+
rds_master_instance_endpoint = module.metaflow-datastore.rds_master_instance_endpoint
34+
s3_bucket_arn = module.metaflow-datastore.s3_bucket_arn
35+
subnet1_id = var.subnet1_id
36+
subnet2_id = var.subnet2_id
37+
vpc_cidr_block = var.vpc_cidr_block
3838

3939
standard_tags = var.tags
4040
}
@@ -47,21 +47,21 @@ module "metaflow-ui" {
4747
resource_prefix = local.resource_prefix
4848
resource_suffix = local.resource_suffix
4949

50-
database_password = module.metaflow-datastore.database_password
51-
database_username = module.metaflow-datastore.database_username
52-
datastore_s3_bucket_kms_key_arn = module.metaflow-datastore.datastore_s3_bucket_kms_key_arn
53-
ecs_cluster_id = var.ecs_cluster_id
54-
fargate_execution_role_arn = module.metaflow-computation.ecs_execution_role_arn
55-
iam_partition = var.iam_partition
56-
metaflow_vpc_id = var.vpc_id
57-
rds_master_instance_endpoint = module.metaflow-datastore.rds_master_instance_endpoint
58-
s3_bucket_arn = module.metaflow-datastore.s3_bucket_arn
59-
subnet1_id = var.subnet1_id
60-
subnet2_id = var.subnet2_id
61-
ui_backend_container_image = local.metadata_service_container_image
62-
ui_static_container_image = local.ui_static_container_image
63-
alb_internal = var.ui_alb_internal
64-
ui_allow_list = var.ui_allow_list
50+
database_password_secret_manager_arn = module.metaflow-datastore.database_password_secret_manager_arn
51+
database_username = module.metaflow-datastore.database_username
52+
datastore_s3_bucket_kms_key_arn = module.metaflow-datastore.datastore_s3_bucket_kms_key_arn
53+
ecs_cluster_id = var.ecs_cluster_id
54+
fargate_execution_role_arn = module.metaflow-computation.ecs_execution_role_arn
55+
iam_partition = var.iam_partition
56+
metaflow_vpc_id = var.vpc_id
57+
rds_master_instance_endpoint = module.metaflow-datastore.rds_master_instance_endpoint
58+
s3_bucket_arn = module.metaflow-datastore.s3_bucket_arn
59+
subnet1_id = var.subnet1_id
60+
subnet2_id = var.subnet2_id
61+
ui_backend_container_image = local.metadata_service_container_image
62+
ui_static_container_image = local.ui_static_container_image
63+
alb_internal = var.ui_alb_internal
64+
ui_allow_list = var.ui_allow_list
6565

6666
METAFLOW_DATASTORE_SYSROOT_S3 = module.metaflow-datastore.METAFLOW_DATASTORE_SYSROOT_S3
6767
certificate_arn = var.ui_certificate_arn
@@ -78,6 +78,8 @@ module "metaflow-computation" {
7878
resource_prefix = local.resource_prefix
7979
resource_suffix = local.resource_suffix
8080

81+
database_password_secret_manager_arn = module.metaflow-datastore.database_password_secret_manager_arn
82+
8183
batch_type = var.batch_type
8284
compute_environment_ami_id = var.compute_environment_ami_id
8385
compute_environment_spot_bid_percentage = var.compute_environment_spot_bid_percentage

‎modules/computation/README.md

+1
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ To read more, see [the Metaflow docs](https://docs.metaflow.org/metaflow-on-aws/
2121
| <a name="input_compute_environment_min_vcpus"></a> [compute\_environment\_min\_vcpus](#input\_compute\_environment\_min\_vcpus) | Minimum VCPUs for Batch Compute Environment [0-16] for EC2 Batch Compute Environment (ignored for Fargate) | `number` | n/a | yes |
2222
| <a name="input_compute_environment_spot_bid_percentage"></a> [compute\_environment\_spot\_bid\_percentage](#input\_compute\_environment\_spot\_bid\_percentage) | The maximum percentage of on-demand EC2 instance price to bid for spot instances when using the 'spot' AWS Batch Compute Type. | `number` | `100` | no |
2323
| <a name="input_compute_environment_user_data_base64"></a> [compute\_environment\_user\_data\_base64](#input\_compute\_environment\_user\_data\_base64) | Base64 hash of the user data to use for Batch Compute Environment EC2 instances. | `string` | `null` | no |
24+
| <a name="input_database_password_secret_manager_arn"></a> [database\_password\_secret\_manager\_arn](#input\_database\_password\_secret\_manager\_arn) | The arn of the database password stored in AWS secrets manager | `string` | n/a | yes |
2425
| <a name="input_ecs_cluster_id"></a> [ecs\_cluster\_id](#input\_ecs\_cluster\_id) | The ID of an existing ECS cluster to run services on. If no cluster ID is specfied, a new cluster will be created. | `string` | `null` | no |
2526
| <a name="input_iam_partition"></a> [iam\_partition](#input\_iam\_partition) | IAM Partition (Select aws-us-gov for AWS GovCloud, otherwise leave as is) | `string` | `"aws"` | no |
2627
| <a name="input_metaflow_vpc_id"></a> [metaflow\_vpc\_id](#input\_metaflow\_vpc\_id) | ID of the Metaflow VPC this SageMaker notebook instance is to be deployed in | `string` | n/a | yes |

‎modules/computation/iam-ecs-execution.tf

+41
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,41 @@ resource "aws_iam_role" "ecs_execution_role" {
2727
tags = var.standard_tags
2828
}
2929

30+
31+
data "aws_iam_policy_document" "db_secrets" {
32+
statement {
33+
sid = "AllowDBSecretAccess"
34+
35+
effect = "Allow"
36+
37+
actions = [
38+
"secretsmanager:GetResourcePolicy",
39+
"secretsmanager:GetSecretValue",
40+
"secretsmanager:DescribeSecret",
41+
"secretsmanager:ListSecretVersionIds"
42+
]
43+
44+
resources = [
45+
var.database_password_secret_manager_arn
46+
]
47+
}
48+
49+
statement {
50+
sid = "AllowSecretsManagerList"
51+
52+
effect = "Allow"
53+
54+
actions = [
55+
"secretsmanager:ListSecrets",
56+
]
57+
58+
resources = [
59+
"*"
60+
]
61+
}
62+
}
63+
64+
3065
data "aws_iam_policy_document" "ecs_task_execution_policy" {
3166
statement {
3267
effect = "Allow"
@@ -53,3 +88,9 @@ resource "aws_iam_role_policy" "grant_ecs_access" {
5388
role = aws_iam_role.ecs_execution_role.name
5489
policy = data.aws_iam_policy_document.ecs_task_execution_policy.json
5590
}
91+
92+
resource "aws_iam_role_policy" "grant_db_secrets" {
93+
name = "dbt_secrets"
94+
role = aws_iam_role.ecs_execution_role.name
95+
policy = data.aws_iam_policy_document.db_secrets.json
96+
}

‎modules/computation/variables.tf

+6
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,12 @@ variable "batch_type" {
99
}
1010
}
1111

12+
variable "database_password_secret_manager_arn" {
13+
type = string
14+
description = "The arn of the database password stored in AWS secrets manager"
15+
sensitive = true
16+
}
17+
1218
variable "compute_environment_ami_id" {
1319
type = string
1420
description = "The AMI ID to use for Batch Compute Environment EC2 instances. If not specified, defaults to the latest ECS optimised AMI."

‎modules/datastore/README.md

+1-1
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@ To read more, see [the Metaflow docs](https://docs.metaflow.org/metaflow-on-aws/
3838
|------|-------------|
3939
| <a name="output_METAFLOW_DATASTORE_SYSROOT_S3"></a> [METAFLOW\_DATASTORE\_SYSROOT\_S3](#output\_METAFLOW\_DATASTORE\_SYSROOT\_S3) | Amazon S3 URL for Metaflow DataStore |
4040
| <a name="output_METAFLOW_DATATOOLS_S3ROOT"></a> [METAFLOW\_DATATOOLS\_S3ROOT](#output\_METAFLOW\_DATATOOLS\_S3ROOT) | Amazon S3 URL for Metaflow DataTools |
41-
| <a name="output_database_password"></a> [database\_password](#output\_database\_password) | The database password |
41+
| <a name="output_database_password_secret_manager_arn"></a> [database\_password\_secret\_manager\_arn](#output\_database\_password\_secret\_manager\_arn) | The arn of the database password stored in AWS secrets manager |
4242
| <a name="output_database_username"></a> [database\_username](#output\_database\_username) | The database username |
4343
| <a name="output_datastore_s3_bucket_kms_key_arn"></a> [datastore\_s3\_bucket\_kms\_key\_arn](#output\_datastore\_s3\_bucket\_kms\_key\_arn) | The ARN of the KMS key used to encrypt the Metaflow datastore S3 bucket |
4444
| <a name="output_rds_master_instance_endpoint"></a> [rds\_master\_instance\_endpoint](#output\_rds\_master\_instance\_endpoint) | The database connection endpoint in address:port format |

‎modules/datastore/outputs.tf

+3-3
Original file line numberDiff line numberDiff line change
@@ -8,9 +8,9 @@ output "METAFLOW_DATASTORE_SYSROOT_S3" {
88
description = "Amazon S3 URL for Metaflow DataStore"
99
}
1010

11-
output "database_password" {
12-
value = random_password.this.result
13-
description = "The database password"
11+
output "database_password_secret_manager_arn" {
12+
value = aws_secretsmanager_secretrds_db_password.arn
13+
description = "The arn of the database password stored in AWS secrets manager"
1414
}
1515

1616
output "database_username" {

‎modules/datastore/rds.tf

+9
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,15 @@ resource "random_password" "this" {
5151
override_special = "!#$%&*()-_=+[]{}<>:?"
5252
}
5353

54+
resource "aws_secretsmanager_secret" "rds_db_password" {
55+
name = "${var.resource_prefix}${var.db_name}_password${var.resource_suffix}"
56+
}
57+
58+
resource "aws_secretsmanager_secret_version" "example" {
59+
secret_id = aws_secretsmanager_secret.rds_db_password.id
60+
secret_string = random_password.this.result
61+
}
62+
5463
resource "random_pet" "final_snapshot_id" {}
5564

5665
/*

‎modules/metadata-service/README.md

+1-1
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ If the `access_list_cidr_blocks` variable is set, only traffic originating from
1717
|------|-------------|------|---------|:--------:|
1818
| <a name="input_access_list_cidr_blocks"></a> [access\_list\_cidr\_blocks](#input\_access\_list\_cidr\_blocks) | List of CIDRs we want to grant access to our Metaflow Metadata Service. Usually this is our VPN's CIDR blocks. | `list(string)` | n/a | yes |
1919
| <a name="input_api_basic_auth"></a> [api\_basic\_auth](#input\_api\_basic\_auth) | Enable basic auth for API Gateway? (requires key export) | `bool` | `true` | no |
20-
| <a name="input_database_password"></a> [database\_password](#input\_database\_password) | The database password | `string` | n/a | yes |
20+
| <a name="input_database_password_secret_manager_arn"></a> [database\_password\_secret\_manager\_arn](#input\_database\_password\_secret\_manager\_arn) | The arn of the database password stored in AWS secrets manager | `string` | n/a | yes |
2121
| <a name="input_database_username"></a> [database\_username](#input\_database\_username) | The database username | `string` | n/a | yes |
2222
| <a name="input_datastore_s3_bucket_kms_key_arn"></a> [datastore\_s3\_bucket\_kms\_key\_arn](#input\_datastore\_s3\_bucket\_kms\_key\_arn) | The ARN of the KMS key used to encrypt the Metaflow datastore S3 bucket | `string` | n/a | yes |
2323
| <a name="input_ecs_cluster_id"></a> [ecs\_cluster\_id](#input\_ecs\_cluster\_id) | The ID of an existing ECS cluster to run services on. If no cluster ID is specfied, a new cluster will be created. | `string` | `null` | no |

‎modules/metadata-service/ecs.tf

+37-35
Original file line numberDiff line numberDiff line change
@@ -14,43 +14,45 @@ resource "aws_ecs_cluster" "this" {
1414
resource "aws_ecs_task_definition" "this" {
1515
family = "${var.resource_prefix}service${var.resource_suffix}" # Unique name for task definition
1616

17-
container_definitions = <<EOF
18-
[
19-
{
20-
"name": "${var.resource_prefix}service${var.resource_suffix}",
21-
"image": "${var.metadata_service_container_image}",
22-
"essential": true,
23-
"cpu": 512,
24-
"memory": 1024,
25-
"portMappings": [
26-
{
27-
"containerPort": 8080,
28-
"hostPort": 8080
29-
},
30-
{
31-
"containerPort": 8082,
32-
"hostPort": 8082
33-
}
34-
],
35-
"environment": [
36-
{"name": "MF_METADATA_DB_HOST", "value": "${replace(var.rds_master_instance_endpoint, ":5432", "")}"},
37-
{"name": "MF_METADATA_DB_NAME", "value": "metaflow"},
38-
{"name": "MF_METADATA_DB_PORT", "value": "5432"},
39-
{"name": "MF_METADATA_DB_PSWD", "value": "${var.database_password}"},
40-
{"name": "MF_METADATA_DB_USER", "value": "${var.database_username}"}
41-
],
42-
"logConfiguration": {
43-
"logDriver": "awslogs",
44-
"options": {
45-
"awslogs-group": "${aws_cloudwatch_log_group.this.name}",
46-
"awslogs-region": "${data.aws_region.current.name}",
47-
"awslogs-stream-prefix": "metadata"
17+
container_definitions = jsonencode([
18+
{
19+
"name" = "${var.resource_prefix}service${var.resource_suffix}",
20+
"image" = "${var.metadata_service_container_image}",
21+
"essential" = true,
22+
"cpu" = 512,
23+
"memory" = 1024,
24+
"portMappings" = [
25+
{
26+
"containerPort" = 8080,
27+
"hostPort" = 8080
28+
},
29+
{
30+
"containerPort" = 8082,
31+
"hostPort" = 8082
32+
}
33+
],
34+
"environment" = [
35+
{ "name" = "MF_METADATA_DB_HOST", "value" = "${replace(var.rds_master_instance_endpoint, ":5432", "")}" },
36+
{ "name" = "MF_METADATA_DB_NAME", "value" = "metaflow" },
37+
{ "name" = "MF_METADATA_DB_PORT", "value" = "5432" },
38+
{ "name" = "MF_METADATA_DB_USER", "value" = "${var.database_username}" },
39+
],
40+
"secrets" = [
41+
{
42+
"name" = "MF_METADATA_DB_PSWD",
43+
"valueFrom" = var.database_password_secret_manager_arn,
44+
}
45+
],
46+
"logConfiguration" = {
47+
"logDriver" = "awslogs",
48+
"options" = {
49+
"awslogs-group" = "${aws_cloudwatch_log_group.this.name}",
50+
"awslogs-region" = "${data.aws_region.current.name}",
51+
"awslogs-stream-prefix" = "metadata"
4852
}
53+
}
4954
}
50-
}
51-
]
52-
EOF
53-
55+
])
5456
network_mode = "awsvpc"
5557
requires_compatibilities = ["FARGATE"]
5658
task_role_arn = aws_iam_role.metadata_svc_ecs_task_role.arn

‎modules/metadata-service/variables.tf

+3-2
Original file line numberDiff line numberDiff line change
@@ -15,9 +15,10 @@ variable "ecs_cluster_id" {
1515
description = "The ID of an existing ECS cluster to run services on. If no cluster ID is specfied, a new cluster will be created."
1616
}
1717

18-
variable "database_password" {
18+
variable "database_password_secret_manager_arn" {
1919
type = string
20-
description = "The database password"
20+
description = "The arn of the database password stored in AWS secrets manager"
21+
sensitive = true
2122
}
2223

2324
variable "database_username" {

‎modules/step-functions/README.md

+1-1
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ To read more, see [the Metaflow docs](https://docs.metaflow.org/going-to-product
1212
| Name | Description | Type | Default | Required |
1313
|------|-------------|------|---------|:--------:|
1414
| <a name="input_active"></a> [active](#input\_active) | When true step function infrastructure is provisioned. | `bool` | `false` | no |
15-
| <a name="input_batch_job_queue_arns"></a> [batch\_job\_queue\_arns](#input\_batch\_job\_queue\_arns) | Batch job queue arns | `list[string]` | n/a | yes |
15+
| <a name="input_batch_job_queue_arns"></a> [batch\_job\_queue\_arns](#input\_batch\_job\_queue\_arns) | Batch job queue arns | `list(string)` | n/a | yes |
1616
| <a name="input_iam_partition"></a> [iam\_partition](#input\_iam\_partition) | IAM Partition (Select aws-us-gov for AWS GovCloud, otherwise leave as is) | `string` | `"aws"` | no |
1717
| <a name="input_resource_prefix"></a> [resource\_prefix](#input\_resource\_prefix) | Prefix given to all AWS resources to differentiate between applications | `string` | n/a | yes |
1818
| <a name="input_resource_suffix"></a> [resource\_suffix](#input\_resource\_suffix) | Suffix given to all AWS resources to differentiate between environment and workspace | `string` | n/a | yes |

‎modules/step-functions/variables.tf

+1-1
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ variable "active" {
55
}
66

77
variable "batch_job_queue_arns" {
8-
type = list[string]
8+
type = list(string)
99
description = "Batch job queue arns"
1010
}
1111

0 commit comments

Comments
 (0)