Custom local file location (#81)

* Inline content of db_migrate_lambda file

This removes the need for a .py file to be generated.

* Add variable for the DB migrate lambda path

This allows the path to be set to a specific location and committed or
cached, so that it is not recreated unnecessarily in an automation
context.

* docs

---------

Co-authored-by: Tom Ashworth <[email protected]>

Author: oavdeev

README.md

@@ -110,6 +110,7 @@ resource "local_file" "metaflow_config" {
 | <a name="input_compute_environment_min_vcpus"></a> [compute\_environment\_min\_vcpus](#input\_compute\_environment\_min\_vcpus) | Minimum VCPUs for Batch Compute Environment [0-16] for EC2 Batch Compute Environment (ignored for Fargate) | `number` | `8` | no |
 | <a name="input_db_engine_version"></a> [db\_engine\_version](#input\_db\_engine\_version) | n/a | `string` | `"11"` | no |
 | <a name="input_db_instance_type"></a> [db\_instance\_type](#input\_db\_instance\_type) | RDS instance type to launch for PostgresQL database. | `string` | `"db.t2.small"` | no |
+| <a name="input_db_migrate_lambda_zip_file"></a> [db\_migrate\_lambda\_zip\_file](#input\_db\_migrate\_lambda\_zip\_file) | Output path for the zip file containing the DB migrate lambda | `string` | `null` | no |
 | <a name="input_enable_custom_batch_container_registry"></a> [enable\_custom\_batch\_container\_registry](#input\_enable\_custom\_batch\_container\_registry) | Provisions infrastructure for custom Amazon ECR container registry if enabled | `bool` | `false` | no |
 | <a name="input_enable_step_functions"></a> [enable\_step\_functions](#input\_enable\_step\_functions) | Provisions infrastructure for step functions if enabled | `bool` | n/a | yes |
 | <a name="input_extra_ui_backend_env_vars"></a> [extra\_ui\_backend\_env\_vars](#input\_extra\_ui\_backend\_env\_vars) | Additional environment variables for UI backend container | `map(string)` | `{}` | no |

main.tf

@@ -27,6 +27,7 @@ module "metaflow-metadata-service" {
   database_name                    = module.metaflow-datastore.database_name
   database_password                = module.metaflow-datastore.database_password
   database_username                = module.metaflow-datastore.database_username
+  db_migrate_lambda_zip_file       = var.db_migrate_lambda_zip_file
   datastore_s3_bucket_kms_key_arn  = module.metaflow-datastore.datastore_s3_bucket_kms_key_arn
   enable_api_basic_auth            = var.metadata_service_enable_api_basic_auth
   enable_api_gateway               = var.metadata_service_enable_api_gateway

modules/metadata-service/README.md

@@ -20,6 +20,7 @@ If the `access_list_cidr_blocks` variable is set, only traffic originating from
 | <a name="input_database_password"></a> [database\_password](#input\_database\_password) | The database password | `string` | n/a | yes |
 | <a name="input_database_username"></a> [database\_username](#input\_database\_username) | The database username | `string` | n/a | yes |
 | <a name="input_datastore_s3_bucket_kms_key_arn"></a> [datastore\_s3\_bucket\_kms\_key\_arn](#input\_datastore\_s3\_bucket\_kms\_key\_arn) | The ARN of the KMS key used to encrypt the Metaflow datastore S3 bucket | `string` | n/a | yes |
+| <a name="input_db_migrate_lambda_zip_file"></a> [db\_migrate\_lambda\_zip\_file](#input\_db\_migrate\_lambda\_zip\_file) | Output path for the zip file containing the DB migrate lambda | `string` | `null` | no |
 | <a name="input_enable_api_basic_auth"></a> [enable\_api\_basic\_auth](#input\_enable\_api\_basic\_auth) | Enable basic auth for API Gateway? (requires key export) | `bool` | `true` | no |
 | <a name="input_enable_api_gateway"></a> [enable\_api\_gateway](#input\_enable\_api\_gateway) | Enable API Gateway for public metadata service endpoint | `bool` | `true` | no |
 | <a name="input_fargate_execution_role_arn"></a> [fargate\_execution\_role\_arn](#input\_fargate\_execution\_role\_arn) | The IAM role that grants access to ECS and Batch services which we'll use as our Metadata Service API's execution\_role for our Fargate instance | `string` | n/a | yes |

modules/metadata-service/lambda.tf

@@ -78,8 +78,13 @@ resource "aws_iam_role_policy" "grant_lambda_ecs_vpc" {
   policy = data.aws_iam_policy_document.lambda_ecs_task_execute_policy_vpc.json
 }
 
-resource "local_file" "db_migrate_lambda" {
-  content  = <<EOF
+data "archive_file" "db_migrate_lambda" {
+  type             = "zip"
+  output_file_mode = "0666"
+  output_path      = local.db_migrate_lambda_zip_file
+
+  source {
+    content  = <<EOF
 import os, json
 from urllib import request
 
@@ -101,15 +106,8 @@ def handler(event, context):
   print(response)
   return(response)
 EOF
-  filename = local.db_migrate_lambda_source_file
-}
-
-data "archive_file" "db_migrate_lambda" {
-  type             = "zip"
-  source_file      = local.db_migrate_lambda_source_file
-  output_file_mode = "0666"
-  output_path      = local.db_migrate_lambda_zip_file
-  depends_on       = [local_file.db_migrate_lambda]
+    filename = "index.py"
+  }
 }
 
 resource "aws_lambda_function" "db_migrate_lambda" {

modules/metadata-service/locals.tf

@@ -22,10 +22,9 @@ locals {
   api_gateway_stage_name                  = "api"
   api_gateway_usage_plan_name             = "${var.resource_prefix}usage-plan${var.resource_suffix}"
 
-  db_migrate_lambda_source_file = "${path.module}/index.py"
-  db_migrate_lambda_zip_file    = "${path.module}/db_migrate_lambda.zip"
-  db_migrate_lambda_name        = "${var.resource_prefix}db_migrate${var.resource_suffix}"
-  lambda_ecs_execute_role_name  = "${var.resource_prefix}lambda_ecs_execute${var.resource_suffix}"
+  db_migrate_lambda_zip_file   = coalesce(var.db_migrate_lambda_zip_file, "${path.module}/db_migrate_lambda.zip")
+  db_migrate_lambda_name       = "${var.resource_prefix}db_migrate${var.resource_suffix}"
+  lambda_ecs_execute_role_name = "${var.resource_prefix}lambda_ecs_execute${var.resource_suffix}"
 
   cloudwatch_logs_arn_prefix = "arn:${var.iam_partition}:logs:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}"
 }

modules/metadata-service/variables.tf

@@ -37,6 +37,12 @@ variable "enable_api_gateway" {
   description = "Enable API Gateway for public metadata service endpoint"
 }
 
+variable "db_migrate_lambda_zip_file" {
+  type        = string
+  description = "Output path for the zip file containing the DB migrate lambda"
+  default     = null
+}
+
 variable "fargate_execution_role_arn" {
   type        = string
   description = "The IAM role that grants access to ECS and Batch services which we'll use as our Metadata Service API's execution_role for our Fargate instance"

variables.tf

@@ -10,6 +10,12 @@ variable "batch_type" {
   default     = "ec2"
 }
 
+variable "db_migrate_lambda_zip_file" {
+  type        = string
+  description = "Output path for the zip file containing the DB migrate lambda"
+  default     = null
+}
+
 variable "enable_custom_batch_container_registry" {
   type        = bool
   default     = false