Experiment: multi-environment configuration in Python

app/src/__main__.py

@@ -6,8 +6,10 @@
 # https://docs.python.org/3/library/__main__.html
 
 import logging
+import os
 
 import src.app
+import src.config
 import src.logging
 from src.app_config import AppConfig
 from src.util.local import load_local_env_vars
@@ -16,23 +18,26 @@
 
 
 def main() -> None:
-    load_local_env_vars()
-    app_config = AppConfig()
+    config = src.config.load(environment_name=os.getenv("ENVIRONMENT", "local"), environ=os.environ)
 
-    app = src.app.create_app()
+    app = src.app.create_app(config)
+    logger.info("loaded configuration", extra={"config": config})
 
-    environment = app_config.environment
+    all_configs = src.config.load_all()
+    logger.info("loaded all", extra={"all_configs": all_configs})
+
+    environment = config.app.environment
 
     # When running in a container, the host needs to be set to 0.0.0.0 so that the app can be
     # accessed from outside the container. See Dockerfile
-    host = app_config.host
-    port = app_config.port
+    host = config.app.host
+    port = config.app.port
 
     logger.info(
         "Running API Application", extra={"environment": environment, "host": host, "port": port}
     )
 
-    if app_config.environment == "local":
+    if config.app.environment == "local":
         # If python files are changed, the app will auto-reload
         # Note this doesn't have the OpenAPI yaml file configured at the moment
         app.run(host=host, port=port, use_reloader=True, reloader_type="stat")

app/src/adapters/db/clients/postgres_client.py

@@ -8,7 +8,7 @@
 import sqlalchemy.pool as pool
 
 from src.adapters.db.client import DBClient
-from src.adapters.db.clients.postgres_config import PostgresDBConfig, get_db_config
+from src.adapters.db.clients.postgres_config import PostgresDBConfig
 
 logger = logging.getLogger(__name__)
 
@@ -19,9 +19,7 @@ class PostgresDBClient(DBClient):
     as configured by parameters passed in from the db_config
     """
 
-    def __init__(self, db_config: PostgresDBConfig | None = None) -> None:
-        if not db_config:
-            db_config = get_db_config()
+    def __init__(self, db_config: PostgresDBConfig) -> None:
         self._engine = self._configure_engine(db_config)
 
         if db_config.check_connection_on_init:
@@ -82,17 +80,17 @@ def check_db_connection(self) -> None:
 def get_connection_parameters(db_config: PostgresDBConfig) -> dict[str, Any]:
     connect_args = {}
     environment = os.getenv("ENVIRONMENT")
-    if not environment:
-        raise Exception("ENVIRONMENT is not set")
-
-    if environment != "local":
-        connect_args["sslmode"] = "require"
+    # if not environment:
+    #     raise Exception("ENVIRONMENT is not set")
+    #
+    # if environment != "local":
+    #     connect_args["sslmode"] = "require"
 
     return dict(
         host=db_config.host,
         dbname=db_config.name,
         user=db_config.username,
-        password=db_config.password,
+        password=db_config.password.get_secret_value(),
         port=db_config.port,
         options=f"-c search_path={db_config.db_schema}",
         connect_timeout=3,

app/src/adapters/db/clients/postgres_config.py

@@ -1,6 +1,7 @@
 import logging
 from typing import Optional
 
+import pydantic
 from pydantic import Field
 
 from src.util.env_config import PydanticBaseEnvConfig
@@ -13,26 +14,7 @@ class PostgresDBConfig(PydanticBaseEnvConfig):
     host: str = Field("localhost", env="DB_HOST")
     name: str = Field("main-db", env="POSTGRES_DB")
     username: str = Field("local_db_user", env="POSTGRES_USER")
-    password: Optional[str] = Field(..., env="POSTGRES_PASSWORD")
+    password: Optional[pydantic.types.SecretStr] = Field(None, env="POSTGRES_PASSWORD")
     db_schema: str = Field("public", env="DB_SCHEMA")
-    port: str = Field("5432", env="DB_PORT")
+    port: int = Field(5432, env="DB_PORT")
     hide_sql_parameter_logs: bool = Field(True, env="HIDE_SQL_PARAMETER_LOGS")
-
-
-def get_db_config() -> PostgresDBConfig:
-    db_config = PostgresDBConfig()
-
-    logger.info(
-        "Constructed database configuration",
-        extra={
-            "host": db_config.host,
-            "dbname": db_config.name,
-            "username": db_config.username,
-            "password": "***" if db_config.password is not None else None,
-            "db_schema": db_config.db_schema,
-            "port": db_config.port,
-            "hide_sql_parameter_logs": db_config.hide_sql_parameter_logs,
-        },
-    )
-
-    return db_config

app/src/app.py

@@ -18,13 +18,13 @@
 logger = logging.getLogger(__name__)
 
 
-def create_app() -> APIFlask:
+def create_app(config) -> APIFlask:
     app = APIFlask(__name__)
 
-    src.logging.init(__package__)
+    src.logging.init(__package__, config.logging)
     flask_logger.init_app(logging.root, app)
 
-    db_client = db.PostgresDBClient()
+    db_client = db.PostgresDBClient(config.database)
     flask_db.register_db_client(db_client, app)
 
     configure_app(app)

app/src/app_config.py

@@ -2,7 +2,7 @@
 
 
 class AppConfig(PydanticBaseEnvConfig):
-    environment: str
+    environment: str = "unknown"
 
     # Set HOST to 127.0.0.1 by default to avoid other machines on the network
     # from accessing the application. This is especially important if you are

app/src/config/__init__.py

@@ -0,0 +1,39 @@
+#
+# Multi-environment configuration expressed in Python.
+#
+
+import importlib
+import logging
+import pathlib
+
+from src.adapters.db.clients.postgres_config import PostgresDBConfig
+from src.app_config import AppConfig
+from src.logging.config import LoggingConfig
+from src.util.env_config import PydanticBaseEnvConfig
+
+logger = logging.getLogger(__name__)
+
+
+class RootConfig(PydanticBaseEnvConfig):
+    app: AppConfig
+    database: PostgresDBConfig
+    logging: LoggingConfig
+
+
+def load(environment_name, environ=None) -> RootConfig:
+    """Load the configuration for the given environment name."""
+    logger.info("loading configuration", extra={"environment": environment_name})
+    module = importlib.import_module(name=".env." + environment_name, package=__package__)
+    config = module.config
+
+    if environ:
+        config.override_from_environment(environ)
+    config.app.environment = environment_name
+
+    return config
+
+
+def load_all() -> dict[str, RootConfig]:
+    """Load all environment configurations, to ensure they are valid. Used in tests."""
+    directory = pathlib.Path(__file__).parent / "env"
+    return {item.stem: load(str(item.stem)) for item in directory.glob("*.py")}

app/src/config/default.py

@@ -0,0 +1,26 @@
+#
+# Default configuration.
+#
+# This is the base layer of configuration. It is used if an environment does not override a value.
+# Each environment may override individual values (see local.py, dev.py, prod.py, etc.).
+#
+# This configuration is also used when running tests (individual test cases may have code to use
+# different configuration).
+#
+
+from src.adapters.db.clients.postgres_config import PostgresDBConfig
+from src.app_config import AppConfig
+from src.config import RootConfig
+from src.logging.config import LoggingConfig
+
+
+def default_config():
+    return RootConfig(
+        app=AppConfig(),
+        database=PostgresDBConfig(),
+        logging=LoggingConfig(
+            format="json",
+            level="INFO",
+            enable_audit=True,
+        ),
+    )

app/src/config/env/dev.py

@@ -0,0 +1,9 @@
+#
+# Configuration for dev environments.
+#
+# This file only contains overrides (differences) from the defaults in default.py.
+#
+
+from .. import default
+
+config = default.default_config()

app/src/config/env/local.py

@@ -0,0 +1,16 @@
+#
+# Configuration for local development environments.
+#
+# This file only contains overrides (differences) from the defaults in default.py.
+#
+
+import pydantic.types
+
+from .. import default
+
+config = default.default_config()
+
+config.database.password = pydantic.types.SecretStr("secret123")
+config.database.hide_sql_parameter_logs = False
+config.logging.format = "human_readable"
+config.logging.enable_audit = False

app/src/config/env/prod.py

@@ -0,0 +1,9 @@
+#
+# Configuration for prod environment.
+#
+# This file only contains overrides (differences) from the defaults in default.py.
+#
+
+from .. import default
+
+config = default.default_config()

app/src/config/env/staging.py

@@ -0,0 +1,9 @@
+#
+# Configuration for staging environment.
+#
+# This file only contains overrides (differences) from the defaults in default.py.
+#
+
+from .. import default
+
+config = default.default_config()

app/src/logging/__init__.py

@@ -28,5 +28,5 @@
 import src.logging.config as config
 
 
-def init(program_name: str) -> config.LoggingContext:
-    return config.LoggingContext(program_name)
+def init(program_name: str, logging_config: config.LoggingConfig) -> config.LoggingContext:
+    return config.LoggingContext(program_name, logging_config)

app/src/logging/config.py

@@ -1,10 +1,13 @@
+import enum
 import logging
 import os
 import platform
 import pwd
 import sys
 from typing import Any, ContextManager, cast
 
+import pydantic
+
 import src.logging.audit
 import src.logging.formatters as formatters
 import src.logging.pii as pii
@@ -15,15 +18,27 @@
 _original_argv = tuple(sys.argv)
 
 
+class LoggingFormat(str, enum.Enum):
+    json = "json"
+    human_readable = "human_readable"
+
+
 class HumanReadableFormatterConfig(PydanticBaseEnvConfig):
     message_width: int = formatters.HUMAN_READABLE_FORMATTER_DEFAULT_MESSAGE_WIDTH
 
 
 class LoggingConfig(PydanticBaseEnvConfig):
-    format = "json"
-    level = "INFO"
-    enable_audit = True
-    human_readable_formatter = HumanReadableFormatterConfig()
+    format: LoggingFormat = LoggingFormat.json
+    level: str = "INFO"
+    enable_audit: bool = True
+    human_readable_formatter: HumanReadableFormatterConfig = HumanReadableFormatterConfig()
+
+    @pydantic.validator("level")
+    def valid_level(cls, v):
+        value = logging.getLevelName(v)
+        if not isinstance(value, int):
+            raise ValueError("invalid logging level %s" % v)
+        return v
 
     class Config:
         env_prefix = "log_"
@@ -58,8 +73,8 @@ class LoggingContext(ContextManager[None]):
     and calling this multiple times before exit would result in duplicate logs.
     """
 
-    def __init__(self, program_name: str) -> None:
-        self._configure_logging()
+    def __init__(self, program_name: str, config: LoggingConfig) -> None:
+        self._configure_logging(config)
         log_program_info(program_name)
 
     def __enter__(self) -> None:
@@ -72,14 +87,14 @@ def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None:
         # of those tests.
         logging.root.removeHandler(self.console_handler)
 
-    def _configure_logging(self) -> None:
+    def _configure_logging(self, config: LoggingConfig) -> None:
         """Configure logging for the application.
 
         Configures the root module logger to log to stdout.
         Adds a PII mask filter to the root logger.
         Also configures log levels third party packages.
         """
-        config = LoggingConfig()
+        # config = LoggingConfig()  # TODO inject
 
         # Loggers can be configured using config functions defined
         # in logging.config or by directly making calls to the main API
@@ -112,7 +127,7 @@ def get_formatter(config: LoggingConfig) -> logging.Formatter:
     The formatter is determined by the environment variable LOG_FORMAT. If the
     environment variable is not set, the JSON formatter is used by default.
     """
-    if config.format == "human-readable":
+    if config.format == LoggingFormat.human_readable:
         return get_human_readable_formatter(config.human_readable_formatter)
     return formatters.JsonFormatter()
 

app/src/util/env_config.py

@@ -1,16 +1,30 @@
-import os
+import logging
 
-from pydantic import BaseSettings
+from pydantic import BaseModel
 
-import src
+logger = logging.getLogger(__name__)
 
-env_file = os.path.join(
-    os.path.dirname(os.path.dirname(src.__file__)),
-    "config",
-    "%s.env" % os.getenv("ENVIRONMENT", "local"),
-)
 
+class PydanticBaseEnvConfig(BaseModel):
+    """Base class for application configuration.
+
+    Similar to Pydantic's BaseSettings class, but we implement our own method to override from the
+    environment so that it can be run later, after an instance was constructed."""
 
-class PydanticBaseEnvConfig(BaseSettings):
     class Config:
-        env_file = env_file
+        validate_assignment = True
+
+    def override_from_environment(self, environ, prefix=""):
+        """Recursively override field values from the given environment variable mapping."""
+        for name, field in self.__fields__.items():
+            if field.is_complex():
+                # Nested models must be instances of this class too.
+                getattr(self, name).override_from_environment(environ, prefix=name + "_")
+                continue
+
+            env_var_name = field.field_info.extra.get("env", prefix + name)
+            for key in (env_var_name, env_var_name.lower(), env_var_name.upper()):
+                if key in environ:
+                    logging.info("override from environment", extra={"key": key})
+                    setattr(self, field.name, environ[key])
+                    break