feat: secrets and storages (#148)

* feat: secrets and storages

* cast env

* fix: key errors

* fix: urls

* AWS_S3_OBJECT_PARAMETERS

* fix: types

* fix: linting

* refresh pipfile.lock

* fix: imports

Author: SKairinos

Diff for: Pipfile

@@ -13,7 +13,9 @@ django-two-factor-auth = "==1.13.2"
 django-cors-headers = "==4.1.0"
 django-csp = "==3.7"
 django-import-export = "==4.0.3"
+django-storages = {version = "==1.14.4", extras = ["s3"]}
 pyotp = "==2.9.0"
+python-dotenv = "==1.0.1"
 psycopg2-binary = "==2.9.9"
 requests = "==2.32.2"
 gunicorn = "==23.0.0"

Diff for: Pipfile.lock

@@ -3,10 +3,123 @@
 Created on 20/02/2024 at 09:28:27(+00:00).
 """
 
+import os
+import sys
+import typing as t
+from io import StringIO
 from pathlib import Path
+from types import SimpleNamespace
 
+from .types import Env
 from .version import __version__
 
 BASE_DIR = Path(__file__).resolve().parent
 DATA_DIR = BASE_DIR.joinpath("data")
 USER_DIR = BASE_DIR.joinpath("user")
+
+
+if t.TYPE_CHECKING:
+    from mypy_boto3_s3.client import S3Client
+
+
+# pylint: disable-next=too-few-public-methods
+class Secrets(SimpleNamespace):
+    """The secrets for this service.
+
+    If a key does not exist, the value None will be returned.
+    """
+
+    def __getattribute__(self, name: str) -> t.Optional[str]:
+        try:
+            return super().__getattribute__(name)
+        except AttributeError:
+            return None
+
+
+def set_up_settings(service_base_dir: Path, service_name: str):
+    """Set up the settings for the service.
+
+    *This needs to be called before importing the CFL settings!*
+
+    To expose a secret to your Django project, you'll need to create a setting
+    for it following Django's conventions.
+
+    Examples:
+        ```
+        from codeforlife import set_up_settings
+
+        # Must set up settings before importing them!
+        secrets = set_up_settings("my-service")
+
+        from codeforlife.settings import *
+
+        # Expose secret to django project.
+        MY_SECRET = secrets.MY_SECRET
+        ```
+
+    Args:
+        service_base_dir: The base directory of the service.
+        service_name: The name of the service.
+
+    Returns:
+        The secrets. These are not loaded as environment variables so that 3rd
+        party packages cannot read them.
+    """
+
+    # Validate CFL settings have not been imported yet.
+    if "codeforlife.settings" in sys.modules:
+        raise ImportError(
+            "You must set up the CFL settings before importing them."
+        )
+
+    # pylint: disable-next=import-outside-toplevel
+    from dotenv import dotenv_values, load_dotenv
+
+    # Set required environment variables.
+    os.environ["SERVICE_BASE_DIR"] = str(service_base_dir)
+    os.environ["SERVICE_NAME"] = service_name
+
+    # Get environment name.
+    os.environ.setdefault("ENV", "local")
+    env = t.cast(Env, os.environ["ENV"])
+
+    # Load environment variables.
+    load_dotenv(f".env/.env.{env}", override=False)
+    load_dotenv(".env/.env", override=False)
+
+    # Get secrets.
+    if env == "local":
+        secrets_path = ".env/.env.local.secrets"
+        # TODO: move this to the dev container setup script.
+        if not os.path.exists(secrets_path):
+            # pylint: disable=line-too-long
+            secrets_file_comment = (
+                "# 📝 Local Secret Variables 📝\n"
+                "# These secret variables are only loaded in your local environment (on your PC).\n"
+                "#\n"
+                "# This file is git-ignored intentionally to keep these variables a secret.\n"
+                "#\n"
+                "# 🚫 DO NOT PUSH SECRETS TO THE CODE REPO 🚫\n"
+                "\n"
+            )
+            # pylint: enable=line-too-long
+
+            with open(secrets_path, "w+", encoding="utf-8") as secrets_file:
+                secrets_file.write(secrets_file_comment)
+
+        secrets = dotenv_values(secrets_path)
+    else:
+        # pylint: disable-next=import-outside-toplevel
+        import boto3
+
+        s3: "S3Client" = boto3.client("s3")
+        secrets_object = s3.get_object(
+            Bucket=os.environ["aws_s3_app_bucket"],
+            Key=f"{os.environ['aws_s3_app_folder']}/secure/.env.secrets",
+        )
+
+        secrets = dotenv_values(
+            stream=StringIO(secrets_object["Body"].read().decode("utf-8"))
+        )
+
+    return Secrets(**secrets)

Diff for: codeforlife/__init__.py

@@ -3,13 +3,20 @@
 """
 
 import os
+import typing as t
+from pathlib import Path
+
+from ..types import Env
+
+# The name of the current environment.
+ENV = t.cast(Env, os.getenv("ENV", "local"))
+
+# The base directory of the current service.
+SERVICE_BASE_DIR = Path(os.getenv("SERVICE_BASE_DIR", "/"))
 
 # The name of the current service.
 SERVICE_NAME = os.getenv("SERVICE_NAME", "REPLACE_ME")
 
-# If the current service the root service. This will only be true for portal.
-SERVICE_IS_ROOT = bool(int(os.getenv("SERVICE_IS_ROOT", "0")))
-
 # The protocol, domain and port of the current service.
 SERVICE_PROTOCOL = os.getenv("SERVICE_PROTOCOL", "http")
 SERVICE_DOMAIN = os.getenv("SERVICE_DOMAIN", "localhost")
@@ -18,18 +25,9 @@
 # The base url of the current service.
 # The root service does not need its name included in the base url.
 SERVICE_BASE_URL = f"{SERVICE_PROTOCOL}://{SERVICE_DOMAIN}:{SERVICE_PORT}"
-if not SERVICE_IS_ROOT:
-    SERVICE_BASE_URL += f"/{SERVICE_NAME}"
-
-# The api url of the current service.
-SERVICE_API_URL = f"{SERVICE_BASE_URL}/api"
-
-# The website url of the current service.
-SERVICE_SITE_URL = (
-    "http://localhost:5173"
-    if SERVICE_DOMAIN == "localhost"
-    else SERVICE_BASE_URL
-)
+
+# The frontend url of the current service.
+SERVICE_SITE_URL = os.getenv("SERVICE_SITE_URL", "http://localhost:5173")
 
 # The authorization bearer token used to authenticate with Dotdigital.
 MAIL_AUTH = os.getenv("MAIL_AUTH", "REPLACE_ME")

Diff for: codeforlife/settings/custom.py

@@ -6,13 +6,12 @@
 import json
 import os
 import typing as t
-from pathlib import Path
 
 import boto3
 from django.utils.translation import gettext_lazy as _
 
 from ..types import JsonDict
-from .custom import SERVICE_API_URL, SERVICE_NAME
+from .custom import ENV, SERVICE_BASE_DIR, SERVICE_BASE_URL, SERVICE_NAME
 from .otp import APP_ID, AWS_S3_APP_BUCKET, AWS_S3_APP_FOLDER
 
 if t.TYPE_CHECKING:
@@ -41,11 +40,17 @@ def get_databases():
         The database configs.
     """
 
-    if AWS_S3_APP_BUCKET and AWS_S3_APP_FOLDER and APP_ID:
+    if ENV == "local":
+        name = os.getenv("DB_NAME", SERVICE_NAME)
+        user = os.getenv("DB_USER", "root")
+        password = os.getenv("DB_PASSWORD", "password")
+        host = os.getenv("DB_HOST", "localhost")
+        port = int(os.getenv("DB_PORT", "5432"))
+    else:
         # Get the dbdata object.
         s3: "S3Client" = boto3.client("s3")
         db_data_object = s3.get_object(
-            Bucket=AWS_S3_APP_BUCKET,
+            Bucket=t.cast(str, AWS_S3_APP_BUCKET),
             Key=f"{AWS_S3_APP_FOLDER}/dbMetadata/{APP_ID}/app.dbdata",
         )
 
@@ -56,17 +61,11 @@ def get_databases():
         if not db_data or db_data["DBEngine"] != "postgres":
             raise ConnectionAbortedError("Invalid database data.")
 
-        name = db_data["Database"]
-        user = db_data["user"]
-        password = db_data["password"]
-        host = db_data["Endpoint"]
-        port = db_data["Port"]
-    else:
-        name = os.getenv("DB_NAME", SERVICE_NAME)
-        user = os.getenv("DB_USER", "root")
-        password = os.getenv("DB_PASSWORD", "password")
-        host = os.getenv("DB_HOST", "localhost")
-        port = int(os.getenv("DB_PORT", "5432"))
+        name = t.cast(str, db_data["Database"])
+        user = t.cast(str, db_data["user"])
+        password = t.cast(str, db_data["password"])
+        host = t.cast(str, db_data["Endpoint"])
+        port = t.cast(int, db_data["Port"])
 
     return {
         "default": {
@@ -104,7 +103,7 @@ def get_databases():
 # Auth
 # https://docs.djangoproject.com/en/3.2/topics/auth/default/
 
-LOGIN_URL = f"{SERVICE_API_URL}/session/expired/"
+LOGIN_URL = f"{SERVICE_BASE_URL}/session/expired/"
 
 # Authentication backends
 # https://docs.djangoproject.com/en/3.2/ref/settings/#authentication-backends
@@ -243,25 +242,14 @@ def get_databases():
     "corsheaders",
     "rest_framework",
     "django_filters",
+    "storages",
 ]
 
 # Static files (CSS, JavaScript, Images)
 # https://docs.djangoproject.com/en/3.2/howto/static-files/
 
-
-def get_static_root(base_dir: Path):
-    """Get the static root for the Django project.
-
-    Args:
-        base_dir: The base directory of the Django project.
-
-    Returns:
-        The static root for the django project.
-    """
-    return base_dir / "static"
-
-
-STATIC_URL = "/static/"
+STATIC_ROOT = SERVICE_BASE_DIR / "static"
+STATIC_URL = os.getenv("STATIC_URL", "/static/")
 
 # Templates
 # https://docs.djangoproject.com/en/3.2/ref/templates/
@@ -281,3 +269,12 @@ def get_static_root(base_dir: Path):
         },
     },
 ]
+
+# File storage
+# https://docs.djangoproject.com/en/3.2/topics/files/#file-storage
+
+DEFAULT_FILE_STORAGE = (
+    "django.core.files.storage.FileSystemStorage"
+    if ENV == "local"
+    else "storages.backends.s3.S3Storage"
+)

Diff for: codeforlife/settings/django.py

@@ -2,6 +2,9 @@
 This file contains custom settings defined by third party extensions.
 """
 
+import json
+import os
+
 from .django import DEBUG
 
 # CORS
@@ -23,3 +26,45 @@
     ],
     "DEFAULT_PAGINATION_CLASS": "codeforlife.pagination.LimitOffsetPagination",
 }
+
+# Django storages
+# https://django-storages.readthedocs.io/en/latest/backends/amazon-S3.html#settings
+
+AWS_STORAGE_BUCKET_NAME = os.getenv("AWS_STORAGE_BUCKET_NAME")
+AWS_S3_OBJECT_PARAMETERS = json.loads(
+    os.getenv("AWS_S3_OBJECT_PARAMETERS", "{}")
+)
+AWS_DEFAULT_ACL = os.getenv("AWS_DEFAULT_ACL")
+AWS_QUERYSTRING_AUTH = bool(int(os.getenv("AWS_QUERYSTRING_AUTH", "1")))
+AWS_S3_MAX_MEMORY_SIZE = int(os.getenv("AWS_S3_MAX_MEMORY_SIZE", "0"))
+AWS_QUERYSTRING_EXPIRE = int(os.getenv("AWS_QUERYSTRING_EXPIRE", "3600"))
+AWS_S3_URL_PROTOCOL = os.getenv("AWS_S3_URL_PROTOCOL", "https:")
+AWS_S3_FILE_OVERWRITE = bool(int(os.getenv("AWS_S3_FILE_OVERWRITE", "1")))
+AWS_LOCATION = os.getenv("AWS_LOCATION", "")
+AWS_IS_GZIPPED = bool(int(os.getenv("AWS_IS_GZIPPED", "0")))
+GZIP_CONTENT_TYPES = os.getenv(
+    "GZIP_CONTENT_TYPES",
+    "("
+    + ",".join(
+        [
+            "text/css",
+            "text/javascript",
+            "application/javascript",
+            "application/x-javascript",
+            "image/svg+xml",
+        ]
+    )
+    + ")",
+)
+AWS_S3_REGION_NAME = os.getenv("AWS_S3_REGION_NAME")
+AWS_S3_USE_SSL = bool(int(os.getenv("AWS_S3_USE_SSL", "1")))
+AWS_S3_VERIFY = os.getenv("AWS_S3_VERIFY")
+AWS_S3_ENDPOINT_URL = os.getenv("AWS_S3_ENDPOINT_URL")
+AWS_S3_ADDRESSING_STYLE = os.getenv("AWS_S3_ADDRESSING_STYLE")
+AWS_S3_PROXIES = os.getenv("AWS_S3_PROXIES")
+AWS_S3_TRANSFER_CONFIG = os.getenv("AWS_S3_TRANSFER_CONFIG")
+AWS_S3_CUSTOM_DOMAIN = os.getenv("AWS_S3_CUSTOM_DOMAIN")
+AWS_CLOUDFRONT_KEY = os.getenv("AWS_CLOUDFRONT_KEY")
+AWS_CLOUDFRONT_KEY_ID = os.getenv("AWS_CLOUDFRONT_KEY_ID")
+AWS_S3_SIGNATURE_VERSION = os.getenv("AWS_S3_SIGNATURE_VERSION")
+AWS_S3_CLIENT_CONFIG = os.getenv("AWS_S3_CLIENT_CONFIG")

Diff for: codeforlife/settings/third_party.py

@@ -7,6 +7,8 @@
 
 import typing as t
 
+Env = t.Literal["local", "development", "staging", "production"]
+
 Args = t.Tuple[t.Any, ...]
 KwArgs = t.Dict[str, t.Any]