Skip to content

Commit b960b60

Browse files
authored
feat: secrets and storages (#148)
* feat: secrets and storages * cast env * fix: key errors * fix: urls * AWS_S3_OBJECT_PARAMETERS * fix: types * fix: linting * refresh pipfile.lock * fix: imports
1 parent 4a1bb70 commit b960b60

File tree

11 files changed

+360
-191
lines changed

11 files changed

+360
-191
lines changed

Pipfile

+2
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,9 @@ django-two-factor-auth = "==1.13.2"
1313
django-cors-headers = "==4.1.0"
1414
django-csp = "==3.7"
1515
django-import-export = "==4.0.3"
16+
django-storages = {version = "==1.14.4", extras = ["s3"]}
1617
pyotp = "==2.9.0"
18+
python-dotenv = "==1.0.1"
1719
psycopg2-binary = "==2.9.9"
1820
requests = "==2.32.2"
1921
gunicorn = "==23.0.0"

Pipfile.lock

+107-87
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

codeforlife/__init__.py

+113
Original file line numberDiff line numberDiff line change
@@ -3,10 +3,123 @@
33
Created on 20/02/2024 at 09:28:27(+00:00).
44
"""
55

6+
import os
7+
import sys
8+
import typing as t
9+
from io import StringIO
610
from pathlib import Path
11+
from types import SimpleNamespace
712

13+
from .types import Env
814
from .version import __version__
915

1016
BASE_DIR = Path(__file__).resolve().parent
1117
DATA_DIR = BASE_DIR.joinpath("data")
1218
USER_DIR = BASE_DIR.joinpath("user")
19+
20+
21+
if t.TYPE_CHECKING:
22+
from mypy_boto3_s3.client import S3Client
23+
24+
25+
# pylint: disable-next=too-few-public-methods
26+
class Secrets(SimpleNamespace):
27+
"""The secrets for this service.
28+
29+
If a key does not exist, the value None will be returned.
30+
"""
31+
32+
def __getattribute__(self, name: str) -> t.Optional[str]:
33+
try:
34+
return super().__getattribute__(name)
35+
except AttributeError:
36+
return None
37+
38+
39+
def set_up_settings(service_base_dir: Path, service_name: str):
40+
"""Set up the settings for the service.
41+
42+
*This needs to be called before importing the CFL settings!*
43+
44+
To expose a secret to your Django project, you'll need to create a setting
45+
for it following Django's conventions.
46+
47+
Examples:
48+
```
49+
from codeforlife import set_up_settings
50+
51+
# Must set up settings before importing them!
52+
secrets = set_up_settings("my-service")
53+
54+
from codeforlife.settings import *
55+
56+
# Expose secret to django project.
57+
MY_SECRET = secrets.MY_SECRET
58+
```
59+
60+
Args:
61+
service_base_dir: The base directory of the service.
62+
service_name: The name of the service.
63+
64+
Returns:
65+
The secrets. These are not loaded as environment variables so that 3rd
66+
party packages cannot read them.
67+
"""
68+
69+
# Validate CFL settings have not been imported yet.
70+
if "codeforlife.settings" in sys.modules:
71+
raise ImportError(
72+
"You must set up the CFL settings before importing them."
73+
)
74+
75+
# pylint: disable-next=import-outside-toplevel
76+
from dotenv import dotenv_values, load_dotenv
77+
78+
# Set required environment variables.
79+
os.environ["SERVICE_BASE_DIR"] = str(service_base_dir)
80+
os.environ["SERVICE_NAME"] = service_name
81+
82+
# Get environment name.
83+
os.environ.setdefault("ENV", "local")
84+
env = t.cast(Env, os.environ["ENV"])
85+
86+
# Load environment variables.
87+
load_dotenv(f".env/.env.{env}", override=False)
88+
load_dotenv(".env/.env", override=False)
89+
90+
# Get secrets.
91+
if env == "local":
92+
secrets_path = ".env/.env.local.secrets"
93+
# TODO: move this to the dev container setup script.
94+
if not os.path.exists(secrets_path):
95+
# pylint: disable=line-too-long
96+
secrets_file_comment = (
97+
"# 📝 Local Secret Variables 📝\n"
98+
"# These secret variables are only loaded in your local environment (on your PC).\n"
99+
"#\n"
100+
"# This file is git-ignored intentionally to keep these variables a secret.\n"
101+
"#\n"
102+
"# 🚫 DO NOT PUSH SECRETS TO THE CODE REPO 🚫\n"
103+
"\n"
104+
)
105+
# pylint: enable=line-too-long
106+
107+
with open(secrets_path, "w+", encoding="utf-8") as secrets_file:
108+
secrets_file.write(secrets_file_comment)
109+
110+
secrets = dotenv_values(secrets_path)
111+
else:
112+
# pylint: disable-next=import-outside-toplevel
113+
import boto3
114+
115+
s3: "S3Client" = boto3.client("s3")
116+
secrets_object = s3.get_object(
117+
Bucket=os.environ["aws_s3_app_bucket"],
118+
Key=f"{os.environ['aws_s3_app_folder']}/secure/.env.secrets",
119+
)
120+
121+
secrets = dotenv_values(
122+
stream=StringIO(secrets_object["Body"].read().decode("utf-8"))
123+
)
124+
125+
return Secrets(**secrets)

codeforlife/settings/custom.py

+13-15
Original file line numberDiff line numberDiff line change
@@ -3,13 +3,20 @@
33
"""
44

55
import os
6+
import typing as t
7+
from pathlib import Path
8+
9+
from ..types import Env
10+
11+
# The name of the current environment.
12+
ENV = t.cast(Env, os.getenv("ENV", "local"))
13+
14+
# The base directory of the current service.
15+
SERVICE_BASE_DIR = Path(os.getenv("SERVICE_BASE_DIR", "/"))
616

717
# The name of the current service.
818
SERVICE_NAME = os.getenv("SERVICE_NAME", "REPLACE_ME")
919

10-
# If the current service the root service. This will only be true for portal.
11-
SERVICE_IS_ROOT = bool(int(os.getenv("SERVICE_IS_ROOT", "0")))
12-
1320
# The protocol, domain and port of the current service.
1421
SERVICE_PROTOCOL = os.getenv("SERVICE_PROTOCOL", "http")
1522
SERVICE_DOMAIN = os.getenv("SERVICE_DOMAIN", "localhost")
@@ -18,18 +25,9 @@
1825
# The base url of the current service.
1926
# The root service does not need its name included in the base url.
2027
SERVICE_BASE_URL = f"{SERVICE_PROTOCOL}://{SERVICE_DOMAIN}:{SERVICE_PORT}"
21-
if not SERVICE_IS_ROOT:
22-
SERVICE_BASE_URL += f"/{SERVICE_NAME}"
23-
24-
# The api url of the current service.
25-
SERVICE_API_URL = f"{SERVICE_BASE_URL}/api"
26-
27-
# The website url of the current service.
28-
SERVICE_SITE_URL = (
29-
"http://localhost:5173"
30-
if SERVICE_DOMAIN == "localhost"
31-
else SERVICE_BASE_URL
32-
)
28+
29+
# The frontend url of the current service.
30+
SERVICE_SITE_URL = os.getenv("SERVICE_SITE_URL", "http://localhost:5173")
3331

3432
# The authorization bearer token used to authenticate with Dotdigital.
3533
MAIL_AUTH = os.getenv("MAIL_AUTH", "REPLACE_ME")

codeforlife/settings/django.py

+27-30
Original file line numberDiff line numberDiff line change
@@ -6,13 +6,12 @@
66
import json
77
import os
88
import typing as t
9-
from pathlib import Path
109

1110
import boto3
1211
from django.utils.translation import gettext_lazy as _
1312

1413
from ..types import JsonDict
15-
from .custom import SERVICE_API_URL, SERVICE_NAME
14+
from .custom import ENV, SERVICE_BASE_DIR, SERVICE_BASE_URL, SERVICE_NAME
1615
from .otp import APP_ID, AWS_S3_APP_BUCKET, AWS_S3_APP_FOLDER
1716

1817
if t.TYPE_CHECKING:
@@ -41,11 +40,17 @@ def get_databases():
4140
The database configs.
4241
"""
4342

44-
if AWS_S3_APP_BUCKET and AWS_S3_APP_FOLDER and APP_ID:
43+
if ENV == "local":
44+
name = os.getenv("DB_NAME", SERVICE_NAME)
45+
user = os.getenv("DB_USER", "root")
46+
password = os.getenv("DB_PASSWORD", "password")
47+
host = os.getenv("DB_HOST", "localhost")
48+
port = int(os.getenv("DB_PORT", "5432"))
49+
else:
4550
# Get the dbdata object.
4651
s3: "S3Client" = boto3.client("s3")
4752
db_data_object = s3.get_object(
48-
Bucket=AWS_S3_APP_BUCKET,
53+
Bucket=t.cast(str, AWS_S3_APP_BUCKET),
4954
Key=f"{AWS_S3_APP_FOLDER}/dbMetadata/{APP_ID}/app.dbdata",
5055
)
5156

@@ -56,17 +61,11 @@ def get_databases():
5661
if not db_data or db_data["DBEngine"] != "postgres":
5762
raise ConnectionAbortedError("Invalid database data.")
5863

59-
name = db_data["Database"]
60-
user = db_data["user"]
61-
password = db_data["password"]
62-
host = db_data["Endpoint"]
63-
port = db_data["Port"]
64-
else:
65-
name = os.getenv("DB_NAME", SERVICE_NAME)
66-
user = os.getenv("DB_USER", "root")
67-
password = os.getenv("DB_PASSWORD", "password")
68-
host = os.getenv("DB_HOST", "localhost")
69-
port = int(os.getenv("DB_PORT", "5432"))
64+
name = t.cast(str, db_data["Database"])
65+
user = t.cast(str, db_data["user"])
66+
password = t.cast(str, db_data["password"])
67+
host = t.cast(str, db_data["Endpoint"])
68+
port = t.cast(int, db_data["Port"])
7069

7170
return {
7271
"default": {
@@ -104,7 +103,7 @@ def get_databases():
104103
# Auth
105104
# https://docs.djangoproject.com/en/3.2/topics/auth/default/
106105

107-
LOGIN_URL = f"{SERVICE_API_URL}/session/expired/"
106+
LOGIN_URL = f"{SERVICE_BASE_URL}/session/expired/"
108107

109108
# Authentication backends
110109
# https://docs.djangoproject.com/en/3.2/ref/settings/#authentication-backends
@@ -243,25 +242,14 @@ def get_databases():
243242
"corsheaders",
244243
"rest_framework",
245244
"django_filters",
245+
"storages",
246246
]
247247

248248
# Static files (CSS, JavaScript, Images)
249249
# https://docs.djangoproject.com/en/3.2/howto/static-files/
250250

251-
252-
def get_static_root(base_dir: Path):
253-
"""Get the static root for the Django project.
254-
255-
Args:
256-
base_dir: The base directory of the Django project.
257-
258-
Returns:
259-
The static root for the django project.
260-
"""
261-
return base_dir / "static"
262-
263-
264-
STATIC_URL = "/static/"
251+
STATIC_ROOT = SERVICE_BASE_DIR / "static"
252+
STATIC_URL = os.getenv("STATIC_URL", "/static/")
265253

266254
# Templates
267255
# https://docs.djangoproject.com/en/3.2/ref/templates/
@@ -281,3 +269,12 @@ def get_static_root(base_dir: Path):
281269
},
282270
},
283271
]
272+
273+
# File storage
274+
# https://docs.djangoproject.com/en/3.2/topics/files/#file-storage
275+
276+
DEFAULT_FILE_STORAGE = (
277+
"django.core.files.storage.FileSystemStorage"
278+
if ENV == "local"
279+
else "storages.backends.s3.S3Storage"
280+
)

codeforlife/settings/third_party.py

+45
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,9 @@
22
This file contains custom settings defined by third party extensions.
33
"""
44

5+
import json
6+
import os
7+
58
from .django import DEBUG
69

710
# CORS
@@ -23,3 +26,45 @@
2326
],
2427
"DEFAULT_PAGINATION_CLASS": "codeforlife.pagination.LimitOffsetPagination",
2528
}
29+
30+
# Django storages
31+
# https://django-storages.readthedocs.io/en/latest/backends/amazon-S3.html#settings
32+
33+
AWS_STORAGE_BUCKET_NAME = os.getenv("AWS_STORAGE_BUCKET_NAME")
34+
AWS_S3_OBJECT_PARAMETERS = json.loads(
35+
os.getenv("AWS_S3_OBJECT_PARAMETERS", "{}")
36+
)
37+
AWS_DEFAULT_ACL = os.getenv("AWS_DEFAULT_ACL")
38+
AWS_QUERYSTRING_AUTH = bool(int(os.getenv("AWS_QUERYSTRING_AUTH", "1")))
39+
AWS_S3_MAX_MEMORY_SIZE = int(os.getenv("AWS_S3_MAX_MEMORY_SIZE", "0"))
40+
AWS_QUERYSTRING_EXPIRE = int(os.getenv("AWS_QUERYSTRING_EXPIRE", "3600"))
41+
AWS_S3_URL_PROTOCOL = os.getenv("AWS_S3_URL_PROTOCOL", "https:")
42+
AWS_S3_FILE_OVERWRITE = bool(int(os.getenv("AWS_S3_FILE_OVERWRITE", "1")))
43+
AWS_LOCATION = os.getenv("AWS_LOCATION", "")
44+
AWS_IS_GZIPPED = bool(int(os.getenv("AWS_IS_GZIPPED", "0")))
45+
GZIP_CONTENT_TYPES = os.getenv(
46+
"GZIP_CONTENT_TYPES",
47+
"("
48+
+ ",".join(
49+
[
50+
"text/css",
51+
"text/javascript",
52+
"application/javascript",
53+
"application/x-javascript",
54+
"image/svg+xml",
55+
]
56+
)
57+
+ ")",
58+
)
59+
AWS_S3_REGION_NAME = os.getenv("AWS_S3_REGION_NAME")
60+
AWS_S3_USE_SSL = bool(int(os.getenv("AWS_S3_USE_SSL", "1")))
61+
AWS_S3_VERIFY = os.getenv("AWS_S3_VERIFY")
62+
AWS_S3_ENDPOINT_URL = os.getenv("AWS_S3_ENDPOINT_URL")
63+
AWS_S3_ADDRESSING_STYLE = os.getenv("AWS_S3_ADDRESSING_STYLE")
64+
AWS_S3_PROXIES = os.getenv("AWS_S3_PROXIES")
65+
AWS_S3_TRANSFER_CONFIG = os.getenv("AWS_S3_TRANSFER_CONFIG")
66+
AWS_S3_CUSTOM_DOMAIN = os.getenv("AWS_S3_CUSTOM_DOMAIN")
67+
AWS_CLOUDFRONT_KEY = os.getenv("AWS_CLOUDFRONT_KEY")
68+
AWS_CLOUDFRONT_KEY_ID = os.getenv("AWS_CLOUDFRONT_KEY_ID")
69+
AWS_S3_SIGNATURE_VERSION = os.getenv("AWS_S3_SIGNATURE_VERSION")
70+
AWS_S3_CLIENT_CONFIG = os.getenv("AWS_S3_CLIENT_CONFIG")

codeforlife/types.py

+2
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,8 @@
77

88
import typing as t
99

10+
Env = t.Literal["local", "development", "staging", "production"]
11+
1012
Args = t.Tuple[t.Any, ...]
1113
KwArgs = t.Dict[str, t.Any]
1214

0 commit comments

Comments
 (0)