TG-940 Add local MinIO service

README.md

@@ -33,6 +33,8 @@ python3 -m pip install -r requirements/common.txt
 
 The `terraform` cli package is required, unless you want to generate a project only locally. To install it we suggest to use the official [install guide](https://learn.hashicorp.com/tutorials/terraform/install-cli).
 
+Should you opt for an S3-like object storage, you must install and launch MinIO Server's `mc` package as outlined in this [install guide](https://min.io/docs/minio/linux/index.html).
+
 ## 🔑 Credentials (optional)
 
 ### 🦊 GitLab
@@ -202,6 +204,18 @@ If you don't want DigitalOcean DNS configuration the following args are required
 | local           | Docker Volume are used to store media       | `--media-storage=local`           |
 | none            | Project have no media                       | `--media-storage=none`            |
 
+#### Local S3 storage
+
+For enabling a local S3-like object storage the following argument is needed:
+
+For enabling redis integration the following arguments are needed:
+
+`--local-s3-storage`
+
+Disabled arg:
+
+`--no-local-s3-storage`
+
 #### Redis
 
 For enabling redis integration the following arguments are needed:

bootstrap/collector.py

@@ -61,6 +61,7 @@ class Collector:
     sentry_org: str | None = None
     sentry_url: str | None = None
     media_storage: str | None = None
+    local_s3_storage: bool | None = None
     gitlab_url: str | None = None
     gitlab_token: str | None = None
     gitlab_namespace_path: str | None = None
@@ -89,6 +90,7 @@ def collect(self):
         self.set_sentry()
         self.set_gitlab()
         self.set_media_storage()
+        self.set_local_s3_storage()
 
     def set_project_slug(self):
         """Set the project slug option."""
@@ -286,6 +288,13 @@ def set_media_storage(self):
                 type=click.Choice(MEDIA_STORAGE_CHOICES, case_sensitive=False),
             ).lower()
 
+    def set_local_s3_storage(self):
+        """Set the local S3 storage option."""
+        if "s3" in self.media_storage and self.local_s3_storage is None:
+            self.local_s3_storage = click.confirm(
+                warning("Do you want to use the local S3 storage?"), default=False
+            )
+
     def get_runner(self):
         """Get the bootstrap runner instance."""
         return Runner(
@@ -315,6 +324,7 @@ def get_runner(self):
             sentry_org=self.sentry_org,
             sentry_url=self.sentry_url,
             media_storage=self.media_storage,
+            local_s3_storage=self.local_s3_storage,
             use_redis=self.use_redis,
             gitlab_url=self.gitlab_url,
             gitlab_token=self.gitlab_token,

bootstrap/runner.py

@@ -73,6 +73,7 @@ class Runner:
     sentry_url: str | None = None
     media_storage: str
     use_redis: bool = False
+    local_s3_storage: bool = False
     gitlab_url: str | None = None
     gitlab_namespace_path: str | None = None
     gitlab_token: str | None = None
@@ -251,6 +252,7 @@ def init_service(self):
                 "terraform_cloud_organization": self.terraform_cloud_organization,
                 "tfvars": self.tfvars,
                 "use_redis": self.use_redis and "true" or "false",
+                "local_s3_storage": self.local_s3_storage and "true" or "false",
                 "use_vault": self.vault_url and "true" or "false",
             },
             output_dir=self.output_dir,

cookiecutter.json

@@ -10,6 +10,7 @@
   "terraform_cloud_organization": "",
   "media_storage": ["digitalocean-s3", "other-s3", "local", "none"],
   "use_redis": "false",
+  "local_s3_storage": "false",
   "use_vault": "false",
   "environments_distribution": "1",
   "resources": {

start.py

@@ -62,6 +62,7 @@
     "--media-storage",
     type=click.Choice(MEDIA_STORAGE_CHOICES, case_sensitive=False),
 )
+@click.option("--local-s3-storage/--no-local-s3-storage", is_flag=True, default=None)
 @click.option("--use-redis/--no-redis", is_flag=True, default=None)
 @click.option("--gitlab-url")
 @click.option("--gitlab-token", envvar=GITLAB_TOKEN_ENV_VAR)

{{cookiecutter.project_dirname}}/.env_template

@@ -3,7 +3,8 @@ CACHE_URL=locmem://
 COMPOSE_FILE=docker-compose.yaml
 DATABASE_URL=postgres://postgres:postgres@postgres:5432/{{ cookiecutter.project_slug }}
 DJANGO_ADMINS=admin,[email protected]
-DJANGO_ALLOWED_HOSTS=localhost,{{ cookiecutter.service_slug }}
+DJANGO_ALLOWED_HOSTS=localhost,{{ cookiecutter.service_slug }}{% if "s3" in cookiecutter.media_storage %}
+DJANGO_AWS_S3_URL=http://minio-admin:[email protected]:9000/{{ cookiecutter.project_slug }}{% endif %}
 DJANGO_CONFIGURATION=Local
 DJANGO_DEBUG=True
 [email protected]

{{cookiecutter.project_dirname}}/.gitlab-ci.yml

@@ -134,11 +134,15 @@ test:
     {{ cookiecutter.service_slug|upper }}_BUILD_TARGET: "test"
     {{ cookiecutter.service_slug|upper }}_IMAGE_NAME: "gitlabci_{{ cookiecutter.project_slug }}_{{ cookiecutter.service_slug }}"
     {{ cookiecutter.service_slug|upper }}_IMAGE_TAG: "${CI_JOB_NAME}-${CI_JOB_ID}"
-    COMPOSE_PROJECT_NAME: "${CI_PROJECT_PATH_SLUG}-${CI_JOB_NAME}-${CI_JOB_ID}"
+    COMPOSE_PROJECT_NAME: "${CI_PROJECT_PATH_SLUG}-${CI_JOB_NAME}-${CI_JOB_ID}"{% if cookiecutter.local_s3_storage == "true" %}
+    DJANGO_AWS_S3_URL: http://minio-admin:[email protected]:9000/{{ cookiecutter.project_slug }}{% endif %}
   script:
     - docker-compose build
     - docker-compose run --name ${{ "{" }}{{ cookiecutter.service_slug|upper }}_CONTAINER_NAME} {{ cookiecutter.service_slug }}
-    - docker cp ${{ "{" }}{{ cookiecutter.service_slug|upper }}_CONTAINER_NAME}:/app/htmlcov htmlcov
+    - docker cp ${{ "{" }}{{ cookiecutter.service_slug|upper }}_CONTAINER_NAME}:/app/htmlcov htmlcov{% if cookiecutter.local_s3_storage == "true" %}
+    - docker network create --subnet=172.20.0.0/16 custom
+    - docker network connect --ip 172.20.0.11 custom minio
+    - docker network connect --ip 172.20.0.13 custom postgres{% endif %}
   after_script:
     - docker-compose down -v
   coverage: '/^TOTAL.*\s+(\d+\%)$/'

{{cookiecutter.project_dirname}}/Dockerfile

@@ -69,10 +69,11 @@ RUN apt-get update \
         make \
         openssh-client \
         postgresql-client
+RUN curl https://dl.min.io/client/mc/release/linux-amd64/mc > /usr/bin/minio-client
 USER $APPUSER
 COPY --chown=$APPUSER ./requirements/local.txt requirements/local.txt
 RUN python3 -m pip install --user --no-cache-dir -r requirements/local.txt
 COPY --chown=$APPUSER . .
 RUN DJANGO_SECRET_KEY=build python3 -m manage collectstatic --clear --link --noinput
-ENTRYPOINT ["./scripts/entrypoint.sh"]
+ENTRYPOINT ["./scripts/entrypoint_local.sh"]
 CMD python3 -m manage runserver 0.0.0.0:${INTERNAL_SERVICE_PORT}

{{cookiecutter.project_dirname}}/Makefile

@@ -18,7 +18,12 @@ compilemessages:  ## Django compilemessages
 
 .PHONY: coverage
 coverage:  ## Run coverage
-	./scripts/coverage.sh
+	./scripts/coverage.sh{% if cookiecutter.local_s3_storage == "true" %}
+
+.PHONY: createbucket
+createbucket:  ## Create MinIO bucket
+	minio-client mb --quiet minio/{{ cookiecutter.project_slug }};
+	minio-client anonymous set public minio/{{ cookiecutter.project_slug }};{% endif %}
 
 .PHONY: createsuperuser
 createsuperuser:  ## Django createsuperuser

{{cookiecutter.project_dirname}}/docker-compose.yaml

@@ -10,12 +10,15 @@ services:
     image: ${{ "{" }}{{ cookiecutter.service_slug|upper }}_IMAGE_NAME:-{{ cookiecutter.project_slug }}_{{ cookiecutter.service_slug }}}:${{ "{" }}{{ cookiecutter.service_slug|upper }}_IMAGE_TAG:-latest}
     depends_on:
       postgres:
-        condition: service_healthy
+        condition: service_healthy{% if cookiecutter.local_s3_storage == "true" %}
+      minio:
+        condition: service_healthy{% endif %}
     environment:
       - CACHE_URL
       - DATABASE_URL=${DATABASE_URL:-postgres://postgres:postgres@postgres:5432/{{ cookiecutter.project_slug }}}
       - DJANGO_ADMINS
-      - DJANGO_ALLOWED_HOSTS
+      - DJANGO_ALLOWED_HOSTS{% if "s3" in cookiecutter.media_storage %}
+      - DJANGO_AWS_S3_URL{% endif %}
       - DJANGO_CONFIGURATION=${DJANGO_CONFIGURATION:-Testing}
       - DJANGO_DEBUG
       - DJANGO_DEFAULT_FROM_EMAIL
@@ -32,7 +35,10 @@ services:
       - PYTHONBREAKPOINT
     ports:
       - "${{ '{' }}{{ cookiecutter.service_slug|upper }}_PORT:-{{ cookiecutter.internal_service_port }}{{ '}' }}:${INTERNAL_SERVICE_PORT:-{{ cookiecutter.internal_service_port }}{{ '}' }}"
-    user: ${USER:-appuser}
+    user: ${USER:-appuser}{% if cookiecutter.local_s3_storage == "true" %}
+    networks:
+      custom:
+        ipv4_address: 172.20.0.10{% endif %}
 
   postgres:
     environment:
@@ -46,7 +52,39 @@ services:
       retries: 30
     image: postgres:14-bullseye
     volumes:
-      - pg_data:/var/lib/postgresql/data
+      - pg_data:/var/lib/postgresql/data{% if cookiecutter.local_s3_storage == "true" %}
+    networks:
+      custom:
+        ipv4_address: 172.20.0.11
+
+  minio:
+    command: minio server /var/lib/minio/data --console-address ":9001"
+    environment:
+      - MINIO_ENDPOINT=http://minio:9000
+      - MINIO_BUCKET_NAME={{ cookiecutter.project_slug }}
+      - MINIO_ROOT_USER=minio-admin
+      - MINIO_ROOT_PASSWORD=minio-admin
+    healthcheck:
+      test: ["CMD", "mc", "ready", "local"]
+      interval: 3s
+      timeout: 3s
+      retries: 30
+    image: minio/minio:latest
+    ports:
+      - 9000:9000
+      - 9001:9001
+    volumes:
+      - minio_data:/var/lib/minio/data
+    networks:
+      custom:
+        ipv4_address: 172.20.0.13{% endif %}
 
 volumes:
-  pg_data: {}
+  pg_data: {}{% if cookiecutter.local_s3_storage == "true" %}
+  minio_data: {}
+
+networks:
+  custom:
+    ipam:
+      config:
+      - subnet: 172.20.0.0/16{% endif %}

{{cookiecutter.project_dirname}}/requirements/common.in

@@ -1,3 +1,4 @@
 -r base.in
 django-configurations[cache,database,email]~=2.5.0
-django~=5.0.0
+{% if "s3" in cookiecutter.media_storage %}django-storages[boto3]~=1.14.0
+{% endif %}django~=5.0.0

{{cookiecutter.project_dirname}}/requirements/remote.in

@@ -1,7 +1,6 @@
 -r common.in
 argon2-cffi~=23.1.0
-{% if "s3" in cookiecutter.media_storage %}django-storages[boto3]~=1.14.0
-{% endif %}gunicorn~=21.2.0
+gunicorn~=21.2.0
 {% if cookiecutter.use_redis == "true" %}redis~=5.0.0
 {% endif %}sentry-sdk~=1.39.0
 uvicorn[standard]~=0.25.0

{{cookiecutter.project_dirname}}/scripts/entrypoint_local.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+./entrypoint.sh
+minio-client alias set minio http://minio:9000 minio-admin minio-admin;
+minio-client mb --quiet --ignore-existing minio/{{ cookiecutter.project_slug }};
+minio-client anonymous set none minio/{{ cookiecutter.project_slug }};
+exec "${@}"