Auto make update_container_documentation workflow for PRs

.github/workflows/auto-update-container-documentation.yml

@@ -0,0 +1,42 @@
+name: Auto update container documentation
+
+on:
+  pull_request:
+    paths:
+      - 'docker/compose.yaml'
+  workflow_dispatch:
+
+jobs:
+  auto-update-container-documentation:
+    runs-on: ubuntu-22.04
+    permissions:
+      contents: write
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.head_ref }}
+      - name: Install git
+        uses: awalsh128/cache-apt-pkgs-action@latest
+        with:
+          packages: git
+      - name: Cache Docker images
+        uses: ScribeMD/[email protected]
+        with:
+          key: docker-${{ runner.os }}-${{ hashFiles('.github/workflows/auto-update-container-documentation.yml') }}
+      - name: make update_container_documentation
+        id: update-container-documentation
+        run: |
+          make update_container_documentation
+          C=$(git diff -G'^\| ' documentation/Docker-container-profiles.md)
+          echo "$C"
+          if [ -z "$C" ]; then
+            echo "changed=false" >> "$GITHUB_OUTPUT"
+          else
+            echo "changed=true" >> "$GITHUB_OUTPUT"
+          fi
+      - name: Add commit to PR
+        if: steps.update-container-documentation.outputs.changed == 'true'
+        uses: stefanzweifel/git-auto-commit-action@v5
+        with:
+          commit_message: 🤖 Updated Docker-container-profiles.md after compose change by ${{ github.actor }}

Makefile

@@ -31,7 +31,7 @@ pysrc = $(shell find ${pysrcdirs} -name \*.py)
 bin = .venv/bin
 _env ?= env PATH="${bin}:$$PATH"
 
-.PHONY: translations translations_tar frontend update_padded_macs update_cert_fingerprints update_root_key_file venv frontend clean clen_venv pip-compile pip-upgrade pip-upgrade-package pip-install run run-worker run-worker-batch-callback run-worker-batch-main run-worker-batch-scheduler run-heartbeat run-broker run-rabbit manage run-test-worker version unbound-3.10-github unbound-3.7-github nassl test check autofix integration-tests batch-tests
+.PHONY: translations translations_tar frontend update_cert_fingerprints update_container_documentation update_padded_macs update_root_key_file venv frontend clean clen_venv pip-compile pip-upgrade pip-upgrade-package pip-install run run-worker run-worker-batch-callback run-worker-batch-main run-worker-batch-scheduler run-heartbeat run-broker run-rabbit manage run-test-worker version unbound-3.10-github unbound-3.7-github nassl test check autofix integration-tests batch-tests
 
 help:
 	@echo 'Makefile for internet.nl'
@@ -40,8 +40,9 @@ help:
 	@echo '   make update_content                        update the translation files from content repo.'
 	@echo '                                              Optional branch=x to use a specific content repo branch.'
 	@echo '   make frontend                              (re)generate CSS and Javascript'
-	@echo '   make update_padded_macs                    update padded MAC information'
 	@echo '   make update_cert_fingerprints              update certificate fingerprint information'
+	@echo '   make update_container_documentation        update container table for documentation'
+	@echo '   make update_padded_macs                    update padded MAC information'
 	@echo '   make update_root_key_file                  update DNS root key file'
 
 translations:
@@ -73,15 +74,19 @@ update_content:
 	${DOCKER_COMPOSE_TOOLS_CMD} run --rm tools bin/update_translations.sh
 	rm -rf tmp/locale_files
 
-update_padded_macs:
-	chmod +x $(MACSDIR)/update-macs.sh
-	cd $(MACSDIR); ./update-macs.sh
-
 update_cert_fingerprints:
 	chmod +x $(CERTSSDIR)/update-certs.sh
 	chmod +x $(CERTSSDIR)/mk-ca-bundle.pl
 	cd $(CERTSSDIR); ./update-certs.sh
 
+update_container_documentation:
+	chmod +x bin/update_container_documentation.sh
+	./bin/update_container_documentation.sh
+
+update_padded_macs:
+	chmod +x $(MACSDIR)/update-macs.sh
+	cd $(MACSDIR); ./update-macs.sh
+
 update_root_key_file:
 	unbound-anchor -a $(DNSDIR)/root.key
 

bin/update_container_documentation.sh

@@ -0,0 +1,10 @@
+#!/bin/env sh
+
+( \
+  echo "# Docker container profiles overview\n\nThis overview was last generated at $(date -uIseconds|sed 's/+00:00/Z/g') with \`make update_container_documentation\`.\n\n"
+  docker run --rm --security-opt=no-new-privileges --cap-drop all --network none -v "$PWD/docker/compose.yaml":"/docker/compose.yaml" \
+  mikefarah/yq:4.45.1 -r '"container|profiles|description","-|-|-",.services|to_entries|map([.key,(.value.profiles //[]|join(", "),(.key|head_comment|split("\n")|join("<br>")))]|join("|"))[]' /docker/compose.yaml \
+  | sed 's/$/|/' \
+  | column -ts"|" -o" | " \
+  | sed 's/^/| /;s/ $//;2{s/ /-/g}' \
+) > documentation/Docker-container-profiles.md

docker/build.env

@@ -3,7 +3,7 @@
 # include all compose files
 COMPOSE_FILE=docker/compose.yaml:docker/compose.build.yaml:docker/compose.development.yaml:docker/compose.integration-tests.yaml:docker/compose.test.yaml:docker/compose.tools.yaml:docker/compose.test-runner-develop.yaml:docker/compose.integration-tests.yaml
 # enable all profiles so all containers are built
-COMPOSE_PROFILES=routinator,run-tests
+COMPOSE_PROFILES=monitoring,routinator,run-tests
 
 # don't expose HTTP(S) and DNS ports to the outside, this also causes issues due to being privileged ports
 WEBSERVER_PORT=80

docker/compose.yaml

@@ -4,6 +4,7 @@
 # Hence no `version` is specified in this file.
 
 services:
+  # nginx proxy container, also runs certbot
   webserver:
     image: ${DOCKER_IMAGE_WEBSERVER:-${DOCKER_REGISTRY:-ghcr.io/internetstandards}/webserver:${RELEASE}}
     restart: unless-stopped
@@ -65,6 +66,7 @@ services:
       start_period: 1m
       retries: 10
 
+  # django container
   app:
     image: ${DOCKER_IMAGE_APP:-${DOCKER_REGISTRY:-ghcr.io/internetstandards}/internet.nl:${RELEASE}}
     restart: unless-stopped
@@ -167,6 +169,7 @@ services:
       - batch_results:/app/batch_results
       - manual-hof:/app/manual-hall-of-fame/
 
+  # django DB migrations, runs to completion and exits with 0
   db-migrate:
     image: ${DOCKER_IMAGE_APP:-${DOCKER_REGISTRY:-ghcr.io/internetstandards}/internet.nl:${RELEASE}}
     networks:
@@ -335,6 +338,7 @@ services:
     command: celery --app internetnl worker --without-gossip --pool=eventlet --time-limit=600 --concurrency=$WORKER_SLOW_CONCURRENCY
       --queues slow_db_worker,batch_slow
 
+  # celery task queue
   beat:
     image: ${DOCKER_IMAGE_APP:-${DOCKER_REGISTRY:-ghcr.io/internetstandards}/internet.nl:${RELEASE}}
     restart: unless-stopped
@@ -408,6 +412,10 @@ services:
       start_period: 1m
       retries: 10
 
+  # redis caches state, also used for:
+  # - MAC address lookup
+  # - Django page cache
+  # - client DNS resolver IPs in connectiontest
   redis:
     image: ${DOCKER_IMAGE_REDIS}
     restart: unless-stopped
@@ -431,6 +439,7 @@ services:
     volumes:
       - redis:/data
 
+  # rabbitmq message-broker
   rabbitmq:
     image: ${DOCKER_IMAGE_RABBITMQ}
     configs:
@@ -462,6 +471,7 @@ services:
     volumes:
       - rabbitmq:/var/lib/rabbitmq
 
+  # database
   postgres:
     image: ${DOCKER_IMAGE_POSTGRES}
     restart: unless-stopped
@@ -489,6 +499,7 @@ services:
     volumes:
       - postgres:/var/lib/postgresql/data
 
+  # for RPKI
   routinator:
     image: ${DOCKER_IMAGE_ROUTINATOR}
     restart: unless-stopped
@@ -597,6 +608,7 @@ services:
       start_period: 1m
       retries: 10
 
+  # cron with periodic tasks
   cron:
     image: ${DOCKER_IMAGE_UTIL:-${DOCKER_REGISTRY:-ghcr.io/internetstandards}/util:${RELEASE}}
     command: crond -f -d7
@@ -711,6 +723,9 @@ services:
       internal:
         ipv4_address: $IPV4_IP_GRAFANA_INTERNAL
 
+    profiles:
+      - monitoring
+
   prometheus:
     image: ${DOCKER_IMAGE_PROMETHEUS}
 
@@ -741,6 +756,10 @@ services:
     volumes:
       - prometheus-data:/prometheus
 
+    profiles:
+      - monitoring
+
+  # requires monitoring profile
   alertmanager:
     image: ${DOCKER_IMAGE_PROMETHEUS_ALERTMANAGER}
 
@@ -783,6 +802,9 @@ services:
     networks:
       - internal
 
+    profiles:
+      - monitoring
+
   redis-exporter:
     image: ${DOCKER_IMAGE_REDIS_EXPORTER}
 
@@ -800,6 +822,9 @@ services:
     networks:
       - internal
 
+    profiles:
+      - monitoring
+
   statsd-exporter:
     image: ${DOCKER_IMAGE_STATSD_EXPORTER}
 
@@ -821,6 +846,9 @@ services:
         aliases:
           - statsd
 
+    profiles:
+      - monitoring
+
   celery-exporter:
     image: ${DOCKER_IMAGE_CELERY_EXPORTER}
     command:
@@ -842,8 +870,11 @@ services:
       rabbitmq:
         condition: service_healthy
 
-  # https://github.com/prometheus/node_exporter#docker
+    profiles:
+      - monitoring
+
   node-exporter:
+    # https://github.com/prometheus/node_exporter#docker
     image: ${DOCKER_IMAGE_NODE_EXPORTER}
     command:
       - --path.rootfs=/host
@@ -872,6 +903,9 @@ services:
       - /var/run/dbus/system_bus_socket:/var/run/dbus/system_bus_socket
       - prometheus-textfile-directory:/prometheus-textfile-directory
 
+    profiles:
+      - monitoring
+
   docker_stats_exporter:
     # https://github.com/jan4843/docker_stats_exporter
     image: ${DOCKER_IMAGE_DOCKER_STATSD_EXPORTER}
@@ -891,6 +925,9 @@ services:
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock
 
+    profiles:
+      - monitoring
+
   nginx_logs_exporter:
     image: ${DOCKER_IMAGE_NGINX_LOGS_EXPORTER}
     command:
@@ -912,6 +949,9 @@ services:
     volumes:
       - nginx-logs-exporter:/var/log/nginx/prometheus-nginxlog-exporter/
 
+    profiles:
+      - monitoring
+
 volumes:
   postgres: {}
   postgres-backups: {}

docker/defaults.env

@@ -121,7 +121,7 @@ INTERNET_NL_CHECK_SUPPORT_RPKI=True
 # list of domainnames that can have retry timer be reset via API
 INTERNETNL_CACHE_RESET_ALLOWLIST=
 
-# settings for alertmanager, enable it by adding 'routinator,alertmanager' to COMPOSE_PROFILES in `docker/local.env`
+# settings for alertmanager, enable it by adding 'alertmanager,monitoring,routinator' to COMPOSE_PROFILES in `docker/local.env`
 # sending email address used for alerts
 ALERTMANAGER_MAIL_FROM=
 
@@ -146,7 +146,7 @@ COMPOSE_PROJECT_NAME=internetnl
 COMPOSE_FILE=docker/compose.yaml
 
 # enable routinator, this profile is omitted in develop.env because it is resource intensive for development environments
-COMPOSE_PROFILES=routinator
+COMPOSE_PROFILES=monitoring,routinator
 
 # interval for batch processing
 BATCH_SCHEDULER_INTERVAL=1

docker/develop.env

@@ -76,4 +76,4 @@ PAGE_CACHE_TIME_SECONDS=0
 # Disable (do not enable) the `routinator` profile which is enable by default in `defaults.env`.
 # Routinator is slow to start initially and requires a lot of resources which is not ideal for
 # development environments.
-COMPOSE_PROFILES=
+COMPOSE_PROFILES=monitoring

documentation/Docker-container-profiles.md

@@ -0,0 +1,32 @@
+# Docker container profiles overview
+
+This overview was last generated at 2025-02-09T23:19:39Z with `make update_container_documentation`.
+
+
+| container             | profiles     | description                                                                                                                      |
+|-----------------------|--------------|----------------------------------------------------------------------------------------------------------------------------------|
+| webserver             |              | nginx proxy container, also runs certbot                                                                                         |
+| app                   |              | django container                                                                                                                 |
+| db-migrate            |              | django DB migrations, runs to completion and exits with 0                                                                        |
+| worker                |              |                                                                                                                                  |
+| worker-nassl          |              | worker for queue with potential memory leak                                                                                      |
+| worker-slow           |              | worker for slow and long running tasks that could require a lot of memory (eg: hof update)                                       |
+| beat                  |              | celery task queue                                                                                                                |
+| redis                 |              | redis caches state, also used for:<br>- MAC address lookup<br>- Django page cache<br>- client DNS resolver IPs in connectiontest |
+| rabbitmq              |              | rabbitmq message-broker                                                                                                          |
+| postgres              |              | database                                                                                                                         |
+| routinator            | routinator   | for RPKI                                                                                                                         |
+| unbound               |              | unbound DNS server used for connection test                                                                                      |
+| resolver-validating   |              | unbound resolver used for ldns-dane that require DNSSEC validation                                                               |
+| cron                  |              | cron with periodic tasks                                                                                                         |
+| cron-docker           |              | cron daemon with access to Docker socket but no networking                                                                       |
+| grafana               | monitoring   |                                                                                                                                  |
+| prometheus            | monitoring   |                                                                                                                                  |
+| alertmanager          | alertmanager | requires monitoring profile                                                                                                      |
+| postgresql-exporter   | monitoring   |                                                                                                                                  |
+| redis-exporter        | monitoring   |                                                                                                                                  |
+| statsd-exporter       | monitoring   |                                                                                                                                  |
+| celery-exporter       | monitoring   |                                                                                                                                  |
+| node-exporter         | monitoring   |                                                                                                                                  |
+| docker_stats_exporter | monitoring   |                                                                                                                                  |
+| nginx_logs_exporter   | monitoring   |                                                                                                                                  |

documentation/Docker-deployment.md

@@ -390,7 +390,7 @@ A Prometheus Alertmanager service is available but disabled by default. Enabling
 
 To enable and configure the Alertmanager add the following lines to `docker/local.env` and adjust the values to be applicable for your environment:
 
-    COMPOSE_PROFILES=routinator,alertmanager
+    COMPOSE_PROFILES=alertmanager,monitoring,routinator
     [email protected],[email protected]
     [email protected]
     ALERTMANAGER_SMTP_HOST=smtp.example.com

documentation/github_release_steps.md

@@ -9,6 +9,7 @@ code changes for the next release are already in the main branch.
 3. Run all the update scripts:
    ```
    make update_cert_fingerprints
+   make update_container_documentation
    make update_padded_macs
    make update_root_key_file
    make translate_content_to_main