diff --git a/Makefile b/Makefile
index 6e9e7cc08..9b72ed779 100644
--- a/Makefile
+++ b/Makefile
@@ -31,7 +31,7 @@ pysrc = $(shell find ${pysrcdirs} -name \*.py)
bin = .venv/bin
_env ?= env PATH="${bin}:$$PATH"
-.PHONY: translations translations_tar frontend update_padded_macs update_cert_fingerprints update_root_key_file venv frontend clean clen_venv pip-compile pip-upgrade pip-upgrade-package pip-install run run-worker run-worker-batch-callback run-worker-batch-main run-worker-batch-scheduler run-heartbeat run-broker run-rabbit manage run-test-worker version unbound-3.10-github unbound-3.7-github nassl test check autofix integration-tests batch-tests
+.PHONY: translations translations_tar frontend update_cert_fingerprints update_container_documentation update_padded_macs update_root_key_file venv frontend clean clen_venv pip-compile pip-upgrade pip-upgrade-package pip-install run run-worker run-worker-batch-callback run-worker-batch-main run-worker-batch-scheduler run-heartbeat run-broker run-rabbit manage run-test-worker version unbound-3.10-github unbound-3.7-github nassl test check autofix integration-tests batch-tests
help:
@echo 'Makefile for internet.nl'
@@ -40,8 +40,9 @@ help:
@echo ' make update_content update the translation files from content repo.'
@echo ' Optional branch=x to use a specific content repo branch.'
@echo ' make frontend (re)generate CSS and Javascript'
- @echo ' make update_padded_macs update padded MAC information'
@echo ' make update_cert_fingerprints update certificate fingerprint information'
+ @echo ' make update_container_documentation update container table for documentation'
+ @echo ' make update_padded_macs update padded MAC information'
@echo ' make update_root_key_file update DNS root key file'
translations:
@@ -73,15 +74,19 @@ update_content:
${DOCKER_COMPOSE_TOOLS_CMD} run --rm tools bin/update_translations.sh
rm -rf tmp/locale_files
-update_padded_macs:
- chmod +x $(MACSDIR)/update-macs.sh
- cd $(MACSDIR); ./update-macs.sh
-
update_cert_fingerprints:
chmod +x $(CERTSSDIR)/update-certs.sh
chmod +x $(CERTSSDIR)/mk-ca-bundle.pl
cd $(CERTSSDIR); ./update-certs.sh
+update_container_documentation:
+ chmod +x bin/update_container_documentation.sh
+ ./bin/update_container_documentation.sh
+
+update_padded_macs:
+ chmod +x $(MACSDIR)/update-macs.sh
+ cd $(MACSDIR); ./update-macs.sh
+
update_root_key_file:
unbound-anchor -a $(DNSDIR)/root.key
diff --git a/bin/update_container_documentation.sh b/bin/update_container_documentation.sh
new file mode 100755
index 000000000..6306119b7
--- /dev/null
+++ b/bin/update_container_documentation.sh
@@ -0,0 +1,10 @@
+#!/bin/env sh
+
+( \
+ echo "# Docker container profiles overview\n\nThis overview was last generated at $(date -uIseconds|sed 's/+00:00/Z/g') with \`make update_container_documentation\`.\n\n"
+ docker run --rm --security-opt=no-new-privileges --cap-drop all --network none -v "$PWD/docker/compose.yaml":"/docker/compose.yaml" \
+ mikefarah/yq:4.45.1 -r '"container|profiles|description","-|-|-",.services|to_entries|map([.key,(.value.profiles //[]|join(", "),(.key|head_comment|split("\n")|join("
")))]|join("|"))[]' /docker/compose.yaml \
+ | sed 's/$/|/' \
+ | column -ts"|" -o" | " \
+ | sed 's/^/| /;s/ $//;2{s/ /-/g}' \
+) > documentation/Docker-container-profiles.md
diff --git a/docker/build.env b/docker/build.env
index dee03808f..c57ee7d18 100644
--- a/docker/build.env
+++ b/docker/build.env
@@ -3,7 +3,7 @@
# include all compose files
COMPOSE_FILE=docker/compose.yaml:docker/compose.build.yaml:docker/compose.development.yaml:docker/compose.integration-tests.yaml:docker/compose.test.yaml:docker/compose.tools.yaml:docker/compose.test-runner-develop.yaml:docker/compose.integration-tests.yaml
# enable all profiles so all containers are built
-COMPOSE_PROFILES=routinator,run-tests
+COMPOSE_PROFILES=monitoring,routinator,run-tests
# don't expose HTTP(S) and DNS ports to the outside, this also causes issues due to being privileged ports
WEBSERVER_PORT=80
diff --git a/docker/compose.yaml b/docker/compose.yaml
index c666f757a..11c2f9656 100644
--- a/docker/compose.yaml
+++ b/docker/compose.yaml
@@ -4,6 +4,7 @@
# Hence no `version` is specified in this file.
services:
+ # nginx proxy container, also runs certbot
webserver:
image: ${DOCKER_IMAGE_WEBSERVER:-${DOCKER_REGISTRY:-ghcr.io/internetstandards}/webserver:${RELEASE}}
restart: unless-stopped
@@ -65,6 +66,7 @@ services:
start_period: 1m
retries: 10
+ # django container
app:
image: ${DOCKER_IMAGE_APP:-${DOCKER_REGISTRY:-ghcr.io/internetstandards}/internet.nl:${RELEASE}}
restart: unless-stopped
@@ -167,6 +169,7 @@ services:
- batch_results:/app/batch_results
- manual-hof:/app/manual-hall-of-fame/
+ # django DB migrations, runs to completion and exits with 0
db-migrate:
image: ${DOCKER_IMAGE_APP:-${DOCKER_REGISTRY:-ghcr.io/internetstandards}/internet.nl:${RELEASE}}
networks:
@@ -335,6 +338,7 @@ services:
command: celery --app internetnl worker --without-gossip --pool=eventlet --time-limit=600 --concurrency=$WORKER_SLOW_CONCURRENCY
--queues slow_db_worker,batch_slow
+ # celery task queue
beat:
image: ${DOCKER_IMAGE_APP:-${DOCKER_REGISTRY:-ghcr.io/internetstandards}/internet.nl:${RELEASE}}
restart: unless-stopped
@@ -408,6 +412,10 @@ services:
start_period: 1m
retries: 10
+ # redis caches state, also used for:
+ # - MAC address lookup
+ # - Django page cache
+ # - client DNS resolver IPs in connectiontest
redis:
image: ${DOCKER_IMAGE_REDIS}
restart: unless-stopped
@@ -431,6 +439,7 @@ services:
volumes:
- redis:/data
+ # rabbitmq message-broker
rabbitmq:
image: ${DOCKER_IMAGE_RABBITMQ}
configs:
@@ -462,6 +471,7 @@ services:
volumes:
- rabbitmq:/var/lib/rabbitmq
+ # database
postgres:
image: ${DOCKER_IMAGE_POSTGRES}
restart: unless-stopped
@@ -489,6 +499,7 @@ services:
volumes:
- postgres:/var/lib/postgresql/data
+ # for RPKI
routinator:
image: ${DOCKER_IMAGE_ROUTINATOR}
restart: unless-stopped
@@ -597,6 +608,7 @@ services:
start_period: 1m
retries: 10
+ # cron with periodic tasks
cron:
image: ${DOCKER_IMAGE_UTIL:-${DOCKER_REGISTRY:-ghcr.io/internetstandards}/util:${RELEASE}}
command: crond -f -d7
@@ -711,6 +723,9 @@ services:
internal:
ipv4_address: $IPV4_IP_GRAFANA_INTERNAL
+ profiles:
+ - monitoring
+
prometheus:
image: ${DOCKER_IMAGE_PROMETHEUS}
@@ -741,6 +756,10 @@ services:
volumes:
- prometheus-data:/prometheus
+ profiles:
+ - monitoring
+
+ # requires monitoring profile
alertmanager:
image: ${DOCKER_IMAGE_PROMETHEUS_ALERTMANAGER}
@@ -783,6 +802,9 @@ services:
networks:
- internal
+ profiles:
+ - monitoring
+
redis-exporter:
image: ${DOCKER_IMAGE_REDIS_EXPORTER}
@@ -800,6 +822,9 @@ services:
networks:
- internal
+ profiles:
+ - monitoring
+
statsd-exporter:
image: ${DOCKER_IMAGE_STATSD_EXPORTER}
@@ -821,6 +846,9 @@ services:
aliases:
- statsd
+ profiles:
+ - monitoring
+
celery-exporter:
image: ${DOCKER_IMAGE_CELERY_EXPORTER}
command:
@@ -842,8 +870,11 @@ services:
rabbitmq:
condition: service_healthy
- # https://github.com/prometheus/node_exporter#docker
+ profiles:
+ - monitoring
+
node-exporter:
+ # https://github.com/prometheus/node_exporter#docker
image: ${DOCKER_IMAGE_NODE_EXPORTER}
command:
- --path.rootfs=/host
@@ -872,6 +903,9 @@ services:
- /var/run/dbus/system_bus_socket:/var/run/dbus/system_bus_socket
- prometheus-textfile-directory:/prometheus-textfile-directory
+ profiles:
+ - monitoring
+
docker_stats_exporter:
# https://github.com/jan4843/docker_stats_exporter
image: ${DOCKER_IMAGE_DOCKER_STATSD_EXPORTER}
@@ -891,6 +925,9 @@ services:
volumes:
- /var/run/docker.sock:/var/run/docker.sock
+ profiles:
+ - monitoring
+
nginx_logs_exporter:
image: ${DOCKER_IMAGE_NGINX_LOGS_EXPORTER}
command:
@@ -912,6 +949,9 @@ services:
volumes:
- nginx-logs-exporter:/var/log/nginx/prometheus-nginxlog-exporter/
+ profiles:
+ - monitoring
+
volumes:
postgres: {}
postgres-backups: {}
diff --git a/docker/defaults.env b/docker/defaults.env
index 572722d4c..01075f829 100644
--- a/docker/defaults.env
+++ b/docker/defaults.env
@@ -121,7 +121,7 @@ INTERNET_NL_CHECK_SUPPORT_RPKI=True
# list of domainnames that can have retry timer be reset via API
INTERNETNL_CACHE_RESET_ALLOWLIST=
-# settings for alertmanager, enable it by adding 'routinator,alertmanager' to COMPOSE_PROFILES in `docker/local.env`
+# settings for alertmanager, enable it by adding 'alertmanager,monitoring,routinator' to COMPOSE_PROFILES in `docker/local.env`
# sending email address used for alerts
ALERTMANAGER_MAIL_FROM=
@@ -146,7 +146,7 @@ COMPOSE_PROJECT_NAME=internetnl
COMPOSE_FILE=docker/compose.yaml
# enable routinator, this profile is omitted in develop.env because it is resource intensive for development environments
-COMPOSE_PROFILES=routinator
+COMPOSE_PROFILES=monitoring,routinator
# interval for batch processing
BATCH_SCHEDULER_INTERVAL=1
diff --git a/docker/develop.env b/docker/develop.env
index 93ba124e9..52cc4508c 100644
--- a/docker/develop.env
+++ b/docker/develop.env
@@ -76,4 +76,4 @@ PAGE_CACHE_TIME_SECONDS=0
# Disable (do not enable) the `routinator` profile which is enable by default in `defaults.env`.
# Routinator is slow to start initially and requires a lot of resources which is not ideal for
# development environments.
-COMPOSE_PROFILES=
+COMPOSE_PROFILES=monitoring
diff --git a/documentation/Docker-container-profiles.md b/documentation/Docker-container-profiles.md
new file mode 100644
index 000000000..d10f15a93
--- /dev/null
+++ b/documentation/Docker-container-profiles.md
@@ -0,0 +1,32 @@
+# Docker container profiles overview
+
+This overview was last generated at 2025-02-09T23:19:39Z with `make update_container_documentation`.
+
+
+| container | profiles | description |
+|-----------------------|--------------|----------------------------------------------------------------------------------------------------------------------------------|
+| webserver | | nginx proxy container, also runs certbot |
+| app | | django container |
+| db-migrate | | django DB migrations, runs to completion and exits with 0 |
+| worker | | |
+| worker-nassl | | worker for queue with potential memory leak |
+| worker-slow | | worker for slow and long running tasks that could require a lot of memory (eg: hof update) |
+| beat | | celery task queue |
+| redis | | redis caches state, also used for:
- MAC address lookup
- Django page cache
- client DNS resolver IPs in connectiontest |
+| rabbitmq | | rabbitmq message-broker |
+| postgres | | database |
+| routinator | routinator | for RPKI |
+| unbound | | unbound DNS server used for connection test |
+| resolver-validating | | unbound resolver used for ldns-dane that require DNSSEC validation |
+| cron | | cron with periodic tasks |
+| cron-docker | | cron daemon with access to Docker socket but no networking |
+| grafana | monitoring | |
+| prometheus | monitoring | |
+| alertmanager | alertmanager | requires monitoring profile |
+| postgresql-exporter | monitoring | |
+| redis-exporter | monitoring | |
+| statsd-exporter | monitoring | |
+| celery-exporter | monitoring | |
+| node-exporter | monitoring | |
+| docker_stats_exporter | monitoring | |
+| nginx_logs_exporter | monitoring | |
diff --git a/documentation/Docker-deployment.md b/documentation/Docker-deployment.md
index 21de48b63..57bd49771 100644
--- a/documentation/Docker-deployment.md
+++ b/documentation/Docker-deployment.md
@@ -390,7 +390,7 @@ A Prometheus Alertmanager service is available but disabled by default. Enabling
To enable and configure the Alertmanager add the following lines to `docker/local.env` and adjust the values to be applicable for your environment:
- COMPOSE_PROFILES=routinator,alertmanager
+ COMPOSE_PROFILES=alertmanager,monitoring,routinator
ALERTMANAGER_MAIL_TO=rcpt1@example.com,rcpt2@example.com
ALERTMANAGER_MAIL_FROM=noreply@example.com
ALERTMANAGER_SMTP_HOST=smtp.example.com
diff --git a/documentation/github_release_steps.md b/documentation/github_release_steps.md
index 9197d0d4e..fc5874d79 100644
--- a/documentation/github_release_steps.md
+++ b/documentation/github_release_steps.md
@@ -9,6 +9,7 @@ code changes for the next release are already in the main branch.
3. Run all the update scripts:
```
make update_cert_fingerprints
+ make update_container_documentation
make update_padded_macs
make update_root_key_file
make translate_content_to_main