From 667d7daee4b2ad040656fd71398cc0c5bbee1cd7 Mon Sep 17 00:00:00 2001 From: "Benjamin W. Broersma" Date: Tue, 28 Jan 2025 22:51:00 +0100 Subject: [PATCH 1/2] Add update_container_documentation script and monitoring profile Fixes #1588 --- Makefile | 17 +++++---- bin/update_container_documentation.sh | 10 ++++++ docker/build.env | 2 +- docker/compose.yaml | 42 +++++++++++++++++++++- docker/defaults.env | 4 +-- docker/develop.env | 2 +- documentation/Docker-container-profiles.md | 32 +++++++++++++++++ documentation/Docker-deployment.md | 2 +- 8 files changed, 99 insertions(+), 12 deletions(-) create mode 100755 bin/update_container_documentation.sh create mode 100644 documentation/Docker-container-profiles.md diff --git a/Makefile b/Makefile index 6e9e7cc08..9b72ed779 100644 --- a/Makefile +++ b/Makefile @@ -31,7 +31,7 @@ pysrc = $(shell find ${pysrcdirs} -name \*.py) bin = .venv/bin _env ?= env PATH="${bin}:$$PATH" -.PHONY: translations translations_tar frontend update_padded_macs update_cert_fingerprints update_root_key_file venv frontend clean clen_venv pip-compile pip-upgrade pip-upgrade-package pip-install run run-worker run-worker-batch-callback run-worker-batch-main run-worker-batch-scheduler run-heartbeat run-broker run-rabbit manage run-test-worker version unbound-3.10-github unbound-3.7-github nassl test check autofix integration-tests batch-tests +.PHONY: translations translations_tar frontend update_cert_fingerprints update_container_documentation update_padded_macs update_root_key_file venv frontend clean clen_venv pip-compile pip-upgrade pip-upgrade-package pip-install run run-worker run-worker-batch-callback run-worker-batch-main run-worker-batch-scheduler run-heartbeat run-broker run-rabbit manage run-test-worker version unbound-3.10-github unbound-3.7-github nassl test check autofix integration-tests batch-tests help: @echo 'Makefile for internet.nl' @@ -40,8 +40,9 @@ help: @echo ' make update_content update the translation files from content repo.' @echo ' Optional branch=x to use a specific content repo branch.' @echo ' make frontend (re)generate CSS and Javascript' - @echo ' make update_padded_macs update padded MAC information' @echo ' make update_cert_fingerprints update certificate fingerprint information' + @echo ' make update_container_documentation update container table for documentation' + @echo ' make update_padded_macs update padded MAC information' @echo ' make update_root_key_file update DNS root key file' translations: @@ -73,15 +74,19 @@ update_content: ${DOCKER_COMPOSE_TOOLS_CMD} run --rm tools bin/update_translations.sh rm -rf tmp/locale_files -update_padded_macs: - chmod +x $(MACSDIR)/update-macs.sh - cd $(MACSDIR); ./update-macs.sh - update_cert_fingerprints: chmod +x $(CERTSSDIR)/update-certs.sh chmod +x $(CERTSSDIR)/mk-ca-bundle.pl cd $(CERTSSDIR); ./update-certs.sh +update_container_documentation: + chmod +x bin/update_container_documentation.sh + ./bin/update_container_documentation.sh + +update_padded_macs: + chmod +x $(MACSDIR)/update-macs.sh + cd $(MACSDIR); ./update-macs.sh + update_root_key_file: unbound-anchor -a $(DNSDIR)/root.key diff --git a/bin/update_container_documentation.sh b/bin/update_container_documentation.sh new file mode 100755 index 000000000..6306119b7 --- /dev/null +++ b/bin/update_container_documentation.sh @@ -0,0 +1,10 @@ +#!/bin/env sh + +( \ + echo "# Docker container profiles overview\n\nThis overview was last generated at $(date -uIseconds|sed 's/+00:00/Z/g') with \`make update_container_documentation\`.\n\n" + docker run --rm --security-opt=no-new-privileges --cap-drop all --network none -v "$PWD/docker/compose.yaml":"/docker/compose.yaml" \ + mikefarah/yq:4.45.1 -r '"container|profiles|description","-|-|-",.services|to_entries|map([.key,(.value.profiles //[]|join(", "),(.key|head_comment|split("\n")|join("
")))]|join("|"))[]' /docker/compose.yaml \ + | sed 's/$/|/' \ + | column -ts"|" -o" | " \ + | sed 's/^/| /;s/ $//;2{s/ /-/g}' \ +) > documentation/Docker-container-profiles.md diff --git a/docker/build.env b/docker/build.env index dee03808f..c57ee7d18 100644 --- a/docker/build.env +++ b/docker/build.env @@ -3,7 +3,7 @@ # include all compose files COMPOSE_FILE=docker/compose.yaml:docker/compose.build.yaml:docker/compose.development.yaml:docker/compose.integration-tests.yaml:docker/compose.test.yaml:docker/compose.tools.yaml:docker/compose.test-runner-develop.yaml:docker/compose.integration-tests.yaml # enable all profiles so all containers are built -COMPOSE_PROFILES=routinator,run-tests +COMPOSE_PROFILES=monitoring,routinator,run-tests # don't expose HTTP(S) and DNS ports to the outside, this also causes issues due to being privileged ports WEBSERVER_PORT=80 diff --git a/docker/compose.yaml b/docker/compose.yaml index c666f757a..11c2f9656 100644 --- a/docker/compose.yaml +++ b/docker/compose.yaml @@ -4,6 +4,7 @@ # Hence no `version` is specified in this file. services: + # nginx proxy container, also runs certbot webserver: image: ${DOCKER_IMAGE_WEBSERVER:-${DOCKER_REGISTRY:-ghcr.io/internetstandards}/webserver:${RELEASE}} restart: unless-stopped @@ -65,6 +66,7 @@ services: start_period: 1m retries: 10 + # django container app: image: ${DOCKER_IMAGE_APP:-${DOCKER_REGISTRY:-ghcr.io/internetstandards}/internet.nl:${RELEASE}} restart: unless-stopped @@ -167,6 +169,7 @@ services: - batch_results:/app/batch_results - manual-hof:/app/manual-hall-of-fame/ + # django DB migrations, runs to completion and exits with 0 db-migrate: image: ${DOCKER_IMAGE_APP:-${DOCKER_REGISTRY:-ghcr.io/internetstandards}/internet.nl:${RELEASE}} networks: @@ -335,6 +338,7 @@ services: command: celery --app internetnl worker --without-gossip --pool=eventlet --time-limit=600 --concurrency=$WORKER_SLOW_CONCURRENCY --queues slow_db_worker,batch_slow + # celery task queue beat: image: ${DOCKER_IMAGE_APP:-${DOCKER_REGISTRY:-ghcr.io/internetstandards}/internet.nl:${RELEASE}} restart: unless-stopped @@ -408,6 +412,10 @@ services: start_period: 1m retries: 10 + # redis caches state, also used for: + # - MAC address lookup + # - Django page cache + # - client DNS resolver IPs in connectiontest redis: image: ${DOCKER_IMAGE_REDIS} restart: unless-stopped @@ -431,6 +439,7 @@ services: volumes: - redis:/data + # rabbitmq message-broker rabbitmq: image: ${DOCKER_IMAGE_RABBITMQ} configs: @@ -462,6 +471,7 @@ services: volumes: - rabbitmq:/var/lib/rabbitmq + # database postgres: image: ${DOCKER_IMAGE_POSTGRES} restart: unless-stopped @@ -489,6 +499,7 @@ services: volumes: - postgres:/var/lib/postgresql/data + # for RPKI routinator: image: ${DOCKER_IMAGE_ROUTINATOR} restart: unless-stopped @@ -597,6 +608,7 @@ services: start_period: 1m retries: 10 + # cron with periodic tasks cron: image: ${DOCKER_IMAGE_UTIL:-${DOCKER_REGISTRY:-ghcr.io/internetstandards}/util:${RELEASE}} command: crond -f -d7 @@ -711,6 +723,9 @@ services: internal: ipv4_address: $IPV4_IP_GRAFANA_INTERNAL + profiles: + - monitoring + prometheus: image: ${DOCKER_IMAGE_PROMETHEUS} @@ -741,6 +756,10 @@ services: volumes: - prometheus-data:/prometheus + profiles: + - monitoring + + # requires monitoring profile alertmanager: image: ${DOCKER_IMAGE_PROMETHEUS_ALERTMANAGER} @@ -783,6 +802,9 @@ services: networks: - internal + profiles: + - monitoring + redis-exporter: image: ${DOCKER_IMAGE_REDIS_EXPORTER} @@ -800,6 +822,9 @@ services: networks: - internal + profiles: + - monitoring + statsd-exporter: image: ${DOCKER_IMAGE_STATSD_EXPORTER} @@ -821,6 +846,9 @@ services: aliases: - statsd + profiles: + - monitoring + celery-exporter: image: ${DOCKER_IMAGE_CELERY_EXPORTER} command: @@ -842,8 +870,11 @@ services: rabbitmq: condition: service_healthy - # https://github.com/prometheus/node_exporter#docker + profiles: + - monitoring + node-exporter: + # https://github.com/prometheus/node_exporter#docker image: ${DOCKER_IMAGE_NODE_EXPORTER} command: - --path.rootfs=/host @@ -872,6 +903,9 @@ services: - /var/run/dbus/system_bus_socket:/var/run/dbus/system_bus_socket - prometheus-textfile-directory:/prometheus-textfile-directory + profiles: + - monitoring + docker_stats_exporter: # https://github.com/jan4843/docker_stats_exporter image: ${DOCKER_IMAGE_DOCKER_STATSD_EXPORTER} @@ -891,6 +925,9 @@ services: volumes: - /var/run/docker.sock:/var/run/docker.sock + profiles: + - monitoring + nginx_logs_exporter: image: ${DOCKER_IMAGE_NGINX_LOGS_EXPORTER} command: @@ -912,6 +949,9 @@ services: volumes: - nginx-logs-exporter:/var/log/nginx/prometheus-nginxlog-exporter/ + profiles: + - monitoring + volumes: postgres: {} postgres-backups: {} diff --git a/docker/defaults.env b/docker/defaults.env index 572722d4c..01075f829 100644 --- a/docker/defaults.env +++ b/docker/defaults.env @@ -121,7 +121,7 @@ INTERNET_NL_CHECK_SUPPORT_RPKI=True # list of domainnames that can have retry timer be reset via API INTERNETNL_CACHE_RESET_ALLOWLIST= -# settings for alertmanager, enable it by adding 'routinator,alertmanager' to COMPOSE_PROFILES in `docker/local.env` +# settings for alertmanager, enable it by adding 'alertmanager,monitoring,routinator' to COMPOSE_PROFILES in `docker/local.env` # sending email address used for alerts ALERTMANAGER_MAIL_FROM= @@ -146,7 +146,7 @@ COMPOSE_PROJECT_NAME=internetnl COMPOSE_FILE=docker/compose.yaml # enable routinator, this profile is omitted in develop.env because it is resource intensive for development environments -COMPOSE_PROFILES=routinator +COMPOSE_PROFILES=monitoring,routinator # interval for batch processing BATCH_SCHEDULER_INTERVAL=1 diff --git a/docker/develop.env b/docker/develop.env index 93ba124e9..52cc4508c 100644 --- a/docker/develop.env +++ b/docker/develop.env @@ -76,4 +76,4 @@ PAGE_CACHE_TIME_SECONDS=0 # Disable (do not enable) the `routinator` profile which is enable by default in `defaults.env`. # Routinator is slow to start initially and requires a lot of resources which is not ideal for # development environments. -COMPOSE_PROFILES= +COMPOSE_PROFILES=monitoring diff --git a/documentation/Docker-container-profiles.md b/documentation/Docker-container-profiles.md new file mode 100644 index 000000000..d10f15a93 --- /dev/null +++ b/documentation/Docker-container-profiles.md @@ -0,0 +1,32 @@ +# Docker container profiles overview + +This overview was last generated at 2025-02-09T23:19:39Z with `make update_container_documentation`. + + +| container | profiles | description | +|-----------------------|--------------|----------------------------------------------------------------------------------------------------------------------------------| +| webserver | | nginx proxy container, also runs certbot | +| app | | django container | +| db-migrate | | django DB migrations, runs to completion and exits with 0 | +| worker | | | +| worker-nassl | | worker for queue with potential memory leak | +| worker-slow | | worker for slow and long running tasks that could require a lot of memory (eg: hof update) | +| beat | | celery task queue | +| redis | | redis caches state, also used for:
- MAC address lookup
- Django page cache
- client DNS resolver IPs in connectiontest | +| rabbitmq | | rabbitmq message-broker | +| postgres | | database | +| routinator | routinator | for RPKI | +| unbound | | unbound DNS server used for connection test | +| resolver-validating | | unbound resolver used for ldns-dane that require DNSSEC validation | +| cron | | cron with periodic tasks | +| cron-docker | | cron daemon with access to Docker socket but no networking | +| grafana | monitoring | | +| prometheus | monitoring | | +| alertmanager | alertmanager | requires monitoring profile | +| postgresql-exporter | monitoring | | +| redis-exporter | monitoring | | +| statsd-exporter | monitoring | | +| celery-exporter | monitoring | | +| node-exporter | monitoring | | +| docker_stats_exporter | monitoring | | +| nginx_logs_exporter | monitoring | | diff --git a/documentation/Docker-deployment.md b/documentation/Docker-deployment.md index 21de48b63..57bd49771 100644 --- a/documentation/Docker-deployment.md +++ b/documentation/Docker-deployment.md @@ -390,7 +390,7 @@ A Prometheus Alertmanager service is available but disabled by default. Enabling To enable and configure the Alertmanager add the following lines to `docker/local.env` and adjust the values to be applicable for your environment: - COMPOSE_PROFILES=routinator,alertmanager + COMPOSE_PROFILES=alertmanager,monitoring,routinator ALERTMANAGER_MAIL_TO=rcpt1@example.com,rcpt2@example.com ALERTMANAGER_MAIL_FROM=noreply@example.com ALERTMANAGER_SMTP_HOST=smtp.example.com From cb0564092d26aff52c27f2b266aaa5a3384899ba Mon Sep 17 00:00:00 2001 From: "Benjamin W. Broersma" Date: Tue, 11 Feb 2025 22:03:23 +0100 Subject: [PATCH 2/2] Add update_container_documentation mention in github release steps --- documentation/github_release_steps.md | 1 + 1 file changed, 1 insertion(+) diff --git a/documentation/github_release_steps.md b/documentation/github_release_steps.md index 9197d0d4e..fc5874d79 100644 --- a/documentation/github_release_steps.md +++ b/documentation/github_release_steps.md @@ -9,6 +9,7 @@ code changes for the next release are already in the main branch. 3. Run all the update scripts: ``` make update_cert_fingerprints + make update_container_documentation make update_padded_macs make update_root_key_file make translate_content_to_main