From 120c9f938a322213aae9e5ac6ffe52c50a07e8c5 Mon Sep 17 00:00:00 2001 From: "Benjamin W. Broersma" Date: Tue, 28 Jan 2025 22:51:00 +0100 Subject: [PATCH] Add update_container_documentation script and monitoring profile Fixes #1588 --- Makefile | 17 +++++---- bin/update_container_documentation.sh | 9 +++++ docker/compose.yaml | 42 +++++++++++++++++++++- documentation/Docker-container-profiles.md | 32 +++++++++++++++++ 4 files changed, 93 insertions(+), 7 deletions(-) create mode 100755 bin/update_container_documentation.sh create mode 100644 documentation/Docker-container-profiles.md diff --git a/Makefile b/Makefile index 6e9e7cc08..9b72ed779 100644 --- a/Makefile +++ b/Makefile @@ -31,7 +31,7 @@ pysrc = $(shell find ${pysrcdirs} -name \*.py) bin = .venv/bin _env ?= env PATH="${bin}:$$PATH" -.PHONY: translations translations_tar frontend update_padded_macs update_cert_fingerprints update_root_key_file venv frontend clean clen_venv pip-compile pip-upgrade pip-upgrade-package pip-install run run-worker run-worker-batch-callback run-worker-batch-main run-worker-batch-scheduler run-heartbeat run-broker run-rabbit manage run-test-worker version unbound-3.10-github unbound-3.7-github nassl test check autofix integration-tests batch-tests +.PHONY: translations translations_tar frontend update_cert_fingerprints update_container_documentation update_padded_macs update_root_key_file venv frontend clean clen_venv pip-compile pip-upgrade pip-upgrade-package pip-install run run-worker run-worker-batch-callback run-worker-batch-main run-worker-batch-scheduler run-heartbeat run-broker run-rabbit manage run-test-worker version unbound-3.10-github unbound-3.7-github nassl test check autofix integration-tests batch-tests help: @echo 'Makefile for internet.nl' @@ -40,8 +40,9 @@ help: @echo ' make update_content update the translation files from content repo.' @echo ' Optional branch=x to use a specific content repo branch.' @echo ' make frontend (re)generate CSS and Javascript' - @echo ' make update_padded_macs update padded MAC information' @echo ' make update_cert_fingerprints update certificate fingerprint information' + @echo ' make update_container_documentation update container table for documentation' + @echo ' make update_padded_macs update padded MAC information' @echo ' make update_root_key_file update DNS root key file' translations: @@ -73,15 +74,19 @@ update_content: ${DOCKER_COMPOSE_TOOLS_CMD} run --rm tools bin/update_translations.sh rm -rf tmp/locale_files -update_padded_macs: - chmod +x $(MACSDIR)/update-macs.sh - cd $(MACSDIR); ./update-macs.sh - update_cert_fingerprints: chmod +x $(CERTSSDIR)/update-certs.sh chmod +x $(CERTSSDIR)/mk-ca-bundle.pl cd $(CERTSSDIR); ./update-certs.sh +update_container_documentation: + chmod +x bin/update_container_documentation.sh + ./bin/update_container_documentation.sh + +update_padded_macs: + chmod +x $(MACSDIR)/update-macs.sh + cd $(MACSDIR); ./update-macs.sh + update_root_key_file: unbound-anchor -a $(DNSDIR)/root.key diff --git a/bin/update_container_documentation.sh b/bin/update_container_documentation.sh new file mode 100755 index 000000000..b24eeaa96 --- /dev/null +++ b/bin/update_container_documentation.sh @@ -0,0 +1,9 @@ +#!/bin/env sh + +( \ + echo "# Docker container profiles overview\n\nThis overview was last generated at $(date -uIseconds|sed 's/+00:00/Z/g') with \`make update_container_documentation\`.\n\n" + docker run --rm --security-opt=no-new-privileges --cap-drop all --network none -v "$PWD/docker/compose.yaml":"/docker/compose.yaml" mikefarah/yq:4.45.1 -r '"container|profiles|description","-|-|-",.services|to_entries|map([.key,(.value.profiles //[]|join(", "),(.key|head_comment|split("\n")|join("
")))]|join("|"))[]' /docker/compose.yaml \ + | sed 's/$/|/' \ + | column -ts\| -o" | " -W3 -c240 \ + | sed 's/^/| /;s/ $//;2{s/ /-/g}' \ +) > documentation/Docker-container-profiles.md diff --git a/docker/compose.yaml b/docker/compose.yaml index c666f757a..11c2f9656 100644 --- a/docker/compose.yaml +++ b/docker/compose.yaml @@ -4,6 +4,7 @@ # Hence no `version` is specified in this file. services: + # nginx proxy container, also runs certbot webserver: image: ${DOCKER_IMAGE_WEBSERVER:-${DOCKER_REGISTRY:-ghcr.io/internetstandards}/webserver:${RELEASE}} restart: unless-stopped @@ -65,6 +66,7 @@ services: start_period: 1m retries: 10 + # django container app: image: ${DOCKER_IMAGE_APP:-${DOCKER_REGISTRY:-ghcr.io/internetstandards}/internet.nl:${RELEASE}} restart: unless-stopped @@ -167,6 +169,7 @@ services: - batch_results:/app/batch_results - manual-hof:/app/manual-hall-of-fame/ + # django DB migrations, runs to completion and exits with 0 db-migrate: image: ${DOCKER_IMAGE_APP:-${DOCKER_REGISTRY:-ghcr.io/internetstandards}/internet.nl:${RELEASE}} networks: @@ -335,6 +338,7 @@ services: command: celery --app internetnl worker --without-gossip --pool=eventlet --time-limit=600 --concurrency=$WORKER_SLOW_CONCURRENCY --queues slow_db_worker,batch_slow + # celery task queue beat: image: ${DOCKER_IMAGE_APP:-${DOCKER_REGISTRY:-ghcr.io/internetstandards}/internet.nl:${RELEASE}} restart: unless-stopped @@ -408,6 +412,10 @@ services: start_period: 1m retries: 10 + # redis caches state, also used for: + # - MAC address lookup + # - Django page cache + # - client DNS resolver IPs in connectiontest redis: image: ${DOCKER_IMAGE_REDIS} restart: unless-stopped @@ -431,6 +439,7 @@ services: volumes: - redis:/data + # rabbitmq message-broker rabbitmq: image: ${DOCKER_IMAGE_RABBITMQ} configs: @@ -462,6 +471,7 @@ services: volumes: - rabbitmq:/var/lib/rabbitmq + # database postgres: image: ${DOCKER_IMAGE_POSTGRES} restart: unless-stopped @@ -489,6 +499,7 @@ services: volumes: - postgres:/var/lib/postgresql/data + # for RPKI routinator: image: ${DOCKER_IMAGE_ROUTINATOR} restart: unless-stopped @@ -597,6 +608,7 @@ services: start_period: 1m retries: 10 + # cron with periodic tasks cron: image: ${DOCKER_IMAGE_UTIL:-${DOCKER_REGISTRY:-ghcr.io/internetstandards}/util:${RELEASE}} command: crond -f -d7 @@ -711,6 +723,9 @@ services: internal: ipv4_address: $IPV4_IP_GRAFANA_INTERNAL + profiles: + - monitoring + prometheus: image: ${DOCKER_IMAGE_PROMETHEUS} @@ -741,6 +756,10 @@ services: volumes: - prometheus-data:/prometheus + profiles: + - monitoring + + # requires monitoring profile alertmanager: image: ${DOCKER_IMAGE_PROMETHEUS_ALERTMANAGER} @@ -783,6 +802,9 @@ services: networks: - internal + profiles: + - monitoring + redis-exporter: image: ${DOCKER_IMAGE_REDIS_EXPORTER} @@ -800,6 +822,9 @@ services: networks: - internal + profiles: + - monitoring + statsd-exporter: image: ${DOCKER_IMAGE_STATSD_EXPORTER} @@ -821,6 +846,9 @@ services: aliases: - statsd + profiles: + - monitoring + celery-exporter: image: ${DOCKER_IMAGE_CELERY_EXPORTER} command: @@ -842,8 +870,11 @@ services: rabbitmq: condition: service_healthy - # https://github.com/prometheus/node_exporter#docker + profiles: + - monitoring + node-exporter: + # https://github.com/prometheus/node_exporter#docker image: ${DOCKER_IMAGE_NODE_EXPORTER} command: - --path.rootfs=/host @@ -872,6 +903,9 @@ services: - /var/run/dbus/system_bus_socket:/var/run/dbus/system_bus_socket - prometheus-textfile-directory:/prometheus-textfile-directory + profiles: + - monitoring + docker_stats_exporter: # https://github.com/jan4843/docker_stats_exporter image: ${DOCKER_IMAGE_DOCKER_STATSD_EXPORTER} @@ -891,6 +925,9 @@ services: volumes: - /var/run/docker.sock:/var/run/docker.sock + profiles: + - monitoring + nginx_logs_exporter: image: ${DOCKER_IMAGE_NGINX_LOGS_EXPORTER} command: @@ -912,6 +949,9 @@ services: volumes: - nginx-logs-exporter:/var/log/nginx/prometheus-nginxlog-exporter/ + profiles: + - monitoring + volumes: postgres: {} postgres-backups: {} diff --git a/documentation/Docker-container-profiles.md b/documentation/Docker-container-profiles.md new file mode 100644 index 000000000..d10f15a93 --- /dev/null +++ b/documentation/Docker-container-profiles.md @@ -0,0 +1,32 @@ +# Docker container profiles overview + +This overview was last generated at 2025-02-09T23:19:39Z with `make update_container_documentation`. + + +| container | profiles | description | +|-----------------------|--------------|----------------------------------------------------------------------------------------------------------------------------------| +| webserver | | nginx proxy container, also runs certbot | +| app | | django container | +| db-migrate | | django DB migrations, runs to completion and exits with 0 | +| worker | | | +| worker-nassl | | worker for queue with potential memory leak | +| worker-slow | | worker for slow and long running tasks that could require a lot of memory (eg: hof update) | +| beat | | celery task queue | +| redis | | redis caches state, also used for:
- MAC address lookup
- Django page cache
- client DNS resolver IPs in connectiontest | +| rabbitmq | | rabbitmq message-broker | +| postgres | | database | +| routinator | routinator | for RPKI | +| unbound | | unbound DNS server used for connection test | +| resolver-validating | | unbound resolver used for ldns-dane that require DNSSEC validation | +| cron | | cron with periodic tasks | +| cron-docker | | cron daemon with access to Docker socket but no networking | +| grafana | monitoring | | +| prometheus | monitoring | | +| alertmanager | alertmanager | requires monitoring profile | +| postgresql-exporter | monitoring | | +| redis-exporter | monitoring | | +| statsd-exporter | monitoring | | +| celery-exporter | monitoring | | +| node-exporter | monitoring | | +| docker_stats_exporter | monitoring | | +| nginx_logs_exporter | monitoring | |