Add let's encrypt for cron renew SSL certificate, modified default_ssl.conf for challenging let's encrypt, added support nginx to use basic authentication, tested with AWS docker environment. Fix run_pylint.sh and move pylintrc to app. Fix CI/CD. Moved config_default to app for usage w/o docker.

Ubuntu · bkueng · commit a6936c31009b · 2021-03-26T10:30:58.000+01:00
diff --git a/README.md b/README.md
@@ -140,11 +140,13 @@ the root directory (Eg `tornado_handlers.py`). Then to make sure the same module
 is only loaded once, we use `import xy` instead of `import plot_app.xy`.
 It's useful to look at `print('\n'.join(sys.modules.keys()))` to check this.
 
-# Description
+# Docker usage
 
 This section explain about how to work with docker.
 
-# Arguments
+## Arguments
+
+Edit the `.env` file according to your setup:
 
 - PORT - The number of port, what listen service in docker, default 5006
 - USE_PROXY - The set his, if you use reverse proxy (Nginx, ...)
@@ -188,6 +190,8 @@ Remember to Change NGINX_CONF to use default_ssl.conf and add the EMAIL for prod
 
 ### Production
 ```bash
+htpasswd -c ./nginx/.htpasswd username
+# here to create a .htpasswd for nginx basic authentication
 chmod u+x init-letsencrypt.sh
 ./init-letsencrypt.sh
 ```
diff --git a/app/config_default.ini b/app/config_default.ini
@@ -5,7 +5,7 @@ domain_name = review.px4.io
 http_protocol = http
 
 # path for everything that will be stored on disk (including the database)
-storage_path = data
+storage_path = ../data
 
 # DB file name (if empty, $storage_path/logs.sqlite is used)
 db_filename =
diff --git a/app/plot_app/plotted_tables.py b/app/plot_app/plotted_tables.py
@@ -481,7 +481,7 @@ def get_changed_parameters(ulog, plot_width):
                     is_default = abs(float(system_default) - float(param_value)) < 0.00001
                     if 'decimal' in default_param:
                         param_value = round(param_value, int(default_param['decimal']))
-                        airframe_default = round(float(airframe_default), int(default_param['decimal']))
+                        airframe_default = round(float(airframe_default), int(default_param['decimal'])) #pylint: disable=line-too-long
                 else:
                     is_default = int(system_default) == int(param_value)
                 if not is_default:
diff --git a/app/pylintrc b/app/pylintrc
diff --git a/app/serve.py b/app/serve.py
@@ -13,10 +13,11 @@
 from bokeh.server.server import Server
 from bokeh.application.handlers import DirectoryHandler
 
-from tornado.web import StaticFileHandler
+
 
 # this is needed for the following imports
 sys.path.append(os.path.join(os.path.dirname(os.path.realpath(__file__)), 'plot_app'))
+from tornado.web import StaticFileHandler
 from tornado.web import RedirectHandler
 from tornado_handlers.download import DownloadHandler
 from tornado_handlers.upload import UploadHandler
@@ -27,8 +28,8 @@
 from tornado_handlers.radio_controller import RadioControllerHandler
 from tornado_handlers.error_labels import UpdateErrorLabelHandler
 
-from helper import set_log_id_is_filename, print_cache_info
-from config import debug_print_timing, get_overview_img_filepath
+from helper import set_log_id_is_filename, print_cache_info #pylint: disable=C0411
+from config import debug_print_timing, get_overview_img_filepath #pylint: disable=C0411
 
 #pylint: disable=invalid-name
 
diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
@@ -9,6 +9,6 @@ services:
      - 5006:5006
     volumes:
       - ./app:/opt/service/
-      - ./data:/opt/service/data
-      - ${PWD}/config_default.ini:/opt/service/config_default.ini:ro # Absolute for volume a file.
+      - ./data:/opt/data
+      - ${PWD}/app/config_default.ini:/opt/service/config_default.ini:ro # Absolute for volume a file.
     restart: always
diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml
@@ -5,42 +5,45 @@ services:
     build:
       context: ./app
       dockerfile: Dockerfile
-    env_file: .env_prod
+    env_file: .env
     volumes:
-      - ./data:/opt/service/data
+      - ./data:/opt/data
       # Absolute for volume a file.
-      - ${PWD}/config_default.ini:/opt/service/config_default.ini:ro
-      - ${PWD}/config_user.ini:/opt/service/config_user.ini:ro
-    restart: always
-  certbot:
-    image: certbot/certbot
-    volumes:
-      - ./data/certbot/conf:/etc/letsencrypt
-      - ./data/certbot/www:/var/www/certbot
-      - ./logs/letsencrypt:/var/log/letsencrypt
-      - /etc/localtime:/etc/localtime:ro # for synchronize with host timezone
-    entrypoint: "/bin/sh -c 'trap exit TERM; while :; do certbot renew; sleep 12h & wait $${!}; done;'"
+      - ${PWD}/app/config_default.ini:/opt/service/config_default.ini:ro
+      - ${PWD}/app/config_user.ini:/opt/service/config_user.ini:ro
     restart: always
   nginx:
     build:
       context: ./nginx
       dockerfile: Dockerfile
       args:
         - NGINX_CONF=${NGINX_CONF}
-    env_file: .env_prod
+    env_file: .env
     ports:
       - 80:80
       - 443:443
     volumes:
       - ./data/certbot/conf:/etc/letsencrypt
       - ./data/certbot/www:/var/www/certbot
       - ./logs/nginx:/var/log/nginx/
+      - ${PWD}/nginx/.htpasswd:/etc/nginx/.htpasswd # for nginx basic authentication
       - /etc/localtime:/etc/localtime:ro # for synchronize with host timezone
     command: "/bin/sh -c 'while :; do sleep 6h & wait $${!}; nginx -s reload; done & nginx -g \"daemon off;\"'"
     links:
       - flight_review_app
     depends_on:
       - flight_review_app
     restart: always
-
+  certbot:
+    build:
+      context: ./letsencrypt
+      dockerfile: Dockerfile
+    volumes:
+      - ./data/certbot/conf:/etc/letsencrypt
+      - ./data/certbot/www:/var/www/certbot
+      - ./logs/letsencrypt:/var/log/letsencrypt
+      - /etc/localtime:/etc/localtime:ro # for synchronize with host timezone
+    entrypoint: "/bin/sh -c 'trap exit TERM; crond -f'"
+    # tty: true
+    restart: always
 
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -7,8 +7,8 @@ services:
       dockerfile: Dockerfile
     env_file: .env
     volumes:
-      - ./data:/opt/service/data
-      - ${PWD}/config_default.ini:/opt/service/config_default.ini:ro # Absolute for volume a file.
+      - ./data:/opt/data
+      - ${PWD}/app/config_default.ini:/opt/service/config_default.ini:ro # Absolute for volume a file.
     restart: always
   nginx:
     build:
diff --git a/init-letsencrypt.sh b/init-letsencrypt.sh
@@ -5,9 +5,9 @@ if ! [ -x "$(command -v docker-compose -f docker-compose.prod.yml)" ]; then
   exit 1
 fi
 
-export $(cat .env_prod | grep -v ^\# | xargs); # get variable from .env file
+. .env # get variable from .env file
 
-domain=(${DOMAIN} www.${DOMAIN})
+domain=${DOMAIN} # www.${DOMAIN})
 rsa_key_size=4096
 cert_path=${CERT_PATH}
 email=${EMAIL} # Adding a valid address is strongly recommended
@@ -21,6 +21,7 @@ if [ -d "$cert_path" ]; then
 fi
 
 
+# Download TLS parameters
 if [ ! -e "$cert_path/conf/options-ssl-nginx.conf" ] || [ ! -e "$cert_path/conf/ssl-dhparams.pem" ]; then
   echo "### Downloading recommended TLS parameters ..."
   mkdir -p "$cert_path/conf"
@@ -80,3 +81,5 @@ echo
 
 echo "### Reloading nginx ..."
 docker-compose -f docker-compose.prod.yml exec nginx nginx -s reload
+docker-compose -f docker-compose.prod.yml build certbot
+docker-compose -f docker-compose.prod.yml up -d
diff --git a/letsencrypt/Dockerfile b/letsencrypt/Dockerfile
@@ -0,0 +1,2 @@
+FROM certbot/certbot
+ADD letsencrypt-renew /var/spool/cron/crontabs/root
diff --git a/letsencrypt/letsencrypt-renew b/letsencrypt/letsencrypt-renew
@@ -0,0 +1,2 @@
+# renew every week
+0 12 * * */1 certbot renew
diff --git a/nginx/Dockerfile b/nginx/Dockerfile
@@ -1,3 +1,4 @@
 FROM nginx
 ARG NGINX_CONF
+RUN echo ${NGINX_CONF}
 COPY ./${NGINX_CONF} /etc/nginx/conf.d/default.conf
diff --git a/nginx/default_ssl.conf b/nginx/default_ssl.conf
@@ -5,34 +5,40 @@ upstream flight_review_app {
 
 server {
     listen 80;
-    server_name example.org;
+    server_name review.px4.io;
+    location ^~ /.well-known/acme-challenge/ {
+        root /var/www/certbot;
+        allow all;
+    }
     location / {
-        return 301 https://$host$request_uri;
-    }    
+        return 307 https://$host$request_uri;
+    }
 }
 
 server {
-    listen 443 ssl;
-    server_name example.org;
+    listen 443 ssl http2;
+    server_name review.px4.io;
     access_log  /var/log/nginx/access.log;
 	error_log  /var/log/nginx/error.log;
-	ssl_certificate /etc/letsencrypt/live/example.org/fullchain.pem;
-	ssl_certificate_key /etc/letsencrypt/live/example.org/privkey.pem;
+	ssl_certificate /etc/letsencrypt/live/review.px4.io/fullchain.pem;
+	ssl_certificate_key /etc/letsencrypt/live/review.px4.io/privkey.pem;
     include /etc/letsencrypt/options-ssl-nginx.conf;
     ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem;
+    proxy_connect_timeout   180s;
+    proxy_read_timeout        180s;
+    proxy_send_timeout        180s;
+    charset utf-8;
 
     client_max_body_size 100M;
 	location / {
+        proxy_request_buffering off;
         proxy_pass http://flight_review_app;
         proxy_set_header Upgrade $http_upgrade;
         proxy_set_header Connection "upgrade";
         proxy_http_version 1.1;
         proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-        proxy_set_header Host $host;#:$server_port;
-        proxy_buffering off;
+        proxy_set_header Host $host; #:$server_port;
+        auth_basic "Restricted";                   # message to show when authentication error
+        auth_basic_user_file /etc/nginx/.htpasswd; # .htpasswd path
     }
-
-	location /.well-known/acme-challenge/ {
-		root /var/www/certbot;
-	}
 }
diff --git a/run_pylint.sh b/run_pylint.sh
@@ -7,8 +7,9 @@ pylint_exec=$(which pylint 2>/dev/null)
 
 set -e
 
-export PYTHONPATH=app/plot_app
-python $pylint_exec app/tornado_handlers/*.py app/serve.py \
-	app/plot_app/*.py app/download_logs.py
-
+pushd app
+export PYTHONPATH=plot_app
+python $pylint_exec tornado_handlers/*.py serve.py \
+	plot_app/*.py download_logs.py
+popd
 exit 0

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,2 @@`
	`1`	`+FROM certbot/certbot`
	`2`	`+ADD letsencrypt-renew /var/spool/cron/crontabs/root`
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,2 @@`
	`1`	`+# renew every week`
	`2`	`+0 12 * * */1 certbot renew`