Skip to content

Commit

Permalink
Merge pull request #78 from geoadmin/develop
Browse files Browse the repository at this point in the history
New Release v1.4.0 - #minor
  • Loading branch information
LukasJoss authored Feb 2, 2024
2 parents fcb6965 + 11395aa commit 2fd722f
Show file tree
Hide file tree
Showing 12 changed files with 589 additions and 415 deletions.
3 changes: 0 additions & 3 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -14,11 +14,9 @@ GIT_DIRTY = `git status --porcelain`
GIT_TAG = `git describe --tags || echo "no version info"`
AUTHOR = $(USER)


# general targets
LOGS_DIR = $(PWD)/logs


# Docker variables
DOCKER_REGISTRY = 974517877189.dkr.ecr.eu-central-1.amazonaws.com
DOCKER_IMG_LOCAL_TAG := $(DOCKER_REGISTRY)/$(SERVICE_NAME):local-$(USER)-$(GIT_HASH_SHORT)
Expand Down Expand Up @@ -128,7 +126,6 @@ format-lint: format lint
test:
ENV_FILE=.env.test $(NOSE) -c tests/unittest.cfg --verbose -s tests/

# Serve targets. Using these will run the application on your local machine. You can either serve with a wsgi front (like it would be within the container), or without.

.PHONY: serve
serve: clean_logs $(LOGS_DIR)
Expand Down
4 changes: 3 additions & 1 deletion Pipfile
Original file line number Diff line number Diff line change
Expand Up @@ -11,12 +11,14 @@ Flask = "~=2.0.1"
Pillow = "~=9.0.1"
python-dotenv = "~=0.17.0"
logging-utilities = "~=3.0"
werkzeug = "~=2.2"

[dev-packages]
yapf = "*"
nose2 = "*"
pylint = "*"
pylint-flask = "*"
cairosvg = "*"

[requires]
python_version = "3.9"
python_version = "3.9"
706 changes: 401 additions & 305 deletions Pipfile.lock

Large diffs are not rendered by default.

20 changes: 19 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,8 @@
- [Linting and formatting your work](#linting-and-formatting-your-work)
- [Test your work](#test-your-work)
- [Docker](#docker)
- [Maintenance](#maintenance)
- [Convert Symbols from svg to png](#convert-symbols-from-svg-to-png)
- [Deployment](#deployment)
- [Deployment configuration](#deployment-configuration)

Expand Down Expand Up @@ -156,6 +158,23 @@ You can also check these metadata on a running container as follows
docker ps --format="table {{.ID}}\t{{.Image}}\t{{.Labels}}"
```

## Maintenance

### Convert Symbols from svg to png

Sometimes it may happen, that we get a new set of icons. In general these icons have to be quadratic in a resolution of 48px x 48px in the format .png. Neverthanless there is a script to convert .svg images towards .png images. This script is located in the folder `scripts/svg2png.py`. There is a help provided

```bash
pipenv run python scripts/svg2png.py --help
```

Here is an example of such a convertion

```bash
pipenv run python scripts/svg2png.py --help
```pipenv run python scripts/svg2png.py -I ./tmp/new-icons -O ./static/images/babs2 -W 48 -H 48
```

## Deployment

### Deployment configuration
Expand All @@ -165,7 +184,6 @@ The service is configured by Environment Variable:
| Env | Default | Description |
| ----------- | --------------------- | -------------------------- |
| LOGGING_CFG | `logging-cfg-local.yml` | Logging configuration file |
| ALLOWED_DOMAINS | `.*` | Comma separated list of regex that are allowed as domain in Origin header |
| CACHE_CONTROL | `public, max-age=86400` | Cache Control header value of the `GET /*` endpoints |
| CACHE_CONTROL_4XX | `public, max-age=3600` | Cache Control header for 4XX responses |
| FORWARED_ALLOW_IPS | `*` | Sets the gunicorn `forwarded_allow_ips`. See [Gunicorn Doc](https://docs.gunicorn.org/en/stable/settings.html#forwarded-allow-ips). This setting is required in order to `secure_scheme_headers` to work. |
Expand Down
51 changes: 1 addition & 50 deletions app/__init__.py
Original file line number Diff line number Diff line change
@@ -1,17 +1,14 @@
import logging
import re
import time

from werkzeug.exceptions import HTTPException

from flask import Flask
from flask import abort
from flask import g
from flask import request

from app.helpers import make_error_msg
from app.helpers.service_icon_custom_serializer import CustomJSONEncoder
from app.settings import ALLOWED_DOMAINS_PATTERN
from app.settings import CACHE_CONTROL
from app.settings import CACHE_CONTROL_4XX

Expand All @@ -25,12 +22,6 @@
app.json_encoder = CustomJSONEncoder


def is_domain_allowed(domain):
return re.match(ALLOWED_DOMAINS_PATTERN, domain) is not None


# NOTE it is better to have this method registered first (before validate_origin) otherwise
# the route might not be logged if another method reject the request.
@app.before_request
def log_route():
g.setdefault('started', time.time())
Expand All @@ -44,9 +35,7 @@ def add_cors_header(response):
if request.endpoint == 'checker':
return response

response.headers['Access-Control-Allow-Origin'] = request.host_url
if 'Origin' in request.headers and is_domain_allowed(request.headers['Origin']):
response.headers['Access-Control-Allow-Origin'] = request.headers['Origin']
response.headers['Access-Control-Allow-Origin'] = '*'
response.headers['Vary'] = 'Origin'
response.headers['Access-Control-Allow-Methods'] = 'GET, HEAD, OPTIONS'
response.headers['Access-Control-Allow-Headers'] = '*'
Expand All @@ -63,44 +52,6 @@ def add_cache_control_header(response):
return response


# Reject request from non allowed origins
@app.before_request
def validate_origin():
# The Origin headers is automatically set by the browser and cannot be changed by the javascript
# application. Unfortunately this header is only set if the request comes from another origin.
# Sec-Fetch-Site header is set to `same-origin` by most of the browser except by Safari !
# The best protection would be to use the Sec-Fetch-Site and Origin header, however this is
# not supported by Safari. Therefore we added a fallback to the Referer header for Safari.
sec_fetch_site = request.headers.get('Sec-Fetch-Site', None)
origin = request.headers.get('Origin', None)
referrer = request.headers.get('Referer', None)

if origin is not None:
if is_domain_allowed(origin):
return
logger.error('Origin=%s does not match %s', origin, ALLOWED_DOMAINS_PATTERN)
abort(403, 'Permission denied')

# BGDIINF_SB-3115: Apparently IOS 16 has a bug and set Sec-Fetch-Site=cross-site even if the
# request is originated (same origin and/or referrer) from the same site ! Therefore to avoid
# issue on IOS we first checks the referrer before checking Sec-Fetch-Site even if this not
# correct.
if referrer is not None:
if is_domain_allowed(referrer):
return
logger.error('Referer=%s does not match %s', referrer, ALLOWED_DOMAINS_PATTERN)
abort(403, 'Permission denied')

if sec_fetch_site is not None:
if sec_fetch_site in ['same-origin', 'same-site']:
return
logger.error('Sec-Fetch-Site=%s is not allowed', sec_fetch_site)
abort(403, 'Permission denied')

logger.error('Referer and/or Origin and/or Sec-Fetch-Site headers not set')
abort(403, 'Permission denied')


@app.after_request
def log_response(response):
logger.info(
Expand Down
16 changes: 4 additions & 12 deletions app/routes.py
Original file line number Diff line number Diff line change
Expand Up @@ -65,18 +65,10 @@ def icon_metadata(icon_set_name, icon_name):
return make_api_compliant_response(icon)


@app.route(
'/sets/<icon_set_name>/icons/<icon_name>.png',
)
@app.route(
'/sets/<icon_set_name>/icons/<icon_name>-<red>,<green>,<blue>.png',
)
@app.route(
'/sets/<icon_set_name>/icons/<icon_name>@<scale>.png',
)
@app.route(
'/sets/<icon_set_name>/icons/<icon_name>@<scale>-<red>,<green>,<blue>.png',
)
@app.route('/sets/<icon_set_name>/icons/<icon_name>.png',)
@app.route('/sets/<icon_set_name>/icons/<icon_name>-<red>,<green>,<blue>.png',)
@app.route('/sets/<icon_set_name>/icons/<icon_name>@<scale>.png',)
@app.route('/sets/<icon_set_name>/icons/<icon_name>@<scale>-<red>,<green>,<blue>.png',)
def colorized_icon(
icon_set_name,
icon_name,
Expand Down
1 change: 0 additions & 1 deletion app/settings.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,5 @@

# Definition of the allowed domains for CORS implementation
ALLOWED_DOMAINS = os.getenv('ALLOWED_DOMAINS', r'.*').split(',')
ALLOWED_DOMAINS_PATTERN = f"({'|'.join(ALLOWED_DOMAINS)})"
CACHE_CONTROL = os.getenv('CACHE_CONTROL', 'public, max-age=86400')
CACHE_CONTROL_4XX = os.getenv('CACHE_CONTROL_4XX', 'public, max-age=3600')
160 changes: 160 additions & 0 deletions scripts/svg2png.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,160 @@
import argparse as ap
import logging
import os
import sys
from textwrap import dedent

import cairosvg

logger = logging.getLogger()
logger.setLevel(logging.DEBUG)

handler = logging.StreamHandler(sys.stdout)
handler.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
handler.setFormatter(formatter)
logger.addHandler(handler)


def create_parser():
parser = ap.ArgumentParser(
description=dedent(
"""\
Purpose:
This script is used to transform svg images into png
Example usage:
Transforming svg images into png using a source and a destination folder:
svg2png.py
--width 48
--height 48
--input /tmp/babs2
--output ../app/static/images/babs2
"""
),
formatter_class=ap.RawDescriptionHelpFormatter
)

option_group = parser.add_argument_group('Program options')
option_group.add_argument(
'-I',
'--input',
dest='input',
action='store',
type=the_dir,
required=True,
help='The dir where the svg images are located'
)

option_group.add_argument(
'-O',
'--output',
dest='output',
action='store',
type=the_dir,
required=True,
help='The dir where the png images will be stored'
)

option_group.add_argument(
'-W',
'--width',
dest='width',
action='store',
default=None,
type=pixel_size,
help='The width in pixel of the png image'
)

option_group.add_argument(
'-H',
'--height',
dest='height',
action='store',
default=None,
type=pixel_size,
help='The width in pixel of the png image'
)

option_group.add_argument(
'-R',
'--dpi',
dest='dpi',
action='store',
default=92,
type=dpi,
help='The resolution of the png image'
)

option_group.add_argument(
'-D',
'--dryrun',
dest='dryrun',
action='store_false',
default=True,
help='The width in pixel of the png image'
)

return parser


def dpi(d):
d = float(d) # this may lead to an error, but that is ok
if d > 20:
return d
logger.error("The resolution has to be bigger than 20 dpi")
sys.exit(1)


def pixel_size(size):
size = float(size) # this may lead to an error, but that is ok
if size > 0.0:
return size
logger.error("size (width and heigh) has to be > 0")
sys.exit(1)


def the_dir(d):
if len(d) > 0:
return d
logger.error("the path has to be specified")
sys.exit(1)


def validate_args(argv):
parser = create_parser()
the_opts = parser.parse_args(argv[1:])
return the_opts


def svg2png():
# create output folder if not exists
if not os.path.exists(opts.output):
os.makedirs(opts.output)

for file in os.listdir(opts.input):
if file.endswith(".svg"):
svg_image = os.path.join(opts.input, file)
png_image = os.path.join(opts.output, file[:-3] + 'png')
if opts.dryrun:
logger.debug("Treating image: %s > %s", svg_image, png_image)
cairosvg.svg2png(
dpi=opts.dpi,
file_obj=open(svg_image, "rb"), # pylint: disable=consider-using-with
write_to=png_image,
output_height=opts.height,
output_width=opts.width
)
else:
logger.debug("dryrun image: %s > %s", svg_image, png_image)


def main():
global opts # pylint: disable=global-variable-undefined
opts = validate_args(sys.argv)
svg2png()


if __name__ == '__main__':
main()
8 changes: 1 addition & 7 deletions tests/unit_tests/base_test.py
Original file line number Diff line number Diff line change
@@ -1,10 +1,8 @@
import re
import unittest

from flask import url_for

from app import app
from app.settings import ALLOWED_DOMAINS_PATTERN

ORIGIN_FOR_TESTING = "some_random_domain"

Expand All @@ -28,11 +26,7 @@ def tearDown(self):

def assertCors(self, response): # pylint: disable=invalid-name
self.assertIn('Access-Control-Allow-Origin', response.headers)
self.assertIsNotNone(
re.match(ALLOWED_DOMAINS_PATTERN, response.headers['Access-Control-Allow-Origin']),
msg=f"Access-Control-Allow-Origin={response.headers['Access-Control-Allow-Origin']} "
f"doesn't match {ALLOWED_DOMAINS_PATTERN}"
)
self.assertEqual(response.headers['Access-Control-Allow-Origin'], '*')
self.assertIn('Access-Control-Allow-Methods', response.headers)
self.assertListEqual(
sorted(['GET', 'HEAD', 'OPTIONS']),
Expand Down
Loading

0 comments on commit 2fd722f

Please sign in to comment.