diff --git a/splash/browser_tab.py b/splash/browser_tab.py index fce2817a1..8333cac4a 100644 --- a/splash/browser_tab.py +++ b/splash/browser_tab.py @@ -13,7 +13,7 @@ from twisted.internet import defer from twisted.python import log -from splash import defaults +from splash.config import settings from splash.har.qt import cookies2har from splash.qtrender_image import QtImageRenderer from splash.qtutils import OPERATION_QT_CONSTANTS, WrappedSignal, qt2py, qurl2ascii @@ -84,10 +84,10 @@ def _init_webpage(self, verbosity, network_manager, splash_proxy_factory, render self.web_view.move(0, 0) self.web_view.show() - self.set_viewport(defaults.VIEWPORT_SIZE) + self.set_viewport(settings.VIEWPORT_SIZE) # XXX: hack to ensure that default window size is not 640x480. self.web_view.resize( - QSize(*map(int, defaults.VIEWPORT_SIZE.split('x')))) + QSize(*map(int, settings.VIEWPORT_SIZE.split('x')))) def set_js_enabled(self, val): settings = self.web_page.settings() @@ -184,7 +184,7 @@ def set_viewport(self, size, raise_if_empty=False): if raise_if_empty: raise RuntimeError("Cannot detect viewport size") else: - size = defaults.VIEWPORT_SIZE + size = settings.VIEWPORT_SIZE self.logger.log("Viewport is empty, falling back to: %s" % size) diff --git a/splash/cache.py b/splash/cache.py index d64d94aa9..46c306f70 100644 --- a/splash/cache.py +++ b/splash/cache.py @@ -2,10 +2,10 @@ from __future__ import absolute_import from PyQt4.QtNetwork import QNetworkDiskCache from twisted.python import log -from splash import defaults +from splash.config import settings -def construct(path=defaults.CACHE_PATH, size=defaults.CACHE_SIZE): +def construct(path=settings.CACHE_PATH, size=settings.CACHE_SIZE): log.msg("Initializing cache on %s (maxsize: %d Mb)" % (path, size)) cache = QNetworkDiskCache() cache.setCacheDirectory(path) diff --git a/splash/config.py b/splash/config.py new file mode 100644 index 000000000..73a22d117 --- /dev/null +++ b/splash/config.py @@ -0,0 +1,67 @@ +# -*- coding: utf-8 -*- + +import __builtin__ +import ast +import ConfigParser +import os + +from . import defaults + + +class ConfigError(Exception): + pass + +global CONFIG_PATH + + +class Settings(object): + """Handles config files and default values of config settings.""" + + NO_CONFIG_FILE_MSG = "Config file doesn't exist at %s" + + def __init__(self): + try: + self.config_path = CONFIG_PATH + except NameError: + # CONFIG_PATH is not defined. User hasn't passed in a config file. + self.config_path = None + self.defaults = {} + for name in dir(defaults): + if name.isupper(): + self.defaults[name] = getattr(defaults, name) + parser = ConfigParser.SafeConfigParser() + # don't convert keys to lowercase. + parser.optionxform = str + if parser.read(self._get_configfile_paths()): + # Safely evaluate configuration values. + self.cfg = {key: ast.literal_eval(val) for (key, val) in parser.items('settings')} + else: + self.cfg = {} + + def _get_configfile_paths(self): + """Returns a list of config file paths.""" + if self.config_path: + config_dir_path = os.path.abspath(os.path.expanduser(self.config_path)) + configfile_path = os.path.abspath(os.path.join(config_dir_path, 'splash.cfg')) + if not os.path.isfile(configfile_path): + # file doesn't exist + raise ConfigError(self.NO_CONFIG_FILE_MSG % configfile_path) + else: + return configfile_path + else: + xdg_config_home = os.environ.get('XDG_CONFIG_HOME') or \ + os.path.expanduser('~/.config') + return ['/etc/splash.cfg', + 'C:\\splash\splash.cfg', + os.path.join(xdg_config_home, 'splash.cfg'), + os.path.expanduser('~/.splash.cfg')] + + def __getattr__(self, item): + val = self.cfg.get(item, None) + if val is None: + val = self.defaults.get(item, None) + if val is None: + raise AttributeError("There is no settings named %s" % item) + return val + +settings = Settings() diff --git a/splash/network_manager.py b/splash/network_manager.py index 1aab5230c..9c5507ebf 100644 --- a/splash/network_manager.py +++ b/splash/network_manager.py @@ -24,13 +24,13 @@ AdblockRulesRegistry, ResourceTimeoutMiddleware) from splash.response_middleware import ContentTypeMiddleware -from splash import defaults +from splash.config import settings def create_default(filters_path=None, verbosity=None, allowed_schemes=None): - verbosity = defaults.VERBOSITY if verbosity is None else verbosity + verbosity = settings.VERBOSITY if verbosity is None else verbosity if allowed_schemes is None: - allowed_schemes = defaults.ALLOWED_SCHEMES + allowed_schemes = settings.ALLOWED_SCHEMES else: allowed_schemes = allowed_schemes.split(',') manager = SplashQNetworkAccessManager( diff --git a/splash/qtrender.py b/splash/qtrender.py index 6fe4e08ad..01b21eb2f 100644 --- a/splash/qtrender.py +++ b/splash/qtrender.py @@ -3,7 +3,7 @@ import json import functools import pprint -from splash import defaults +from splash.config import settings from splash.browser_tab import BrowserTab from splash.exceptions import RenderError @@ -80,11 +80,11 @@ def start(self, url, baseurl=None, wait=None, viewport=None, render_all=False, resource_timeout=None): self.url = url - self.wait_time = defaults.WAIT_TIME if wait is None else wait + self.wait_time = settings.WAIT_TIME if wait is None else wait self.js_source = js_source self.js_profile = js_profile self.console = console - self.viewport = defaults.VIEWPORT_SIZE if viewport is None else viewport + self.viewport = settings.VIEWPORT_SIZE if viewport is None else viewport self.render_all = render_all or viewport == 'full' if resource_timeout: diff --git a/splash/qtrender_image.py b/splash/qtrender_image.py index 43cb4e527..46253852a 100644 --- a/splash/qtrender_image.py +++ b/splash/qtrender_image.py @@ -9,7 +9,7 @@ from PyQt4.QtCore import QBuffer, QPoint, QRect, QSize, Qt from PyQt4.QtGui import QImage, QPainter, QRegion -from splash import defaults +from splash.config import settings class QtImageRenderer(object): @@ -36,7 +36,7 @@ def __init__(self, web_page, logger=None, image_format=None, self.width = width self.height = height if scale_method is None: - scale_method = defaults.IMAGE_SCALE_METHOD + scale_method = settings.IMAGE_SCALE_METHOD self.scale_method = scale_method self.image_format = image_format.upper() if not (self.is_png() or self.is_jpeg()): @@ -319,7 +319,7 @@ def _calculate_image_parameters(self, web_viewport, img_width, img_height): return image_viewport, image_size def _calculate_tiling(self, to_paint): - tile_maxsize = defaults.TILE_MAXSIZE + tile_maxsize = settings.TILE_MAXSIZE tile_hsize = min(tile_maxsize, to_paint.width()) tile_vsize = min(tile_maxsize, to_paint.height()) htiles = 1 + (to_paint.width() - 1) // tile_hsize @@ -420,7 +420,7 @@ def crop(self, rect): assert isinstance(rect, QRect) self.img = self.img.copy(rect) - def to_png(self, complevel=defaults.PNG_COMPRESSION_LEVEL): + def to_png(self, complevel=settings.PNG_COMPRESSION_LEVEL): quality = 90 - (complevel * 10) buf = QBuffer() self.img.save(buf, 'png', quality) @@ -428,7 +428,7 @@ def to_png(self, complevel=defaults.PNG_COMPRESSION_LEVEL): def to_jpeg(self, quality=None): if quality is None: - quality = defaults.JPEG_QUALITY + quality = settings.JPEG_QUALITY buf = QBuffer() self.img.save(buf, 'jpeg', quality) return bytes(buf.data()) @@ -454,14 +454,14 @@ def crop(self, rect): top, bottom = rect.top(), rect.top() + rect.height() self.img = self.img.crop((left, top, right, bottom)) - def to_png(self, complevel=defaults.PNG_COMPRESSION_LEVEL): + def to_png(self, complevel=settings.PNG_COMPRESSION_LEVEL): buf = StringIO() self.img.save(buf, 'png', compress_level=complevel) return buf.getvalue() def to_jpeg(self, quality=None): if quality is None: - quality = defaults.JPEG_QUALITY + quality = settings.JPEG_QUALITY buf = StringIO() self.img.save(buf, 'jpeg', quality=quality) return buf.getvalue() diff --git a/splash/render_options.py b/splash/render_options.py index c3667ae87..bee523b5a 100644 --- a/splash/render_options.py +++ b/splash/render_options.py @@ -2,7 +2,7 @@ from __future__ import absolute_import import os import json -from splash import defaults +from splash.config import settings from splash.utils import path_join_secure from splash.exceptions import BadOption @@ -115,19 +115,19 @@ def get_baseurl(self): return self._get_url("baseurl", default=None) def get_wait(self): - return self.get("wait", defaults.WAIT_TIME, - type=float, range=(0, defaults.MAX_WAIT_TIME)) + return self.get("wait", settings.WAIT_TIME, + type=float, range=(0, settings.MAX_WAIT_TIME)) def get_timeout(self): - default = min(self.max_timeout, defaults.TIMEOUT) + default = min(self.max_timeout, settings.TIMEOUT) return self.get("timeout", default, type=float, range=(0, self.max_timeout)) def get_resource_timeout(self): - return self.get("resource_timeout", defaults.RESOURCE_TIMEOUT, + return self.get("resource_timeout", settings.RESOURCE_TIMEOUT, type=float, range=(0, 1e6)) def get_images(self): - return self._get_bool("images", defaults.AUTOLOAD_IMAGES) + return self._get_bool("images", settings.AUTOLOAD_IMAGES) def get_proxy(self): return self.get("proxy", default=None) @@ -136,13 +136,13 @@ def get_js_source(self): return self.get("js_source", default=None) def get_width(self): - return self.get("width", None, type=int, range=(1, defaults.MAX_WIDTH)) + return self.get("width", None, type=int, range=(1, settings.MAX_WIDTH)) def get_height(self): - return self.get("height", None, type=int, range=(1, defaults.MAX_HEIGTH)) + return self.get("height", None, type=int, range=(1, settings.MAX_HEIGTH)) def get_scale_method(self): - scale_method = self.get("scale_method", defaults.IMAGE_SCALE_METHOD) + scale_method = self.get("scale_method", settings.IMAGE_SCALE_METHOD) allowed_scale_methods = ['raster', 'vector'] if scale_method not in allowed_scale_methods: self.raise_error( @@ -155,7 +155,7 @@ def get_scale_method(self): return scale_method def get_quality(self): - return self.get("quality", defaults.JPEG_QUALITY, type=int, range=(0, 100)) + return self.get("quality", settings.JPEG_QUALITY, type=int, range=(0, 100)) def get_http_method(self): method = self.get("http_method", "GET") @@ -226,7 +226,7 @@ def get_headers(self): return headers def get_viewport(self, wait=None): - viewport = self.get("viewport", defaults.VIEWPORT_SIZE) + viewport = self.get("viewport", settings.VIEWPORT_SIZE) if viewport == 'full': if wait == 0: @@ -323,14 +323,14 @@ def get_jpeg_params(self): def get_include_params(self): return dict( - html=self._get_bool("html", defaults.DO_HTML), - iframes=self._get_bool("iframes", defaults.DO_IFRAMES), - png=self._get_bool("png", defaults.DO_PNG), - jpeg=self._get_bool("jpeg", defaults.DO_JPEG), - script=self._get_bool("script", defaults.SHOW_SCRIPT), - console=self._get_bool("console", defaults.SHOW_CONSOLE), - history=self._get_bool("history", defaults.SHOW_HISTORY), - har=self._get_bool("har", defaults.SHOW_HAR), + html=self._get_bool("html", settings.DO_HTML), + iframes=self._get_bool("iframes", settings.DO_IFRAMES), + png=self._get_bool("png", settings.DO_PNG), + jpeg=self._get_bool("jpeg", settings.DO_JPEG), + script=self._get_bool("script", settings.SHOW_SCRIPT), + console=self._get_bool("console", settings.SHOW_CONSOLE), + history=self._get_bool("history", settings.SHOW_HISTORY), + har=self._get_bool("har", settings.SHOW_HAR), ) @@ -345,9 +345,9 @@ def validate_size_str(size_str): :param size_str: string to validate """ - max_width = defaults.VIEWPORT_MAX_WIDTH - max_heigth = defaults.VIEWPORT_MAX_HEIGTH - max_area = defaults.VIEWPORT_MAX_AREA + max_width = settings.VIEWPORT_MAX_WIDTH + max_heigth = settings.VIEWPORT_MAX_HEIGTH + max_area = settings.VIEWPORT_MAX_AREA try: w, h = map(int, size_str.split('x')) except ValueError: diff --git a/splash/server.py b/splash/server.py index 24724d1df..c6f8c4cf6 100644 --- a/splash/server.py +++ b/splash/server.py @@ -7,10 +7,14 @@ import signal import functools -from splash import defaults, __version__ -from splash import xvfb +from splash import config +from splash import xvfb, __version__ from splash.qtutils import init_qt_app + +settings = config.Settings() + + def install_qtreactor(verbose): init_qt_app(verbose) import qt4reactor @@ -21,14 +25,16 @@ def parse_opts(): _bool_default = {True:' (default)', False: ''} op = optparse.OptionParser() + op.add_option("--config-path", + help="path to a folder with a config file named splash.cfg") op.add_option("-f", "--logfile", help="log file") op.add_option("-m", "--maxrss", type=float, default=0, help="exit if max RSS reaches this value (in MB or ratio of physical mem) (default: %default)") - op.add_option("-p", "--port", type="int", default=defaults.SPLASH_PORT, + op.add_option("-p", "--port", type="int", default=settings.SPLASH_PORT, help="port to listen to (default: %default)") - op.add_option("-s", "--slots", type="int", default=defaults.SLOTS, + op.add_option("-s", "--slots", type="int", default=settings.SLOTS, help="number of render slots (default: %default)") - op.add_option("--max-timeout", type="float", default=defaults.MAX_TIMEOUT, + op.add_option("--max-timeout", type="float", default=settings.MAX_TIMEOUT, help="maximum allowed value for timeout (default: %default)") op.add_option("--proxy-profiles-path", help="path to a folder with proxy profiles") @@ -37,20 +43,20 @@ def parse_opts(): op.add_option("--no-js-cross-domain-access", action="store_false", dest="js_cross_domain_enabled", - default=not defaults.JS_CROSS_DOMAIN_ENABLED, - help="disable support for cross domain access when executing custom javascript" + _bool_default[not defaults.JS_CROSS_DOMAIN_ENABLED]) + default=not settings.JS_CROSS_DOMAIN_ENABLED, + help="disable support for cross domain access when executing custom javascript" + _bool_default[not settings.JS_CROSS_DOMAIN_ENABLED]) op.add_option("--js-cross-domain-access", action="store_true", dest="js_cross_domain_enabled", - default=defaults.JS_CROSS_DOMAIN_ENABLED, + default=settings.JS_CROSS_DOMAIN_ENABLED, help="enable support for cross domain access when executing custom javascript " - "(WARNING: it could break rendering for some of the websites)" + _bool_default[defaults.JS_CROSS_DOMAIN_ENABLED]) + "(WARNING: it could break rendering for some of the websites)" + _bool_default[settings.JS_CROSS_DOMAIN_ENABLED]) op.add_option("--no-cache", action="store_false", dest="cache_enabled", - help="disable local cache" + _bool_default[not defaults.CACHE_ENABLED]) + help="disable local cache" + _bool_default[not settings.CACHE_ENABLED]) op.add_option("--cache", action="store_true", dest="cache_enabled", - help="enable local cache (WARNING: don't enable it unless you know what are you doing)" + _bool_default[defaults.CACHE_ENABLED]) + help="enable local cache (WARNING: don't enable it unless you know what are you doing)" + _bool_default[settings.CACHE_ENABLED]) op.add_option("-c", "--cache-path", help="local cache folder") - op.add_option("--cache-size", type=int, default=defaults.CACHE_SIZE, + op.add_option("--cache-size", type=int, default=settings.CACHE_SIZE, help="maximum cache size in MB (default: %default)") op.add_option("--manhole", action="store_true", help="enable manhole server") @@ -58,14 +64,14 @@ def parse_opts(): help="disable proxy server") op.add_option("--disable-ui", action="store_true", default=False, help="disable web UI") - op.add_option("--proxy-portnum", type="int", default=defaults.PROXY_PORT, + op.add_option("--proxy-portnum", type="int", default=settings.PROXY_PORT, help="proxy port to listen to (default: %default)") - op.add_option('--allowed-schemes', default=",".join(defaults.ALLOWED_SCHEMES), + op.add_option('--allowed-schemes', default=",".join(settings.ALLOWED_SCHEMES), help="comma-separated list of allowed URI schemes (defaut: %default)") op.add_option("--filters-path", help="path to a folder with network request filters") - op.add_option("--disable-private-mode", action="store_true", default=not defaults.PRIVATE_MODE, - help="disable private mode (WARNING: data may leak between requests)" + _bool_default[not defaults.PRIVATE_MODE]) + op.add_option("--disable-private-mode", action="store_true", default=not settings.PRIVATE_MODE, + help="disable private mode (WARNING: data may leak between requests)" + _bool_default[not settings.PRIVATE_MODE]) op.add_option("--disable-xvfb", action="store_true", default=False, help="disable Xvfb auto start") op.add_option("--disable-lua", action="store_true", default=False, @@ -77,7 +83,7 @@ def parse_opts(): "Each place can have a ? in it that's replaced with the module name.") op.add_option("--lua-sandbox-allowed-modules", default="", help="semicolon-separated list of Lua module names allowed to be required from a sandbox.") - op.add_option("-v", "--verbosity", type=int, default=defaults.VERBOSITY, + op.add_option("-v", "--verbosity", type=int, default=settings.VERBOSITY, help="verbosity level; valid values are integers from 0 to 5 (default: %default)") op.add_option("--version", action="store_true", help="print Splash version number and exit") @@ -150,9 +156,9 @@ def manhole_server(portnum=None, username=None, password=None): from twisted.manhole import telnet f = telnet.ShellFactory() - f.username = defaults.MANHOLE_USERNAME if username is None else username - f.password = defaults.MANHOLE_PASSWORD if password is None else password - portnum = defaults.MANHOLE_PORT if portnum is None else portnum + f.username = settings.MANHOLE_USERNAME if username is None else username + f.password = settings.MANHOLE_PASSWORD if password is None else password + portnum = settings.MANHOLE_PORT if portnum is None else portnum reactor.listenTCP(portnum, f) @@ -172,10 +178,10 @@ def splash_server(portnum, slots, network_manager, max_timeout, from twisted.python import log from splash import lua - verbosity = defaults.VERBOSITY if verbosity is None else verbosity + verbosity = settings.VERBOSITY if verbosity is None else verbosity log.msg("verbosity=%d" % verbosity) - slots = defaults.SLOTS if slots is None else slots + slots = settings.SLOTS if slots is None else slots log.msg("slots=%s" % slots) pool = RenderPool( @@ -217,7 +223,7 @@ def splash_server(portnum, slots, network_manager, max_timeout, if not disable_proxy: from splash.proxy_server import SplashProxyServerFactory proxy_server_factory = SplashProxyServerFactory(pool, max_timeout=max_timeout) - proxy_portnum = defaults.PROXY_PORT if proxy_portnum is None else proxy_portnum + proxy_portnum = settings.PROXY_PORT if proxy_portnum is None else proxy_portnum reactor.listenTCP(proxy_portnum, proxy_server_factory) @@ -286,9 +292,9 @@ def _default_cache(cache_enabled, cache_path, cache_size): from twisted.python import log from splash import cache - cache_enabled = defaults.CACHE_ENABLED if cache_enabled is None else cache_enabled - cache_path = defaults.CACHE_PATH if cache_path is None else cache_path - cache_size = defaults.CACHE_SIZE if cache_size is None else cache_size + cache_enabled = settings.CACHE_ENABLED if cache_enabled is None else cache_enabled + cache_path = settings.CACHE_PATH if cache_path is None else cache_path + cache_size = settings.CACHE_SIZE if cache_size is None else cache_size if cache_enabled: log.msg("cache_enabled=%s, cache_path=%r, cache_size=%sMB" % (cache_enabled, cache_path, cache_size)) @@ -344,6 +350,10 @@ def main(): print(__version__) sys.exit(0) + if opts.config_path: + config.CONFIG_PATH = opts.config_path + reload(config) + start_logging(opts) log_splash_version() bump_nofile_limit() diff --git a/splash/xvfb.py b/splash/xvfb.py index 07369cf70..3b96f8521 100644 --- a/splash/xvfb.py +++ b/splash/xvfb.py @@ -6,7 +6,7 @@ from __future__ import absolute_import import sys from contextlib import contextmanager -from splash import defaults +from splash.config import settings from twisted.python import log @@ -34,7 +34,7 @@ def _get_xvfb(): try: from xvfbwrapper import Xvfb - width, height = map(int, defaults.VIEWPORT_SIZE.split("x")) + width, height = map(int, settings.VIEWPORT_SIZE.split("x")) return Xvfb(width, height) except ImportError: return None