added: New API endpoint _cache_control. Solves scrapinghub#203

Author: starrify

docs/api.rst

@@ -938,6 +938,27 @@ To reclaim some RAM send a POST request to the ``/_gc`` endpoint::
 
 It runs the Python garbage collector and clears internal WebKit caches.
 
+.. _http-cache-control:
+
+_cache_control
+~~~~~~~~~~~~~~
+
+To control the internal memory cache used in WebKit, send a POST to the
+``/_cache_control`` endpoint.
+
+Supported arguments:
+
+* ``max_pages`` : Optional. Non-negative integer. Would be passed to `QWebSettings::setMaximumPagesInCache`_
+
+* ``object_capacities``: Optional. Three non-negative integers separated by commas. Would be passed to `QWebSettings::setObjectCacheCapacities`_
+
+.. _`QWebSettings::setMaximumPagesInCache`: http://doc.qt.io/qt-5/qwebsettings.html#setMaximumPagesInCache
+.. _`QWebSettings::setObjectCacheCapacities`: http://doc.qt.io/qt-5/qwebsettings.html#setObjectCacheCapacities
+
+A sample to disable the internal WebKit memory cache::
+
+    curl -X POST http://localhost:8050/_cache_control --data max_pages=0 --data object_capacities=0,0,0
+
 .. _http-debug:
 
 _debug

splash/resources.py

@@ -5,6 +5,7 @@
 from __future__ import absolute_import
 import os
 import gc
+import re
 import time
 import json
 import types
@@ -16,6 +17,7 @@
 from twisted.internet import reactor, defer
 from twisted.python import log
 import six
+from PyQt5.QtWebKit import QWebSettings
 
 import splash
 from splash.qtrender import (
@@ -347,6 +349,36 @@ def render_POST(self, request):
         }).encode('utf-8')
 
 
+class CacheControlResource(Resource):
+    isLeaf = True
+    content_type = "application/json"
+
+    def render_POST(self, request):
+        for key, pattern in (
+                ('max_pages', r'^\d+$'),
+                ('object_capacities', r'^\d+,\d+,\d+$'),
+            ):
+            if request.args.get(key):
+                data = ''.join(request.args[key])
+                if not re.match(pattern, data):
+                    request.setResponseCode(400)
+                    return json.dumps({
+                        'status': 'error',
+                        'key': key,
+                        'supported_format': pattern,
+                        'received': data,
+                    })
+        if request.args.get('max_pages'):
+            data = ''.join(request.args['max_pages'])
+            QWebSettings.setMaximumPagesInCache(int(data))
+        if request.args.get('object_capacities'):
+            data = ''.join(request.args['object_capacities']).split(',')
+            QWebSettings.setObjectCacheCapacities(*[int(x) for x in data])
+        return json.dumps({
+            "status": "ok",
+        }).encode('utf-8')
+
+
 BOOTSTRAP_THEME = 'simplex'
 CODEMIRROR_OPTIONS = """{
     mode: 'lua',
@@ -704,6 +736,7 @@ def __init__(self, pool, ui_enabled, lua_enabled, lua_sandbox_enabled,
 
         self.putChild(b"_debug", DebugResource(pool))
         self.putChild(b"_gc", ClearCachesResource())
+        self.putChild(b"_cache_control", CacheControlResource())
 
         # backwards compatibility
         self.putChild(b"debug", DebugResource(pool, warn=True))