Merge branch 'merge-1.0.1' into main

deb/postinst

@@ -35,8 +35,8 @@ case "$1" in
         [ -e /etc/pagekite/pagekite.rc.dpkg-bak ] \
           && mv /etc/pagekite/pagekite.rc.dpkg-bak /etc/pagekite.d/89_old_pagekite.rc
 
-	chmod 644 /etc/pagekite.d/* || true
-	chmod 600 /etc/pagekite.d/[019]* || true
+	chmod 644 /etc/pagekite.d/*.rc* || true
+	chmod 600 /etc/pagekite.d/[019]*rc* || true
         [ -d /etc/pagekite ] && rmdir /etc/pagekite || true
     ;;
 

doc/HISTORY.txt

@@ -1,10 +1,15 @@
 Version history - highlights
 ============================
 
-v1.5.0.200412
+v1.5.????????
 -------------
+   - This release (v?) is all about performance and efficiency!
    - Create ping.pagekite fast-path in dedicated thread
-
+   - Make select loop timing and read sizes configurable, tweak defaults
+   - Remove 0.4.x flow-control, fix major bugs in current flow control code
+   - Fix locking-related deadlocks under PyPy
+   - Added --watchdog=N, to self-reap locked up processes
+   - Disabled old ssl workarounds on modern versions of Python (broke PyPy)
 
 v1.5.0.200327
 -------------

etc/pagekite.d/accept.acl.sample

@@ -2,7 +2,10 @@
 #
 # This is a file for use on frontend relays to restrict access. Note
 # that this effects both tunnels and client connections and is really
-# only intended for blacklisting abusive clients on a temporary basis.
+# only intended for blocking abusive clients on a temporary basis.
+#
+# WARNING: This is inefficient and slow. Every line added to this file
+#          has a cost.
 #
 # To enable these rules, rename the file and add the following to one
 # of the `/etc/pagekite.d/*.rc` files:

pagekite/common.py

@@ -75,8 +75,17 @@
 LOOPBACK_BE = LOOPBACK_HN + ':2'
 LOOPBACK = {'FE': LOOPBACK_FE, 'BE': LOOPBACK_BE}
 
+# This is how many bytes we are willing to read per cycle.
+MAX_READ_BYTES = 16 * 1024
+MAX_READ_TUNNEL_X = 3.1     # 3x above, + fudge factor
+
+# Higher values save CPU and prevent individual tunnels
+# from hogging all our resources, but hurt latency and
+# reduce per-tunnel throughput.
+SELECT_LOOP_MIN_MS = 5
+
 # Re-evaluate our choice of frontends every 45-60 minutes.
-FE_PING_INTERVAL     = (45 * 60) + random.randint(0, 900)
+FE_PING_INTERVAL = (45 * 60) + random.randint(0, 900)
 
 # This is a global count of disconnect errors; we use this
 # to adjust the ping interval over time.

pagekite/compat.py

@@ -104,15 +104,14 @@ def format_exc():
 except ImportError:
   from queue import Queue
 
-
 # SSL/TLS strategy: prefer pyOpenSSL, as it comes with built-in Context
 # objects. If that fails, look for Python 2.6+ native ssl support and
 # create a compatibility wrapper. If both fail, bomb with a ConfigError
 # when the user tries to enable anything SSL-related.
 #
 import sockschain
 socks = sockschain
-if socks.HAVE_PYOPENSSL:
+if socks.HAVE_PYOPENSSL or tuple(sys.version_info) > (2, 7, 10):
   SSL = socks.SSL
   SEND_ALWAYS_BUFFERS = False
   SEND_MAX_BYTES = 16 * 1024
@@ -122,7 +121,7 @@ def format_exc():
   SSL = socks.SSL
   SEND_ALWAYS_BUFFERS = True
   SEND_MAX_BYTES = 4 * 1024
-  TUNNEL_SOCKET_BLOCKS = True # Workaround for http://bugs.python.org/issue8240
+  TUNNEL_SOCKET_BLOCKS = True  # Workaround for http://bugs.python.org/issue8240
 
 else:
   SEND_ALWAYS_BUFFERS = False

pagekite/httpd.py

@@ -420,10 +420,8 @@ def do_POST(self, command='POST'):
                            "string (%s bytes).") % clength)
         posted = cgi.parse_qs(self.rfile.read(clength), 1)
       elif self.host_config.get('xmlrpc', False):
-        # We wrap the XMLRPC request handler in _BEGIN/_END in order to
-        # expose the request environment to the RPC functions.
-        RCI = self.server.RCI
-        return RCI._END(SimpleXMLRPCRequestHandler.do_POST(RCI._BEGIN(self)))
+        with self.server.RCI.lock:
+          return SimpleXMLRPCRequestHandler.do_POST(self)
 
       self.post_data.seek(0)
     except socket.error:
@@ -861,7 +859,12 @@ def handleHttpRequest(self, scheme, netloc, path, params, query, frag,
     photobackup = self.host_config.get('photobackup', False)
 
     if path == self.host_config.get('yamon', False):
-      if common.gYamon:
+      if qs.get('view', [None])[0] == 'conns':
+        from pagekite.pk import Watchdog
+        llines = []
+        Watchdog.DumpConnState(self.server.pkite.conns, logfunc=llines.append)
+        data['body'] = '\n'.join(llines) + '\n'
+      elif common.gYamon:
         self.server.pkite.Overloaded(yamon=common.gYamon)
         data['body'] = common.gYamon.render_vars_text(qs.get('view', [None])[0])
       else:
@@ -964,7 +967,7 @@ def __init__(self, httpd, pkite, conns):
     self.conns = conns
     self.modified = False
 
-    self.lock = threading.Lock()
+    self.lock = threading.RLock()
     self.request = None
 
     self.auth_tokens = {httpd.secret: self.ACL_READ}
@@ -978,17 +981,6 @@ def __init__(self, httpd, pkite, conns):
                              'tokens': self.auth_tokens,
                              'data': logging.LOG}}
 
-  def _BEGIN(self, request_object):
-    self.lock.acquire()
-    self.request = request_object
-    return request_object
-
-  def _END(self, rv=None):
-    if self.request:
-      self.request = None
-      self.lock.release()
-    return rv
-
   def connections(self, auth_token):
     if (not self.request.host_config.get('console', False) or
         self.ACL_READ not in self.auth_tokens.get(auth_token, self.ACL_OPEN)):
@@ -1135,15 +1127,12 @@ def __init__(self, sspec, pkite, conns,
       gYamon = common.gYamon = yamond.YamonD(sspec)
       gYamon.vset('started', int(time.time()))
       gYamon.vset('version', APPVER)
+      gYamon.vset('version_python', sys.version.replace('\n', ' '))
       gYamon.vset('httpd_ssl_enabled', self.enable_ssl)
       gYamon.vset('errors', 0)
       gYamon.lcreate("tunnel_rtt", 100)
       gYamon.lcreate("tunnel_wrtt", 100)
       gYamon.lists['buffered_bytes'] = [1, 0, common.buffered_bytes]
-      gYamon.views['selectables'] = (selectables.SELECTABLES, {
-        'idle': [0, 0, self.conns.idle],
-        'conns': [0, 0, self.conns.conns]
-      })
     except:
       pass