release 5.9.10

+ update blacklist.
+ add direct mode
+ query DNS from local first for unknown domain.

Author: Michael-X-Net

README.md

@@ -25,8 +25,8 @@
 
 
 ### 最新公告：
- 2024-02-25
-* 最新版5.9.9, 修复5.9.7智能策略更新的bug。
+ 2024-03-06
+* 最新版5.9.10, 更新黑名单列表。
 * 5.9.0 升级GAE服务端到python3
 * 5.8.8 改进iOS下连接性能
 * 5.7.0 为X-Tunnel增加新通道

code/default/launcher/web_control.py

@@ -923,7 +923,7 @@ def req_log_files(self):
         sys.path.append(x_tunnel_local)
         from upload_logs import pack_logs
 
-        data = pack_logs(10 * 1024 * 1024)
+        data = pack_logs(200 * 1024 * 1024)
         self.send_response("application/zip", data)
 
     def req_mem_info_handler(self):

code/default/lib/noarch/front_base/http1.py

@@ -68,8 +68,9 @@ def request(self, task):
     def keep_alive_thread(self):
         while time.time() - self.ssl_sock.create_time < self.config.http1_first_ping_wait:
             if not self.keep_running:
-                self.close("exit ")
+                self.close("exit")
                 return
+
             time.sleep(3)
 
         if self.config.http1_first_ping_wait and self.processed_tasks == 0:
@@ -87,7 +88,7 @@ def keep_alive_thread(self):
 
         elif self.config.http1_idle_time:
             while self.keep_running:
-                time_to_sleep = max(self.config.http1_idle_time - (time.time() - self.last_recv_time), 0.2)
+                time_to_sleep = max(self.config.http1_idle_time - (time.time() - self.last_recv_time), 3)
                 time.sleep(time_to_sleep)
 
                 if not self.request_onway and time.time() - self.last_recv_time > self.config.http1_idle_time:
@@ -113,7 +114,6 @@ def work_loop(self):
                     self.close("keep alive")
                     return
 
-                self.last_recv_time = time.time()
                 continue
 
             # self.logger.debug("http1 get task")
@@ -127,7 +127,6 @@ def work_loop(self):
             self.request_task(task)
             self.request_onway = False
             self.last_send_time = time_now
-            self.last_recv_time = time_now
 
             life_end_reason = self.is_life_end()
             if life_end_reason:
@@ -138,7 +137,6 @@ def request_task(self, task):
         timeout = task.timeout
         self.request_onway = True
         start_time = time.time()
-        self.last_recv_time = start_time
 
         self.record_active("request")
         task.set_state("h1_req")
@@ -160,10 +158,11 @@ def request_task(self, task):
                 sended = self.ssl_sock.send(task.body[start:start + send_size])
                 start += sended
 
-            task.set_state("h1_req_sended")
+            task.set_state("h1_req_sent")
         except Exception as e:
-            self.logger.warn("%s h1_request send:%r inactive_time:%d task.timeout:%d",
-                             self.ip_str, e, time.time() - self.last_recv_time, task.timeout)
+            self.logger.warn("%s %s h1_request send:%r inactive_time:%d task.timeout:%d",
+                             self.ip_str, self.ssl_sock.getsockname(),
+                             e, time.time() - self.last_recv_time, task.timeout)
             self.logger.warn('%s trace:%s', self.ip_str, self.get_trace())
 
             self.retry_task_cb(task)
@@ -175,6 +174,7 @@ def request_task(self, task):
             response = simple_http_client.Response(self.ssl_sock)
             response.begin(timeout=timeout)
             task.set_state("response_begin")
+            self.last_recv_time = time.time()
         except Exception as e:
             self.logger.warn("%s h1_request recv:%r inactive_time:%d task.timeout:%d",
                              self.ip_str, e, time.time() - self.last_recv_time, task.timeout)
@@ -201,7 +201,7 @@ def request_task(self, task):
             task.queue.put(response)
         else:
             if self.config.http2_show_debug:
-                self.logger.debug("got pong for %s status:%d", self.ip_str, response.status)
+                self.logger.debug("got res for %s status:%d", self.ip_str, response.status)
 
         try:
             read_target = int(response.content_length)
@@ -290,6 +290,7 @@ def head_request(self):
             self.record_active("head end")
             rtt = (time.time() - start_time) * 1000
             self.update_rtt(rtt)
+            self.last_recv_time = time.time()
             return True
         except Exception as e:
             self.logger.warn("h1 %s HEAD keep alive request fail:%r", self.ssl_sock.ip_str, e)

code/default/lib/noarch/front_base/http_common.py

@@ -217,7 +217,7 @@ def __init__(self, logger, ip_manager, config, ssl_sock, close_cb, retry_task_cb
         self.last_send_time = self.ssl_sock.create_time
         self.life_end_time = self.ssl_sock.create_time + \
                              random.randint(self.config.connection_max_life, int(self.config.connection_max_life * 1.5))
-        # self.logger.debug("worker.init %s", self.ip_str)
+        # self.logger.debug("worker.init %s %s", self.ip_str, self.ssl_sock.getsockname())
 
     def __str__(self):
         o = ""

code/default/smart_router/local/dns_query.py

@@ -635,6 +635,10 @@ def query_blocked_domain(self, domain, dns_type):
         ])
 
     def query_unknown_domain(self, domain, dns_type):
+        res = self.local_dns_resolve.query(domain, dns_type)
+        if res:
+            return res
+
         return self.parallel_query.query(domain, dns_type, [
             self.https_query.query,
             self.tls_query.query,
@@ -662,7 +666,7 @@ def query(self, domain, dns_type=1, history=[]):
             xlog.debug("DNS query:%s in black", domain)
             return ips
 
-        elif b"." not in domain or g.gfwlist.in_white_list(domain) or rule in ["direct"]:
+        elif b"." not in domain or g.gfwlist.in_white_list(domain) or rule in ["direct"] or g.config.pac_policy == "all_Direct":
             ips = self.local_dns_resolve.query(domain, timeout=1)
             g.domain_cache.set_ips(domain, ips, dns_type)
             return ips

code/default/smart_router/local/gfw_white_list.txt

@@ -6,6 +6,7 @@
 178.com
 2345.com
 360safe.com
+51.la
 56.com
 5eplay.com
 5ewin.com
@@ -26,6 +27,7 @@ alipay.com
 alipayobjects.com
 alivecdn.com
 aliyuncs.com
+aliyunddos1030.com
 allrace.com
 amap.com
 amemv.com
@@ -53,6 +55,10 @@ bokecs.net
 bootcss.com
 btigroup.io
 bytedance.com
+bytedns1.com
+bytefcdn.com
+byteimg.com
+bytegecko.co
 cccpan.com
 cdncl.net
 chiphell.com
@@ -76,6 +82,8 @@ douyin.com
 douyincdn.com
 douyinliving.com
 douyinpic.com
+douyinstatic.com.w.cdngslb.com
+douyinstatic.com.queniuuf.com
 douyinvod.com
 douyu.com
 duoduocdn.com
@@ -125,6 +133,7 @@ kmf.com
 ksyungslb.com
 kugou.com
 kuiniuca.com
+kunluncan.com
 le.com
 lecloud.com
 leisu.com
@@ -174,6 +183,7 @@ qq-zuidazy.com
 qq.com
 qqmail.com
 qtlglb.com
+queniusy.com
 remuxhdr.com
 ruioushang.com
 sandai.net
@@ -207,6 +217,7 @@ ubuntu.com
 uniqueway.com
 umeng.com
 videocc.net
+volcfcdndvs.com
 vzan.cc
 vzuu.com
 wangyuan.com

code/default/smart_router/local/ip_region.py

@@ -119,7 +119,7 @@ def generate_db(self):
                      '198.51.100.0/24',  # TEST-NET-2
                      '203.0.113.0/24',  # TEST-NET-3
                      # 连续地址直到 IP 结束，特殊处理
-                     # '224.0.0.0/4',  #组播地址（D类）
+                     '224.0.0.0/4',  #组播地址（D类）
                      # '240.0.0.0/4',  #保留地址（E类）
                      )
         keeplist = []

code/default/smart_router/local/pac_server.py

@@ -27,7 +27,7 @@
 gae_ca_file = os.path.join(env_info.data_path, "gae_proxy", "CA.crt")
 
 
-allow_policy = ["black_GAE", "black_X-Tunnel", "smart-router", "all_X-Tunnel"]
+allow_policy = ["black_GAE", "black_X-Tunnel", "smart-router", "all_X-Tunnel", "all_Direct"]
 
 
 def get_serving_pacfile():
@@ -52,6 +52,10 @@ def policy_all_to_proxy(self, host, port):
         content = content.replace(self.PROXY_LISTEN, proxy)
         return content
 
+    def policy_all_to_direct(self):
+        content = """function FindProxyForURL(url, host) { return 'DIRECT';}"""
+        return content
+
     def policy_blacklist_to_proxy(self, host, port):
         content = get_serving_pacfile()
 
@@ -89,6 +93,8 @@ def do_GET(self):
             content = self.policy_blacklist_to_proxy(host, "%s" % g.x_tunnel_socks_port)
         elif g.config.pac_policy == "all_X-Tunnel":
             content = self.policy_all_to_proxy(host, "%s" % g.x_tunnel_socks_port)
+        elif g.config.pac_policy == "all_Direct":
+            content = self.policy_all_to_direct()
         else:
             content = self.policy_all_to_proxy(host, g.config.proxy_port)
 

code/default/smart_router/local/pipe_socks.py

@@ -1,6 +1,8 @@
 import threading
 import time
 import sys
+import socket
+import errno
 
 from xx_six import BlockingIOError
 import utils
@@ -214,18 +216,21 @@ def flush_send_s(s2, d1):
 
                     try:
                         d = s1.recv(65535)
+                    except BlockingIOError as e:
+                        xlog.debug("%s recv BlockingIOError:%r", s1, e)
+                        continue
                     except Exception as e:
-                        # xlog.debug("%s recv e:%r", s1, e)
+                        xlog.debug("%s recv e:%r", s1, e)
                         self.close(s1, "r")
                         continue
 
                     if not d:
                         # socket closed by peer.
-                        # xlog.debug("%s recv empty, close", s1)
+                        xlog.debug("%s recv empty, close", s1)
                         self.close(s1, "r")
                         continue
 
-                    # xlog.debug("direct received %d bytes from:%s", len(d), s1)
+                    xlog.debug("direct received %d bytes from:%s", len(d), s1)
                     s1.recved_data += len(d)
                     s1.recved_times += 1
 
@@ -253,6 +258,10 @@ def flush_send_s(s2, d1):
                                 flush_send_s(s2, d1)
                                 s2.sent_data += len(d1)
                                 s2.sent_times += 1
+                            except BlockingIOError as e:
+                                xlog.warn("Except %s flush_send_s BlockingIOError %r", s2, e)
+                                self.close(s2, "w")
+                                continue
                             except Exception as e:
                                 xlog.warn("send split SNI:%s fail:%r", s2.host, e)
                                 self.close(s2, "w")
@@ -268,12 +277,24 @@ def flush_send_s(s2, d1):
                             s2.sent_data += sent
                             s2.sent_times += 1
                             # xlog.debug("direct send %d to %s from:%s total:%d", sent, s2, s1, len(d))
+                        except BlockingIOError as e:
+                            xlog.warn("Except %s send BlockingIOError %r", s2, e)
+                            sent = 0
+                        except socket.error as e:
+                            if e.errno == errno.EAGAIN:
+                                # if str(e) == "[Errno 35] Resource temporarily unavailable":
+                                xlog.warn("%s send errno.EAGAIN %r", s2, e)
+                                time.sleep(0.1)
+                                sent = 0
+                            else:
+                                self.close(s2, "w")
+                                continue
                         except Exception as e:
                             # xlog.debug("%s send e:%r", s2, e)
                             if sys.version_info[0] == 3 and isinstance(e, BlockingIOError):
                                 # This error happened on upload large file or speed test
                                 # Just ignore this error and will be fine
-                                # xlog.warn("%s send BlockingIOError %r", s2, e)
+                                xlog.warn("%s send BlockingIOError %r", s2, e)
                                 sent = 0
                             else:
                                 # xlog.warn("%s send except:%r", s2, e)
@@ -320,11 +341,23 @@ def flush_send_s(s2, d1):
                             s1.sent_data += sent
                             s1.sent_times += 1
                             # xlog.debug("send buffered %d bytes to %s", sent, s1)
+                        except BlockingIOError as e:
+                            xlog.warn("Except %s send BlockingIOError %r", s1, e)
+                            sent = 0
+                        except socket.error as e:
+                            if e.errno == errno.EAGAIN:
+                                # if str(e) == "[Errno 35] Resource temporarily unavailable":
+                                xlog.warn("%s send errno.EAGAIN %r", s1, e)
+                                time.sleep(0.1)
+                                sent = 0
+                            else:
+                                self.close(s1, "w")
+                                continue
                         except Exception as e:
                             if sys.version_info[0] == 3 and isinstance(e, BlockingIOError):
                                 # This error happened on upload large file or speed test
                                 # Just ignore this error and will be fine
-                                # xlog.debug("%s sent BlockingIOError %r", s1, e)
+                                xlog.debug("%s sent BlockingIOError %r", s1, e)
                                 sent = 0
                             else:
                                 # xlog.debug("%s sent e:%r", s1, e)

code/default/smart_router/local/smart_route.py

@@ -497,6 +497,8 @@ def handle_ip_proxy(sock, ip, port, client_address):
             rule = "direct"
         elif g.config.pac_policy == "all_X-Tunnel":
             rule = "socks"
+        elif g.config.pac_policy == "all_Direct":
+            rule = "direct"
 
     if rule:
         return try_loop("ip user", [rule], sock, ip, port, client_address)
@@ -556,6 +558,8 @@ def handle_domain_proxy(sock, host, port, client_address, left_buf=""):
             rule = "gae"
         elif utils.check_ip_valid(host) and utils.is_private_ip(host):
             rule = "direct"
+        elif g.config.pac_policy == "all_Direct":
+            rule = "direct"
 
     if not rule and (g.config.bypass_speedtest and g.gfwlist.in_speedtest_whitelist(host)):
         xlog.debug("speedtest %s", host)