2017年8月31日 DEBUG

Author: ZKeeer

Config.py

@@ -1,8 +1,12 @@
 DBName = "PROXIES.db"  # 数据库名称
 TabelName = "IPPORT"  # 表
 Column1 = "IP_PORT"  # 列1
-TestTimeOut = 10  # 检测IP可用性设置的超时
+
+TestTimeOut = 10  # 检测IP可用性设置的超时，
+                  # 对IP质量要求不高，就把值设的高一点儿。这样可用IP就会增多
 MaxThreads = 64  # 最大线程数，依据电脑性能修改，性能好的电脑可以设置高一点
+                 # 最好设置为2的n次方，别问我为什么，我也不知道，这是玄学
+
 TestUrl = "https://www.baidu.com/"  # 用以检测的网站
 
 # 头部代理S

GetIP.py

@@ -31,6 +31,8 @@ def GetPageContent(tar_url):
 
 
 def GetIP():
+    global d
+    global ip_list
     thread_list = []
     ips = []
 
@@ -50,17 +52,21 @@ def GetIP():
 
     for item in d.keys():
         ips.append(item)
+    d.clear()
     ProxiesDataBase.AddItems(ips)
 
 
 def RefreshDB():
+    global d
+    global ip_list
     ip_list = ProxiesDataBase.GetItems()
     thread_list = []
     ips = []
 
-    if not ip_list:
+    if ip_list.__len__() < 1:
         return
 
+
     for index in range(0, Config.MaxThreads):
         thread_list.append(Thread(target=VerifyIp))
     for item in thread_list:
@@ -69,14 +75,18 @@ def RefreshDB():
         item.join()
 
     ProxiesDataBase.ClearItems()
+
     for item in d.keys():
         ips.append(item)
+    d.clear()
     ProxiesDataBase.AddItems(ips)
 
 
 def VerifyIp():
+    global d
     while ip_list:
         tmp_ip_port = ip_list.pop(0)
+        print("verify ip: {}".format(tmp_ip_port))
         proxies = {"http": "http://{}".format(tmp_ip_port), "https": "https://{}".format(tmp_ip_port)}
         try:
             url_content = get(Config.TestUrl,

ProxiesDataBase.py

@@ -1,18 +1,16 @@
 # table IPPORT
 # ip_port TEXT NOT NULL
-import random
 import sqlite3
-from threading import Thread
 import traceback
+
 import Config
-import GetIP
 
 
 def InitDB():
     db_conn = sqlite3.connect(Config.DBName)
     try:
         db_conn.execute(
-            """CREATE TABLE IF NOT EXISTS {} (IP_PORT TEXT NOT NULL PRIMARY KEY);""".format(Config.TabelName))
+            """CREATE TABLE IF NOT EXISTS {} (IP_PORT TEXT NOT NULL);""".format(Config.TabelName))
         db_conn.commit()
         return True
     except BaseException as e:
@@ -34,15 +32,18 @@ def AddItem(ip_port):
         traceback.print_exc()
     db_conn.close()
 
+
 def AddItems(ip_list):
+    if ip_list.__len__() < 1:
+        return
+
     sql_str = """INSERT INTO IPPORT VALUES """
 
     for item in ip_list:
         sql_str += ("('{}'),".format(item))
     index = sql_str.__len__()
     sql_str = sql_str[0:index - 1]
     sql_str += ";"
-
     db_conn = sqlite3.connect(Config.DBName)
     try:
         db_conn.execute(sql_str)
@@ -83,14 +84,10 @@ def GetItems():
     db_conn = sqlite3.connect(Config.DBName)
     db_cur = db_conn.cursor()
     try:
-        tmp = db_cur.execute("""SELECT * FROM {};""".format(Config.TabelName)).fetchall()
-        for item in tmp:
+        for item in db_cur.execute("""SELECT * FROM {};""".format(Config.TabelName)).fetchall():
             ip_list.append(item[0])
     except BaseException as e:
         traceback.print_exc()
     finally:
         db_conn.close()
         return ip_list
-
-
-

demo.py

@@ -1,5 +1,5 @@
 import sqlite3
-
+import GetIP
 import Config
 import ProxiesDataBase
 import Util