modify function

Author: f0cu3

LittleFileScan.py

@@ -17,13 +17,15 @@
 import Queue
 import threading
 import argparse
+import multiprocessing
+from libs.common import *
 from libs.GenerateDict import ProcessDic
 
 class myFileScan:
     def __init__(self, url, custom_extion="php",full_scan=False, verbose=True):
         # the url only be www.iqiyi.com or 127.0.0.1:8080
-        self.url = url
-        self.compeleUrl()
+        self.url = compeleUrl(url)
+
         # self.url_queue = Queue.Queue()
         self.file_queue = Queue.Queue()
         self.lock = threading.Lock()
@@ -45,7 +47,8 @@ def __init__(self, url, custom_extion="php",full_scan=False, verbose=True):
         self.error_content_length = 0
         self.dir_in_content = False
         self.error_location = ""
-        self.check_404()
+
+
 
 
         # available dirs
@@ -56,6 +59,11 @@ def __init__(self, url, custom_extion="php",full_scan=False, verbose=True):
 
         self.generateFileAndDir()
 
+        # check 404
+        self.checkUrl()
+        self.check_404(types="file")
+        self.checkFile()
+
 
 
 
@@ -87,34 +95,37 @@ def generateFileAndDir(self):
         # default generate 188 dirs and 868 files(bak & tmp)
         # so next, we fuzz dirs, and add possible dirs, then we generate dirs and files
 
-    def check_404(self):
+    def check_404(self, types="dir"):
         error_dir = "this_is_error_dirs_test/"
         error_files = "this_is_error_files_test/hello.html"
 
 
         # Stitching url
-        _ = self.url + "/" + error_files
+        if types == "dir":
+            _ = self.url + "/" + error_dir
+        elif types == "file":
+            _ = self.url + "/" + error_files
+        else:
+            raise Exception("[-] [Error] [check_404] types not Identify")
         try:
-            resp = requests.get(_, headers=headers, timeout=timeout, allow_redirects=False, verify=allow_ssl_verify)
-            _content = resp.content.decode("utf-8", "ignore")
-            _content = _content.replace("\r\n", "").replace(" ", "")
-            print _content
-            print resp
+            resp = requests.get(_, headers=headers, timeout=5, allow_redirects=True, verify=allow_ssl_verify)
+            _content = decode_response_text(resp.content)
+            #_content = _content.replace("\r\n", "").replace(" ", "")
 
             self.error_status_code = resp.status_code
 
             # if 302 or 301, get Location
-            if resp.status_code in [301, 302] and "Location" in resp.headers:
-                self.error_location = resp.headers["Location"]
-
-            if self.error_pattern.match(_content):
+            if resp.url != _:
+                self.error_location = resp.url
+            else:
+                self.error_location = ""
+            if self.error_pattern.findall(_content):
                 self.error_flag = True
-
-            if error_dir in _content:
-                self.dir_in_content = True
-                self.error_content_length = len(_content) - len(_content)
             else:
-                self.error_content_length = len(_content)
+                self.error_flag = False
+            # if resp.status_code in [301, 302] and "Location" in resp.headers:
+            #     self.error_location = resp.headers["Location"]
+            self.error_content_length = len(_content)
         except Exception as e:
             self.has_404 = False
             print "[-] [Error] [myFileScan] [check_404] Request Error " + str(e)
@@ -151,54 +162,50 @@ def verifyAlive(self, dirs, types="dir", compress=False):
                     return False
 
             else:
-                resp = requests.get(_url, headers=headers, timeout=timeout, allow_redirects=False, verify=allow_ssl_verify)
-                _content = resp.content.decode("utf-8", "ignore")
-                _content = _content.replace("\r\n", "").replace(" ", "")
+                resp = requests.get(_url, headers=headers, timeout=timeout, allow_redirects=True, verify=allow_ssl_verify)
+                _content = decode_response_text(resp.content)
                 # 判断是否在400, 404, 501, 502, 503, 505,如果是直接返回False
                 if self.verbose:
                     print "[+] [Info] [myFileScan] [verifyAlive] verify: {:25} status_code: {}".format(_url, resp.status_code)
-                if resp.status_code in [400, 403, 404, 405, 500, 501, 502, 503, 505]:
+                if resp.status_code in [400, 403, 404, 414, 405, 500, 501, 502, 503, 505]:
                     return False
 
                 if resp.status_code in [301, 302]:
                     if "Location" in resp.headers:
                         if resp.headers["Location"] == self.error_location:
                             return False
+                
+                # 直接匹配错误标识
+                if self.error_flag:
+                    if self.error_pattern.findall(_content):
+                        return False
+                    else:
+                        return True
+
+                
                 # 如果有404错误页面的响应
                 if self.has_404:
                     # 如果返回码不是404, 但是判断是否是与error_status_code
                     if resp.status_code == self.error_status_code:
-                        # 判断是否有404标志
-                        if self.error_flag:
-                            if self.error_pattern.match(_content):
-                                return False
-                            else:
-                                return True
-                        # 如果没有404标志,那么对比长度
+                        mins = min(self.error_content_length, len(_content))
+                        if mins == 0:
+                            mins = 10.0
+                        if abs(float(self.error_content_length, len(_content))) / mins > 0.3:
+                            return True
                         else:
-                            if self.dir_in_content:
-                                l_content = len(_content) - len(dirs)
-                                if abs(l_content - self.error_content_length) < 10:
-                                    return False
-                                else:
-                                    return True
-                            else:
-                                if abs(len(_content) - self.error_content_length) < 10:
-                                    return False
-                                else:
-                                    return True
+                            return False 
 
                     # 如果不在上边，且不和error_code相等，那么先认为为True
                     else:
-                        return True
+                        return False
                 else:
                     # 如果check_404请求失败，怎么办呢？, 先return True
-                    return True
+                    return False
 
         except Exception as e:
             # 如果出错了，认为是True, 即404
             print "[-] [Error] [myFileScan] [verifyAlive] " + str(e)
-            return True
+            return False
 
         # 这样判断，首先判断是否是404,如果是，那么404标志设置为true
         #
@@ -207,7 +214,7 @@ def verifyAlive(self, dirs, types="dir", compress=False):
         # 如果无匹配，那么看错误长度 error_length, 错误长度在一定的相差范围内，则认为是不存在的
 
 
-
+    """
     def compeleUrl(self):
         # judge if self.url contains ":"
         scheme = "http"
@@ -228,7 +235,7 @@ def compeleUrl(self):
             self.url = scheme + "://" + self.url
         # last, remove last "/"
         self.url = self.url.rstrip("/")
-
+    """
 
 
     # 先写一块吧，回头再拆开
@@ -237,7 +244,8 @@ def checkUrl(self):
             if self.verifyAlive(webdir):
                 self.available_dirs.append(webdir)
         #self.available_dirs.append("/")
-
+        if not self.available_dirs:
+            self.available_dirs.append("/")
         print self.available_dirs
         print "that's all available_dirs"
         #time.sleep(10)
@@ -274,8 +282,10 @@ def checkFile(self):
 
 
     def runFuzz(self):
+
         while (not self.file_queue.empty()) and self.STOP_ME == False:
             _ = self.file_queue.get()
+            # print "[runFuzz] Url:\t" + str(_[0])
             result = self.verifyAlive(_[0], types="file", compress=_[1])
             if result:
                 self.lock.acquire()
@@ -286,34 +296,74 @@ def runFuzz(self):
 
 def parseArgs():
     parser =  argparse.ArgumentParser()
-    parser.add_argument("--host", help="the target host, MUST HAVE")
+    parser.add_argument("--host", help="the target host")
     parser.add_argument("--ext", help="the extend name, default php", default="php")
     parser.add_argument("-v", help="show more detail when running", action="store_true")
+    parser.add_argument("-f", help="file contains ip:port or subDomais each line")
     parser.add_argument("--full", help="Use All Dict (May be more False positives and take more time)", action="store_true")
     parser.add_argument("-t", "--threadnum", help="the number of thread count, default 15", default=15)
     args = parser.parse_args()
-    if args.host is None:
+    if args.host is None and args.f is None:
+        print "[--host/-f ] Must Contains One"
         parser.print_usage()
         exit(0)
     else:
         return args
 
 
+def check_url_alive(url):
+    url = compeleUrl(url)
+    print "[check_url_alive]:\t" + url
+    try:
+        resp = requests.get(url, headers={"User-Agent": "check_url_alive", "Connection": "Close"}, timeout=5, allow_redirects=False, verify=False)
+        return True
+    except:
+        return False
 
 
-
-
-def ScanApi(host, custom_extion="php", verbose=True, full_scan=False):
-    a = myFileScan(args.host, custom_extion=custom_extion, verbose=verbose, full_scan=full_scan)
-    a.checkUrl()
-    a.checkFile()
+def compeleUrl(url):
+    # judge if self.url contains ":"
+    scheme = "http"
+    if ":" in url:
+        # judge if 443
+        if url.split(":")[-1] == "443":
+            scheme = "https"
+        else:
+            scheme = "http"
+    # judge if start with "://"
+    if url.startswith("://"):
+        url = url[3:]
+    elif url.startswith("//"):
+        url = url[2:]
+
+    # now judge if start with http
+    if not url.startswith("http"):
+        url = scheme + "://" + url
+    # last, remove last "/"
+    url = url.rstrip("/")
+    return url
+
+
+def ScanApi(host, args):
+    print str(host) + str(args)
+    custom_extion = args.ext
+    verbose = args.v
+    full_scan = args.full
+    a = myFileScan(host, custom_extion=custom_extion, verbose=verbose, full_scan=full_scan)
+    threads = []
     for i in range(int(args.threadnum)):
         thd = threading.Thread(target=a.runFuzz)
+        threads.append(thd)
         thd.setDaemon(True)
         thd.start()
 
     while True:
-        if threading.activeCount() <= 1:
+        count = 0
+        for thd in threads:
+            if thd.is_alive():
+                count += 1
+
+        if count == 0:
             break
         else:
             try:
@@ -323,41 +373,32 @@ def ScanApi(host, custom_extion="php", verbose=True, full_scan=False):
                 a.STOP_ME = True
 
 
-if __name__ == '__main__':
-    """
-    Usage = \"""
-        python LittleFileScan.py host ext
-
-        i.e.  python LittleFileScan.py www.iqiyi.com jsp
-        i.e.  python LittleFileScan.py www.iqiyi.com
-        i.e.  python LittleFileScan.py 127.0.0.1:8832
-        \"""
+def batchFileScan(args):
+    custom_extion = args.ext
+    verbose = args.v
+    full_scan = args.full
+    pool = multiprocessing.Pool(4)
+    target_queue = Queue.Queue()
+    if args.f:
+        with open(args.f, "r") as f:
+            x = f.readlines()
+
+        for line in x:
+            if not check_url_alive(line.strip()):
+                continue
+            print "[batchFileScan] add \t" + line.strip()
+            target_queue.put(line.strip())
 
-    if len(sys.argv) > 3:
-        print Usage
-        sys.exit(-1)
-    if len(sys.argv) == 3:
-        host = sys.argv[1]
-        custom_extion = sys.argv[2]
-        ScanApi(host, custom_extion=custom_extion)
-    elif len(sys.argv) == 2:
-        host = sys.argv[1]
-        ScanApi(host)
-    else:
-        print Usage
-        sys.exit(-1)
-    """
-
-    full_scan = False
-    custom_extion ="php"
-    verbose = False
-    args = parseArgs()
-    if args.full:
-        full_scan = True
-    if args.ext:
-        custom_extion = args.ext
-    if args.v:
-        verbose = True
-    ScanApi(args.host, custom_extion=custom_extion, verbose=verbose, full_scan=full_scan)
+    while not target_queue.empty():
+        host = target_queue.get(timeout=1)
+        print "[test] [Host]:\t" + host
+        pool.apply(ScanApi, (host, args))
+    
+    pool.close()
+    pool.join()
+    print 'All subprocesses done.'
 
 
+if __name__ == '__main__':
+    args = parseArgs()
+    batchFileScan(args)

LittleFileScan_old.py

@@ -113,6 +113,11 @@ def check_404(self, types="dir"):
                 self.error_location = resp.url
             else:
                 self.error_location = ""
+            # 判断是否有404标志
+            if self.error_pattern.findall(_content):
+                self.error_flag = True
+            else:
+                self.error_flag = False
             # if resp.status_code in [301, 302] and "Location" in resp.headers:
             #     self.error_location = resp.headers["Location"]
             self.error_content_length = len(_content)
@@ -166,27 +171,24 @@ def verifyAlive(self, dirs, types="dir", compress=False):
                             return False
 
                 # 直接匹配错误标识
-                if self.error_pattern.findall(_content):
-                    return False
+                if self.error_flag:
+                    if self.error_pattern.findall(_content):
+                        return False
+                    else:
+                        return True
+
+                
                 # 如果有404错误页面的响应
                 if self.has_404:
                     # 如果返回码不是404, 但是判断是否是与error_status_code
                     if resp.status_code == self.error_status_code:
-                        # 判断是否有404标志
-                        if self.error_flag:
-                            if self.error_pattern.findall(_content):
-                                return False
-                            else:
-                                return True
-                        # 如果没有404标志,那么对比长度
+                        mins = min(self.error_content_length, len(_content))
+                        if mins == 0:
+                            mins = 10.0
+                        if abs(float(self.error_content_length, len(_content))) / mins > 0.3:
+                            return True
                         else:
-                            mins = min(self.error_content_length, len(_content))
-                            if mins == 0:
-                                mins = 10.0
-                            if abs(float(self.error_content_length, len(_content))) / mins > 0.3:
-                                return True
-                            else:
-                                return False 
+                            return False 
 
                     # 如果不在上边，且不和error_code相等，那么先认为为True
                     else:

config.py

@@ -58,7 +58,7 @@
 # -------------------------------------------------
 
 # 超时时间
-timeout = 10
+timeout = 6
 
 # 是否允许URL重定向
 allow_redirects = True

config.pyc

@@ -1,5 +1,5 @@
 %DEPEN_NAME%{re=exrex:[0-9]}$
-%DEPEN_NAME%{re=exrex:(200[0-9])|(201[0-5])}$
+%DEPEN_NAME%{re=exrex:(201[5-7])}$
 %DEPEN_NAME%{re=exrex:(!?!|!!!)|(@?@|@@@)|123}$
 %EXT%{re=exrex:(!?!|!!!)|(@?@|@@@)|123}$
 %DOMAIN%{re=exrex:(!?!|!!!)|(@?@|@@@)|123}$

dict/dependents.lst

@@ -1,5 +1,5 @@
-/{date=year:2010-2015}$
-/{date=year_mon:201001-201512}$
+/{date=year:2015-2017}$
+/{date=year_mon:201501-201712}$
 /a3
 /abstract
 /account

dict/directory.lst

@@ -0,0 +1,10 @@
+blogs.autohome.com.cn
+platform.autohome.com.cn
+welcome.autohome.com.cn
+science.autohome.com.cn
+misto.autohome.com.cn
+event.autohome.com.cn
+services.autohome.com.cn
+video.autohome.com.cn
+media.autohome.com.cn
+stats.autohome.com.cn