Replaced urllib with httpx. Supports HTTP/2, more convenient and secure.

Fixes exception raising.

Author: KOLANICH

ipwhois/experimental.py

@@ -112,7 +112,7 @@ def get_bulk_asn_whois(addresses=None, retry_count=3, timeout=120):
 
 def bulk_lookup_rdap(addresses=None, inc_raw=False, retry_count=3, depth=0,
                      excluded_entities=None, rate_limit_timeout=60,
-                     socket_timeout=10, asn_timeout=240, proxy_openers=None):
+                     socket_timeout=10, asn_timeout=240, http_clients=None):
     """
     The function for bulk retrieving and parsing whois information for a list
     of IP addresses via HTTP (RDAP). This bulk lookup method uses bulk
@@ -138,8 +138,8 @@ def bulk_lookup_rdap(addresses=None, inc_raw=False, retry_count=3, depth=0,
             connections in seconds. Defaults to 10.
         asn_timeout (:obj:`int`): The default timeout for bulk ASN lookups in
             seconds. Defaults to 240.
-        proxy_openers (:obj:`list` of :obj:`OpenerDirector`): Proxy openers
-            for single/rotating proxy support. Defaults to None.
+        http_clients (:obj:`list` of :obj:`httpx.Client`): httpx clients
+            for single/rotating proxy and fingerprint support. Defaults to None.
 
     Returns:
         namedtuple:
@@ -209,11 +209,11 @@ def bulk_lookup_rdap(addresses=None, inc_raw=False, retry_count=3, depth=0,
     }
     asn_parsed_results = {}
 
-    if proxy_openers is None:
+    if http_clients is None:
 
-        proxy_openers = [None]
+        http_clients = [None]
 
-    proxy_openers_copy = iter(proxy_openers)
+    http_clients_copy = iter(http_clients)
 
     # Make sure addresses is unique
     unique_ip_list = list(unique_everseen(addresses))
@@ -347,19 +347,19 @@ def bulk_lookup_rdap(addresses=None, inc_raw=False, retry_count=3, depth=0,
 
                         rate_tracker[rir]['count'] += 1
 
-                    # Get the next proxy opener to use, or None
+                    # Get the next HTTP client object to use, or None
                     try:
 
-                        opener = next(proxy_openers_copy)
+                        client = next(http_clients_copy)
 
                     # Start at the beginning if all have been used
                     except StopIteration:
 
-                        proxy_openers_copy = iter(proxy_openers)
-                        opener = next(proxy_openers_copy)
+                        http_clients_copy = iter(http_clients)
+                        client = next(http_clients_copy)
 
                     # Instantiate the objects needed for the RDAP lookup
-                    net = Net(ip, timeout=socket_timeout, proxy_opener=opener)
+                    net = Net(ip, timeout=socket_timeout, http_client=client)
                     rdap = RDAP(net)
 
                     try:

ipwhois/ipwhois.py

@@ -40,14 +40,14 @@ class IPWhois:
             An IPv4 or IPv6 address
         timeout (:obj:`int`): The default timeout for socket connections in
             seconds. Defaults to 5.
-        proxy_opener (:obj:`urllib.request.OpenerDirector`): The request for
-            proxy support. Defaults to None.
+        http_client (:obj:`httpx.Client`): HTTP client object. Proxies are here.
+            Defaults to None.
     """
 
-    def __init__(self, address, timeout=5, proxy_opener=None):
+    def __init__(self, address, timeout=5, http_client=None):
 
         self.net = Net(
-            address=address, timeout=timeout, proxy_opener=proxy_opener
+            address=address, timeout=timeout, http_client=http_client
         )
         self.ipasn = IPASN(self.net)
 
@@ -61,7 +61,7 @@ def __init__(self, address, timeout=5, proxy_opener=None):
     def __repr__(self):
 
         return 'IPWhois({0}, {1}, {2})'.format(
-            self.address_str, str(self.timeout), repr(self.net.opener)
+            self.address_str, str(self.timeout), repr(self.net.http_client)
         )
 
     def lookup_whois(self, inc_raw=False, retry_count=3, get_referral=False,

ipwhois/net.py

@@ -49,22 +49,11 @@
                         IPv4Address,
                         IPv6Address)
 
-try:  # pragma: no cover
-    from urllib.request import (OpenerDirector,
-                                ProxyHandler,
-                                build_opener,
-                                Request,
-                                URLError,
-                                HTTPError)
-    from urllib.parse import urlencode
-except ImportError:  # pragma: no cover
-    from urllib2 import (OpenerDirector,
-                         ProxyHandler,
-                         build_opener,
-                         Request,
-                         URLError,
-                         HTTPError)
-    from urllib import urlencode
+from httpx import (Client,
+                   HTTPStatusError,
+                   TransportError,
+                   InvalidURL)
+from urllib.parse import urlencode
 
 log = logging.getLogger(__name__)
 
@@ -101,15 +90,15 @@ class Net:
             An IPv4 or IPv6 address
         timeout (:obj:`int`): The default timeout for socket connections in
             seconds. Defaults to 5.
-        proxy_opener (:obj:`urllib.request.OpenerDirector`): The request for
-            proxy support. Defaults to None.
+        http_client (:obj:`httpx.client`): httpx client allows you to customize
+            usage of HTTP by this lib. Proxies are also configured via it.
 
     Raises:
         IPDefinedError: The address provided is defined (does not need to be
             resolved).
     """
 
-    def __init__(self, address, timeout=5, proxy_opener=None):
+    def __init__(self, address, timeout=5, http_client=None):
 
         # IPv4Address or IPv6Address
         if isinstance(address, IPv4Address) or isinstance(
@@ -129,15 +118,10 @@ def __init__(self, address, timeout=5, proxy_opener=None):
         self.dns_resolver.timeout = timeout
         self.dns_resolver.lifetime = timeout
 
-        # Proxy opener.
-        if isinstance(proxy_opener, OpenerDirector):
+        if not http_client:
+            http_client = Client()
 
-            self.opener = proxy_opener
-
-        else:
-
-            handler = ProxyHandler()
-            self.opener = build_opener(handler)
+        self.http_client = http_client
 
         # IP address in string format for use in queries.
         self.address_str = self.address.__str__()
@@ -709,10 +693,10 @@ def get_http_json(self, url=None, retry_count=3, rate_limit_timeout=120,
 
             return d
 
-        except HTTPError as e:  # pragma: no cover
+        except HTTPStatusError as e:  # pragma: no cover
 
             # RIPE is producing this HTTP error rather than a JSON error.
-            if e.code == 429:
+            if e.response.status_code == 429:
 
                 log.debug('HTTP query rate limit exceeded.')
 
@@ -737,7 +721,7 @@ def get_http_json(self, url=None, retry_count=3, rate_limit_timeout=120,
                 raise HTTPLookupError('HTTP lookup failed for {0} with error '
                                       'code {1}.'.format(url, str(e.code)))
 
-        except (URLError, socket.timeout, socket.error) as e:
+        except (TransportError,) as e:
 
             log.debug('HTTP query socket error: {0}'.format(e))
             if retry_count > 0:
@@ -865,22 +849,10 @@ def get_http_raw(self, url=None, retry_count=3, headers=None,
             # Create the connection for the HTTP query.
             log.debug('HTTP query for {0} at {1}'.format(
                 self.address_str, url))
-            try:
-                # Py 2 inspection alert bypassed by using kwargs dict.
-                conn = Request(url=url, data=enc_form_data, headers=headers,
-                               **{'method': request_type})
-            except TypeError:  # pragma: no cover
-                conn = Request(url=url, data=enc_form_data, headers=headers)
-            data = self.opener.open(conn, timeout=self.timeout)
-
-            try:
-                d = data.readall().decode('ascii', 'ignore')
-            except AttributeError:  # pragma: no cover
-                d = data.read().decode('ascii', 'ignore')
-
-            return str(d)
+            return self.http_client.request(url=url, data=enc_form_data, headers=headers,
+                                        **{'method': request_type}).text
 
-        except (URLError, socket.timeout, socket.error) as e:
+        except (InvalidURL, TransportError) as e:
 
             log.debug('HTTP query socket error: {0}'.format(e))
             if retry_count > 0:

ipwhois/scripts/ipwhois_cli.py

@@ -31,12 +31,7 @@
 from ipwhois.hr import (HR_ASN, HR_RDAP, HR_RDAP_COMMON, HR_WHOIS,
                         HR_WHOIS_NIR)
 
-try:  # pragma: no cover
-    from urllib.request import (ProxyHandler,
-                                build_opener)
-except ImportError:  # pragma: no cover
-    from urllib2 import (ProxyHandler,
-                         build_opener)
+from httpx import Client
 
 # CLI ANSI rendering
 ANSI = {
@@ -348,18 +343,15 @@ class IPWhoisCLI:
             An IPv4 or IPv6 address
         timeout (:obj:`int`): The default timeout for socket connections in
             seconds. Defaults to 5.
-        proxy_http (:obj:`urllib.request.OpenerDirector`): The request for
-            proxy HTTP support or None.
-        proxy_https (:obj:`urllib.request.OpenerDirector`): The request for
-            proxy HTTPS support or None.
+        http_client (:obj:`httpx.Client`): The httpx.Client objects.
+            Proxies and not only are here.
     """
 
     def __init__(
         self,
         addr,
         timeout,
-        proxy_http,
-        proxy_https
+        http_client
     ):
 
         self.addr = addr
@@ -368,29 +360,28 @@ def __init__(
         handler_dict = None
         if proxy_http is not None:
 
-            handler_dict = {'http': proxy_http}
+            handler_dict = {'http://*': proxy_http}
 
         if proxy_https is not None:
 
             if handler_dict is None:
 
-                handler_dict = {'https': proxy_https}
+                handler_dict = {'https://*': proxy_https}
 
             else:
 
-                handler_dict['https'] = proxy_https
+                handler_dict['https://*'] = proxy_https
 
         if handler_dict is None:
 
-            self.opener = None
+            self.http_client = None
         else:
 
-            handler = ProxyHandler(handler_dict)
-            self.opener = build_opener(handler)
+            self.http_client = Client(proxies=handler_dict)
 
         self.obj = IPWhois(address=self.addr,
                            timeout=self.timeout,
-                           proxy_opener=self.opener)
+                           http_client=self.http_client)
 
     def generate_output_header(self, query_type='RDAP'):
         """

ipwhois/tests/online/test_experimental.py

@@ -2,6 +2,7 @@
 from ipwhois.tests import TestCommon
 from ipwhois.exceptions import (ASNLookupError)
 from ipwhois.experimental import (get_bulk_asn_whois, bulk_lookup_rdap)
+from httpx import Client
 
 LOG_FORMAT = ('[%(asctime)s] [%(levelname)s] [%(filename)s:%(lineno)s] '
               '[%(funcName)s()] %(message)s')
@@ -39,18 +40,9 @@ def test_get_bulk_asn_whois(self):
 
     def test_bulk_lookup_rdap(self):
 
-        try:
-            from urllib.request import (OpenerDirector,
-                                        ProxyHandler,
-                                        build_opener)
-        except ImportError:
-            from urllib2 import (OpenerDirector,
-                                 ProxyHandler,
-                                 build_opener)
+        from httpx import Client
 
-        handler = ProxyHandler()
-        opener = build_opener(handler)
-        bulk_lookup_rdap(addresses=['74.125.225.229'], proxy_openers=[opener])
+        bulk_lookup_rdap(addresses=['74.125.225.229'], http_client=Client())
 
         ips = [
             '74.125.225.229',  # ARIN

ipwhois/tests/online/test_ipwhois.py

@@ -125,10 +125,7 @@ def test_lookup_whois(self):
             break
 
     def test_lookup_rdap(self):
-        try:
-            from urllib.request import ProxyHandler, build_opener
-        except ImportError:
-            from urllib2 import ProxyHandler, build_opener
+        from httpx import Client
 
         ips = [
             '74.125.225.229',  # ARIN
@@ -169,8 +166,7 @@ def test_lookup_rdap(self):
             except Exception as e:
                 self.fail('Unexpected exception raised: {0}'.format(e))
 
-        handler = ProxyHandler({'http': 'http://0.0.0.0:80/'})
-        opener = build_opener(handler)
+        http_client = Client(proxies={'http://*': 'http://0.0.0.0:80/'})
         result = IPWhois(address='74.125.225.229', timeout=0,
-                         proxy_opener=opener)
+                         http_client=http_client)
         self.assertRaises(ASNRegistryError, result.lookup_rdap)

ipwhois/tests/test_net.py

@@ -38,20 +38,12 @@ def test_timeout(self):
         result = Net('74.125.225.229')
         self.assertIsInstance(result.timeout, int)
 
-    def test_proxy_opener(self):
-        try:
-            from urllib.request import (OpenerDirector,
-                                        ProxyHandler,
-                                        build_opener)
-        except ImportError:
-            from urllib2 import (OpenerDirector,
-                                 ProxyHandler,
-                                 build_opener)
+    def test_http_client(self):
+        from httpx import Client
 
         result = Net('74.125.225.229')
-        self.assertIsInstance(result.opener, OpenerDirector)
+        self.assertIsInstance(result.http_client, Client)
 
-        handler = ProxyHandler()
-        opener = build_opener(handler)
-        result = Net(address='74.125.225.229', proxy_opener=opener)
-        self.assertIsInstance(result.opener, OpenerDirector)
+        client = Client()
+        result = Net(address='74.125.225.229', http_client=client)
+        self.assertIsInstance(result.http_client, Client)

setup.py

@@ -67,7 +67,7 @@
 
 PACKAGE_DATA = {'ipwhois': ['data/*.xml', 'data/*.csv']}
 
-INSTALL_REQUIRES = ['dnspython<=2.0.0', 'ipaddr==2.2.0;python_version<"3.3"']
+INSTALL_REQUIRES = ['dnspython<=2.0.0', 'ipaddr==2.2.0;python_version<"3.3"', 'httpx']
 
 setup(
     name=NAME,