From 28177b0d38821668dd73972f4c831d28a539680f Mon Sep 17 00:00:00 2001 From: KOLANICH Date: Tue, 12 Jan 2021 16:52:23 +0300 Subject: [PATCH] Implemented SOCKS5 proxies and fixed h2.settings.ENABLE_PUSH to h2.settings.SettingCodes.ENABLE_PUSH. The code was taken from urllib3 and adapted to hyper. --- LICENSE | 1 + hyper/common/connection.py | 17 ++++++++-- hyper/contrib.py | 66 ++++++++++++++++++++++++++---------- hyper/http11/connection.py | 66 ++++++++++++++++++++++++++---------- hyper/http20/connection.py | 69 +++++++++++++++++++++++++++----------- 5 files changed, 162 insertions(+), 57 deletions(-) diff --git a/LICENSE b/LICENSE index 7ef4aca0..a351a2a2 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,7 @@ The MIT License (MIT) Copyright (c) 2014 Cory Benfield, Google Inc +Copyright (c) 2008-2020 Andrey Petrov and urllib3 contributors (see https://github.com/urllib3/urllib3/blob/master/CONTRIBUTORS.txt) (socks5 code was borrowed from there) Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/hyper/common/connection.py b/hyper/common/connection.py index 855994f8..81a81c04 100644 --- a/hyper/common/connection.py +++ b/hyper/common/connection.py @@ -46,6 +46,7 @@ class HTTPConnection(object): :param proxy_port: (optional) The proxy port to connect to. If not provided and one also isn't provided in the ``proxy_host`` parameter, defaults to 8080. + :param proxy_type: (optional) type of the proxy to use. Allows usage of socks proxies :param proxy_headers: (optional) The headers to send to a proxy. """ def __init__(self, @@ -57,6 +58,7 @@ def __init__(self, ssl_context=None, proxy_host=None, proxy_port=None, + proxy_type=None, proxy_headers=None, timeout=None, **kwargs): @@ -66,14 +68,14 @@ def __init__(self, self._h1_kwargs = { 'secure': secure, 'ssl_context': ssl_context, 'proxy_host': proxy_host, 'proxy_port': proxy_port, - 'proxy_headers': proxy_headers, 'enable_push': enable_push, + 'proxy_headers': proxy_headers, "proxy_type": proxy_type, 'enable_push': enable_push, 'timeout': timeout } self._h2_kwargs = { 'window_manager': window_manager, 'enable_push': enable_push, 'secure': secure, 'ssl_context': ssl_context, 'proxy_host': proxy_host, 'proxy_port': proxy_port, - 'proxy_headers': proxy_headers, + 'proxy_headers': proxy_headers, "proxy_type": proxy_type, 'timeout': timeout } @@ -150,6 +152,17 @@ def get_response(self, *args, **kwargs): return self._conn.get_response(1) + def reanimate(self): + """Reanimate connection reset because of proxy""" + if hasattr(self, "streams"): + for stream in list(self.streams.values()): + stream.remote_closed = True + stream.local_closed = True + self._conn.close() + self._conn = HTTP11Connection( + self._host, self._port, **self._h1_kwargs, + ) + # The following two methods are the implementation of the context manager # protocol. def __enter__(self): # pragma: no cover diff --git a/hyper/contrib.py b/hyper/contrib.py index 79aa7d12..e1abf0b3 100644 --- a/hyper/contrib.py +++ b/hyper/contrib.py @@ -56,11 +56,24 @@ def get_connection(self, host, port, scheme, cert=None, verify=True, ssl_context = init_context(cert_path=verify, cert=cert) if proxy: + proxy = prepend_scheme_if_needed(proxy, 'http') proxy_headers = self.proxy_headers(proxy) - proxy_netloc = urlparse(proxy).netloc + parsed = urlparse(proxy) + proxy_netloc = parsed.netloc + proxy_host_port = proxy_netloc.split(":") + if len(proxy_host_port) == 2: + proxy_host, proxy_port = proxy_host_port + proxy_port = int(proxy_port) + elif len(proxy_port_host) == 1: + raise ValueError("Specify proxy port!") + else: + raise ValueError("Invalid proxy netloc: ", repr(proxy_netloc)) + proxy_type = parsed.scheme else: proxy_headers = None - proxy_netloc = None + proxy_host = None + proxy_port = None + proxy_type = None # We put proxy headers in the connection_key, because # ``proxy_headers`` method might be overridden, so we can't @@ -68,7 +81,7 @@ def get_connection(self, host, port, scheme, cert=None, verify=True, proxy_headers_key = (frozenset(proxy_headers.items()) if proxy_headers else None) connection_key = (host, port, scheme, cert, verify, - proxy_netloc, proxy_headers_key) + proxy_host, proxy_port, proxy_type, proxy_headers_key) try: conn = self.connections[connection_key] except KeyError: @@ -78,9 +91,13 @@ def get_connection(self, host, port, scheme, cert=None, verify=True, secure=secure, window_manager=self.window_manager, ssl_context=ssl_context, - proxy_host=proxy_netloc, + proxy_host=proxy_host, + proxy_port=proxy_port, proxy_headers=proxy_headers, - timeout=timeout) + proxy_type=proxy_type, + timeout=timeout, + enable_push=self.enable_push + ) self.connections[connection_key] = conn return conn @@ -95,6 +112,12 @@ def send(self, request, stream=False, cert=None, verify=True, proxies=None, proxy = prepend_scheme_if_needed(proxy, 'http') parsed = urlparse(request.url) + + # Build the selector. + selector = parsed.path + selector += '?' + parsed.query if parsed.query else '' + selector += '#' + parsed.fragment if parsed.fragment else '' + conn = self.get_connection( parsed.hostname, parsed.port, @@ -104,18 +127,27 @@ def send(self, request, stream=False, cert=None, verify=True, proxies=None, proxy=proxy, timeout=timeout) - # Build the selector. - selector = parsed.path - selector += '?' + parsed.query if parsed.query else '' - selector += '#' + parsed.fragment if parsed.fragment else '' - - conn.request( - request.method, - selector, - request.body, - request.headers - ) - resp = conn.get_response() + def do_req(): + conn.request( + request.method, + selector, + request.body, + request.headers + ) + resp = conn.get_response() + return conn, resp + + retried = 0 + max_retries = 1 + while True: + try: + conn, resp = do_req() + break + except ConnectionAbortedError as e: + if retried < max_retries: + conn.reanimate() + else: + raise r = self.build_response(request, resp) diff --git a/hyper/http11/connection.py b/hyper/http11/connection.py index 4311d307..15e1e984 100644 --- a/hyper/http11/connection.py +++ b/hyper/http11/connection.py @@ -101,7 +101,7 @@ class HTTP11Connection(object): def __init__(self, host, port=None, secure=None, ssl_context=None, proxy_host=None, proxy_port=None, proxy_headers=None, - timeout=None, **kwargs): + proxy_type=None, timeout=None, **kwargs): if port is None: self.host, self.port = to_host_port_tuple(host, default_port=80) else: @@ -133,11 +133,14 @@ def __init__(self, host, port=None, secure=None, ssl_context=None, self.proxy_host, self.proxy_port = to_host_port_tuple( proxy_host, default_port=8080 ) + self.proxy_type = proxy_type elif proxy_host: - self.proxy_host, self.proxy_port = proxy_host, proxy_port + self.proxy_host, self.proxy_port, self.proxy_type = proxy_host, proxy_port, proxy_type else: self.proxy_host = None self.proxy_port = None + self.proxy_type = None + raise ValueError("No proxy was set!") self.proxy_headers = proxy_headers #: The size of the in-memory buffer used to store data from the @@ -169,22 +172,48 @@ def connect(self): connect_timeout = self._timeout read_timeout = self._timeout - if self.proxy_host and self.secure: - # Send http CONNECT method to a proxy and acquire the socket - sock = _create_tunnel( - self.proxy_host, - self.proxy_port, - self.host, - self.port, - proxy_headers=self.proxy_headers, - timeout=self._timeout - ) - elif self.proxy_host: - # Simple http proxy - sock = socket.create_connection( - (self.proxy_host, self.proxy_port), - timeout=connect_timeout - ) + if self.proxy_host: + if self.proxy_type.startswith("socks"): + import socks + rdns = (self.proxy_type[-1]=="h") + # any error will result in silently connecting without a proxy. + # IDK why it is done this way + if not rdns: + raise ValueError("RDNS is disabled. Proxying dns queries is disabled. NSA is spying you.") + if rdns and self.proxy_type.startswith("socks4"): + raise ValueError("RDNS is not supported for socks4. socks.create_connection ignores it silently.") + if not isinstance(self.proxy_host, str): + raise ValueError("self.proxy_host", repr(self.proxy_host), "is not str") + if not isinstance(self.proxy_port, int): + raise ValueError("self.proxy_port", repr(self.proxy_port), "is not int") + socks_version_char = self.proxy_type[5] + sock = socks.create_connection( + (self.host, self.port), + proxy_type=getattr(socks, "PROXY_TYPE_SOCKS" + socks_version_char), + proxy_addr=self.proxy_host, + proxy_port=self.proxy_port, + #proxy_username=username, + #proxy_password=password, + proxy_rdns=rdns, + ) + elif self.proxy_host and self.secure: + # Send http CONNECT method to a proxy and acquire the socket + sock = _create_tunnel( + self.proxy_host, + self.proxy_port, + self.host, + self.port, + proxy_headers=self.proxy_headers, + timeout=self._timeout + ) + elif self.proxy_host: + # Simple http proxy + sock = socket.create_connection( + (self.proxy_host, self.proxy_port), + timeout=connect_timeout + ) + else: + raise Exception("Unsupported proxy type: "+repr(proxy_type)) else: sock = socket.create_connection((self.host, self.port), timeout=connect_timeout) @@ -454,6 +483,7 @@ def _send_file_like_obj(self, fobj): return + def close(self): """ Closes the connection. This closes the socket and then abandons the diff --git a/hyper/http20/connection.py b/hyper/http20/connection.py index b8be292b..0d2981fa 100644 --- a/hyper/http20/connection.py +++ b/hyper/http20/connection.py @@ -101,7 +101,7 @@ class HTTP20Connection(object): def __init__(self, host, port=None, secure=None, window_manager=None, enable_push=False, ssl_context=None, proxy_host=None, - proxy_port=None, force_proto=None, proxy_headers=None, + proxy_port=None, proxy_type=None, force_proto=None, proxy_headers=None, timeout=None, **kwargs): """ Creates an HTTP/2 connection to a specific server. @@ -126,11 +126,13 @@ def __init__(self, host, port=None, secure=None, window_manager=None, self.proxy_host, self.proxy_port = to_host_port_tuple( proxy_host, default_port=8080 ) + self.proxy_type = proxy_type elif proxy_host: - self.proxy_host, self.proxy_port = proxy_host, proxy_port + self.proxy_host, self.proxy_port, self.proxy_type = proxy_host, proxy_port, proxy_type else: self.proxy_host = None self.proxy_port = None + self.proxy_type = None self.proxy_headers = proxy_headers #: The size of the in-memory buffer used to store data from the @@ -353,22 +355,49 @@ def connect(self): connect_timeout = self._timeout read_timeout = self._timeout - if self.proxy_host and self.secure: - # Send http CONNECT method to a proxy and acquire the socket - sock = _create_tunnel( - self.proxy_host, - self.proxy_port, - self.host, - self.port, - proxy_headers=self.proxy_headers, - timeout=self._timeout - ) - elif self.proxy_host: - # Simple http proxy - sock = socket.create_connection( - (self.proxy_host, self.proxy_port), - timeout=connect_timeout - ) + if self.proxy_host: + if self.proxy_type.startswith("socks"): + import socks + rdns = (self.proxy_type[-1]=="h") + # any error will result in silently connecting without a proxy. + # IDK why it is done this way. + if not rdns: + raise ValueError("RDNS is disabled. Proxying dns queries is disabled. NSA is spying you.") + if rdns and self.proxy_type.startswith("socks4"): + raise ValueError("RDNS is not supported for socks4. socks.create_connection ignores it silently.") + if not isinstance(self.proxy_host, str): + raise ValueError("self.proxy_host", repr(self.proxy_host), "is not str") + if not isinstance(self.proxy_port, int): + raise ValueError("self.proxy_port", repr(self.proxy_port), "is not int") + socks_version_char = self.proxy_type[5] + sock = socks.create_connection( + (self.host, self.port), + proxy_type=getattr(socks, "PROXY_TYPE_SOCKS" + socks_version_char), + proxy_addr=self.proxy_host, + proxy_port=self.proxy_port, + #proxy_username=username, + #proxy_password=password, + proxy_rdns=rdns, + ) + #sock.getsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1) + elif self.proxy_host and self.secure: + # Send http CONNECT method to a proxy and acquire the socket + sock = _create_tunnel( + self.proxy_host, + self.proxy_port, + self.host, + self.port, + proxy_headers=self.proxy_headers, + timeout=self._timeout + ) + elif self.proxy_host: + # Simple http proxy + sock = socket.create_connection( + (self.proxy_host, self.proxy_port), + timeout=connect_timeout + ) + else: + raise Exception("Unsupported proxy type: "+repr(proxy_type)) else: sock = socket.create_connection((self.host, self.port), timeout=connect_timeout) @@ -403,7 +432,7 @@ def _connect_upgrade(self, sock): with self._conn as conn: conn.initiate_upgrade_connection() conn.update_settings( - {h2.settings.ENABLE_PUSH: int(self._enable_push)} + {h2.settings.SettingCodes.ENABLE_PUSH: int(self._enable_push)} ) self._send_outstanding_data() @@ -424,7 +453,7 @@ def _send_preamble(self): with self._conn as conn: conn.initiate_connection() conn.update_settings( - {h2.settings.ENABLE_PUSH: int(self._enable_push)} + {h2.settings.SettingCodes.ENABLE_PUSH: int(self._enable_push)} ) self._send_outstanding_data()