Skip to content

Commit

Permalink
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Replaced urllib with httpx. Supports HTTP/2, more convenient and secure.
Browse files Browse the repository at this point in the history
Fixes exception raising.
KOLANICH committed Feb 16, 2021
1 parent 212d4c2 commit f0a3b4a
Showing 13 changed files with 102 additions and 186 deletions.
8 changes: 4 additions & 4 deletions ipwhois/asn.py
Original file line number Diff line number Diff line change
@@ -165,7 +165,7 @@ def parse_fields_dns(self, response):
except Exception as e:

raise ASNParseError('Parsing failed for "{0}" with exception: {1}.'
''.format(response, e)[:100])
''.format(response, e)[:100]) from e

return ret

@@ -222,7 +222,7 @@ def parse_fields_verbose_dns(self, response):
except Exception as e:

raise ASNParseError('Parsing failed for "{0}" with exception: {1}.'
''.format(response, e)[:100])
''.format(response, e)[:100]) from e

return ret

@@ -279,7 +279,7 @@ def parse_fields_whois(self, response):
except Exception as e:

raise ASNParseError('Parsing failed for "{0}" with exception: {1}.'
''.format(response, e)[:100])
''.format(response, e)[:100]) from e

return ret

@@ -379,7 +379,7 @@ def parse_fields_http(self, response, extra_org_map=None):
except Exception as e: # pragma: no cover

raise ASNParseError('Parsing failed for "{0}" with exception: {1}.'
''.format(response, e)[:100])
''.format(response, e)[:100]) from e

return asn_data

28 changes: 14 additions & 14 deletions ipwhois/experimental.py
Original file line number Diff line number Diff line change
@@ -103,16 +103,16 @@ def get_bulk_asn_whois(addresses=None, retry_count=3, timeout=120):

else:

raise ASNLookupError('ASN bulk lookup failed.')
raise ASNLookupError('ASN bulk lookup failed.') from e

except: # pragma: no cover
except BaseException as e: # pragma: no cover

raise ASNLookupError('ASN bulk lookup failed.')
raise ASNLookupError('ASN bulk lookup failed.') from e


def bulk_lookup_rdap(addresses=None, inc_raw=False, retry_count=3, depth=0,
excluded_entities=None, rate_limit_timeout=60,
socket_timeout=10, asn_timeout=240, proxy_openers=None):
socket_timeout=10, asn_timeout=240, http_clients=None):
"""
The function for bulk retrieving and parsing whois information for a list
of IP addresses via HTTP (RDAP). This bulk lookup method uses bulk
@@ -138,8 +138,8 @@ def bulk_lookup_rdap(addresses=None, inc_raw=False, retry_count=3, depth=0,
connections in seconds. Defaults to 10.
asn_timeout (:obj:`int`): The default timeout for bulk ASN lookups in
seconds. Defaults to 240.
proxy_openers (:obj:`list` of :obj:`OpenerDirector`): Proxy openers
for single/rotating proxy support. Defaults to None.
http_clients (:obj:`list` of :obj:`httpx.Client`): httpx clients
for single/rotating proxy and fingerprint support. Defaults to None.
Returns:
namedtuple:
@@ -209,11 +209,11 @@ def bulk_lookup_rdap(addresses=None, inc_raw=False, retry_count=3, depth=0,
}
asn_parsed_results = {}

if proxy_openers is None:
if http_clients is None:

proxy_openers = [None]
http_clients = [None]

proxy_openers_copy = iter(proxy_openers)
http_clients_copy = iter(http_clients)

# Make sure addresses is unique
unique_ip_list = list(unique_everseen(addresses))
@@ -347,19 +347,19 @@ def bulk_lookup_rdap(addresses=None, inc_raw=False, retry_count=3, depth=0,

rate_tracker[rir]['count'] += 1

# Get the next proxy opener to use, or None
# Get the next HTTP client object to use, or None
try:

opener = next(proxy_openers_copy)
client = next(http_clients_copy)

# Start at the beginning if all have been used
except StopIteration:

proxy_openers_copy = iter(proxy_openers)
opener = next(proxy_openers_copy)
http_clients_copy = iter(http_clients)
client = next(http_clients_copy)

# Instantiate the objects needed for the RDAP lookup
net = Net(ip, timeout=socket_timeout, proxy_opener=opener)
net = Net(ip, timeout=socket_timeout, http_client=client)
rdap = RDAP(net)

try:
10 changes: 5 additions & 5 deletions ipwhois/ipwhois.py
Original file line number Diff line number Diff line change
@@ -40,14 +40,14 @@ class IPWhois:
An IPv4 or IPv6 address
timeout (:obj:`int`): The default timeout for socket connections in
seconds. Defaults to 5.
proxy_opener (:obj:`urllib.request.OpenerDirector`): The request for
proxy support. Defaults to None.
http_client (:obj:`httpx.Client`): HTTP client object. Proxies are here.
Defaults to None.
"""

def __init__(self, address, timeout=5, proxy_opener=None):
def __init__(self, address, timeout=5, http_client=None):

self.net = Net(
address=address, timeout=timeout, proxy_opener=proxy_opener
address=address, timeout=timeout, http_client=http_client
)
self.ipasn = IPASN(self.net)

@@ -61,7 +61,7 @@ def __init__(self, address, timeout=5, proxy_opener=None):
def __repr__(self):

return 'IPWhois({0}, {1}, {2})'.format(
self.address_str, str(self.timeout), repr(self.net.opener)
self.address_str, str(self.timeout), repr(self.net.http_client)
)

def lookup_whois(self, inc_raw=False, retry_count=3, get_referral=False,
121 changes: 45 additions & 76 deletions ipwhois/net.py
Original file line number Diff line number Diff line change
@@ -49,22 +49,11 @@
IPv4Address,
IPv6Address)

try: # pragma: no cover
from urllib.request import (OpenerDirector,
ProxyHandler,
build_opener,
Request,
URLError,
HTTPError)
from urllib.parse import urlencode
except ImportError: # pragma: no cover
from urllib2 import (OpenerDirector,
ProxyHandler,
build_opener,
Request,
URLError,
HTTPError)
from urllib import urlencode
from httpx import (Client,
HTTPStatusError,
TransportError,
InvalidURL)
from urllib.parse import urlencode

log = logging.getLogger(__name__)

@@ -101,15 +90,15 @@ class Net:
An IPv4 or IPv6 address
timeout (:obj:`int`): The default timeout for socket connections in
seconds. Defaults to 5.
proxy_opener (:obj:`urllib.request.OpenerDirector`): The request for
proxy support. Defaults to None.
http_client (:obj:`httpx.client`): httpx client allows you to customize
usage of HTTP by this lib. Proxies are also configured via it.
Raises:
IPDefinedError: The address provided is defined (does not need to be
resolved).
"""

def __init__(self, address, timeout=5, proxy_opener=None):
def __init__(self, address, timeout=5, http_client=None):

# IPv4Address or IPv6Address
if isinstance(address, IPv4Address) or isinstance(
@@ -129,15 +118,10 @@ def __init__(self, address, timeout=5, proxy_opener=None):
self.dns_resolver.timeout = timeout
self.dns_resolver.lifetime = timeout

# Proxy opener.
if isinstance(proxy_opener, OpenerDirector):
if not http_client:
http_client = Client()

self.opener = proxy_opener

else:

handler = ProxyHandler()
self.opener = build_opener(handler)
self.http_client = http_client

# IP address in string format for use in queries.
self.address_str = self.address.__str__()
@@ -229,13 +213,13 @@ def get_asn_dns(self):
raise ASNLookupError(
'ASN lookup failed (DNS {0}) for {1}.'.format(
e.__class__.__name__, self.address_str)
)
) from e

except: # pragma: no cover
except BaseException as e: # pragma: no cover

raise ASNLookupError(
'ASN lookup failed for {0}.'.format(self.address_str)
)
) from e

def get_asn_verbose_dns(self, asn=None):
"""
@@ -271,13 +255,13 @@ def get_asn_verbose_dns(self, asn=None):
raise ASNLookupError(
'ASN lookup failed (DNS {0}) for {1}.'.format(
e.__class__.__name__, asn)
)
) from e

except: # pragma: no cover
except BaseException as e: # pragma: no cover

raise ASNLookupError(
'ASN lookup failed for {0}.'.format(asn)
)
) from e

def get_asn_whois(self, retry_count=3):
"""
@@ -337,13 +321,13 @@ def get_asn_whois(self, retry_count=3):

raise ASNLookupError(
'ASN lookup failed for {0}.'.format(self.address_str)
)
) from e

except: # pragma: no cover
except BaseException as e: # pragma: no cover

raise ASNLookupError(
'ASN lookup failed for {0}.'.format(self.address_str)
)
) from e

def get_asn_http(self, retry_count=3):
"""
@@ -391,9 +375,9 @@ def get_asn_http(self, retry_count=3):

raise ASNLookupError(
'ASN lookup failed for {0}.'.format(self.address_str)
)
) from e

except:
except BaseException as e:

raise ASNLookupError(
'ASN lookup failed for {0}.'.format(self.address_str)
@@ -505,11 +489,11 @@ def get_asn_origin_whois(self, asn_registry='radb', asn=None,

raise

except: # pragma: no cover
except BaseException as e: # pragma: no cover

raise WhoisLookupError(
'ASN origin WHOIS lookup failed for {0}.'.format(asn)
)
) from e

def get_whois(self, asn_registry='arin', retry_count=3, server=None,
port=43, extra_blacklist=None):
@@ -624,7 +608,7 @@ def get_whois(self, asn_registry='arin', retry_count=3, server=None,

raise WhoisLookupError(
'WHOIS lookup failed for {0}.'.format(self.address_str)
)
) from e

except WhoisRateLimitError: # pragma: no cover

@@ -634,7 +618,7 @@ def get_whois(self, asn_registry='arin', retry_count=3, server=None,

raise

except: # pragma: no cover
except BaseException as e: # pragma: no cover

raise WhoisLookupError(
'WHOIS lookup failed for {0}.'.format(self.address_str)
@@ -673,12 +657,9 @@ def get_http_json(self, url=None, retry_count=3, rate_limit_timeout=120,
# Create the connection for the whois query.
log.debug('HTTP query for {0} at {1}'.format(
self.address_str, url))
conn = Request(url, headers=headers)
data = self.opener.open(conn, timeout=self.timeout)
try:
d = json.loads(data.readall().decode('utf-8', 'ignore'))
except AttributeError: # pragma: no cover
d = json.loads(data.read().decode('utf-8', 'ignore'))
r = self.http_client.get(url, headers=headers)
r.raise_for_status()
d = r.json

try:
# Tests written but commented out. I do not want to send a
@@ -709,10 +690,10 @@ def get_http_json(self, url=None, retry_count=3, rate_limit_timeout=120,

return d

except HTTPError as e: # pragma: no cover
except HTTPStatusError as e: # pragma: no cover

# RIPE is producing this HTTP error rather than a JSON error.
if e.code == 429:
if e.response.status_code == 429:

log.debug('HTTP query rate limit exceeded.')

@@ -730,14 +711,14 @@ def get_http_json(self, url=None, retry_count=3, rate_limit_timeout=120,
raise HTTPRateLimitError(
'HTTP lookup failed for {0}. Rate limit '
'exceeded, wait and try again (possibly a '
'temporary block).'.format(url))
'temporary block).'.format(url)) from e

else:

raise HTTPLookupError('HTTP lookup failed for {0} with error '
'code {1}.'.format(url, str(e.code)))
'code {1}.'.format(url, str(e.code))) from e

except (URLError, socket.timeout, socket.error) as e:
except (TransportError,) as e:

log.debug('HTTP query socket error: {0}'.format(e))
if retry_count > 0:
@@ -753,15 +734,15 @@ def get_http_json(self, url=None, retry_count=3, rate_limit_timeout=120,
else:

raise HTTPLookupError('HTTP lookup failed for {0}.'.format(
url))
url)) from e

except (HTTPLookupError, HTTPRateLimitError) as e: # pragma: no cover

raise e

except: # pragma: no cover
except BaseException as e: # pragma: no cover

raise HTTPLookupError('HTTP lookup failed for {0}.'.format(url))
raise HTTPLookupError('HTTP lookup failed for {0}.'.format(url)) from e

def get_host(self, retry_count=3):
"""
@@ -819,11 +800,11 @@ def get_host(self, retry_count=3):
'Host lookup failed for {0}.'.format(self.address_str)
)

except: # pragma: no cover
except BaseException as e: # pragma: no cover

raise HostLookupError(
'Host lookup failed for {0}.'.format(self.address_str)
)
) from e

def get_http_raw(self, url=None, retry_count=3, headers=None,
request_type='GET', form_data=None):
@@ -865,22 +846,10 @@ def get_http_raw(self, url=None, retry_count=3, headers=None,
# Create the connection for the HTTP query.
log.debug('HTTP query for {0} at {1}'.format(
self.address_str, url))
try:
# Py 2 inspection alert bypassed by using kwargs dict.
conn = Request(url=url, data=enc_form_data, headers=headers,
**{'method': request_type})
except TypeError: # pragma: no cover
conn = Request(url=url, data=enc_form_data, headers=headers)
data = self.opener.open(conn, timeout=self.timeout)

try:
d = data.readall().decode('ascii', 'ignore')
except AttributeError: # pragma: no cover
d = data.read().decode('ascii', 'ignore')

return str(d)
return self.http_client.request(url=url, data=enc_form_data, headers=headers,
**{'method': request_type}).text

except (URLError, socket.timeout, socket.error) as e:
except (InvalidURL, TransportError) as e:

log.debug('HTTP query socket error: {0}'.format(e))
if retry_count > 0:
@@ -896,12 +865,12 @@ def get_http_raw(self, url=None, retry_count=3, headers=None,
else:

raise HTTPLookupError('HTTP lookup failed for {0}.'.format(
url))
url)) from e

except HTTPLookupError as e: # pragma: no cover

raise e

except Exception: # pragma: no cover
except BaseException as e: # pragma: no cover

raise HTTPLookupError('HTTP lookup failed for {0}.'.format(url))
raise HTTPLookupError('HTTP lookup failed for {0}.'.format(url)) from e
20 changes: 10 additions & 10 deletions ipwhois/rdap.py
Original file line number Diff line number Diff line change
@@ -476,9 +476,9 @@ def __init__(self, json_result):

_RDAPCommon.__init__(self, json_result)

except ValueError:
except ValueError as e:

raise InvalidNetworkObject('JSON result must be a dict.')
raise InvalidNetworkObject('JSON result must be a dict.') from e

self.vars.update({
'start_address': None,
@@ -500,12 +500,12 @@ def parse(self):

self.vars['handle'] = self.json['handle'].strip()

except (KeyError, ValueError):
except (KeyError, ValueError) as e:

log.debug('Handle missing, json_output: {0}'.format(json.dumps(
self.json)))
raise InvalidNetworkObject('Handle is missing for RDAP network '
'object')
'object') from e

try:

@@ -529,12 +529,12 @@ def parse(self):
self.vars['start_address'] = self.json['startAddress'].strip()
self.vars['end_address'] = self.json['endAddress'].strip()

except (KeyError, ValueError, TypeError):
except (KeyError, ValueError, TypeError) as e:

log.debug('IP address data incomplete. Data parsed prior to '
'exception: {0}'.format(json.dumps(self.vars)))
raise InvalidNetworkObject('IP address data is missing for RDAP '
'network object.')
'network object.') from e

try:

@@ -587,9 +587,9 @@ def __init__(self, json_result):

_RDAPCommon.__init__(self, json_result)

except ValueError:
except ValueError as e:

raise InvalidEntityObject('JSON result must be a dict.')
raise InvalidEntityObject('JSON result must be a dict.') from e

self.vars.update({
'roles': None,
@@ -607,9 +607,9 @@ def parse(self):

self.vars['handle'] = self.json['handle'].strip()

except (KeyError, ValueError, TypeError):
except (KeyError, ValueError, TypeError) as e:

raise InvalidEntityObject('Handle is missing for RDAP entity')
raise InvalidEntityObject('Handle is missing for RDAP entity') from e

for v in ['roles', 'country']:

29 changes: 10 additions & 19 deletions ipwhois/scripts/ipwhois_cli.py
Original file line number Diff line number Diff line change
@@ -31,12 +31,7 @@
from ipwhois.hr import (HR_ASN, HR_RDAP, HR_RDAP_COMMON, HR_WHOIS,
HR_WHOIS_NIR)

try: # pragma: no cover
from urllib.request import (ProxyHandler,
build_opener)
except ImportError: # pragma: no cover
from urllib2 import (ProxyHandler,
build_opener)
from httpx import Client

# CLI ANSI rendering
ANSI = {
@@ -348,18 +343,15 @@ class IPWhoisCLI:
An IPv4 or IPv6 address
timeout (:obj:`int`): The default timeout for socket connections in
seconds. Defaults to 5.
proxy_http (:obj:`urllib.request.OpenerDirector`): The request for
proxy HTTP support or None.
proxy_https (:obj:`urllib.request.OpenerDirector`): The request for
proxy HTTPS support or None.
http_client (:obj:`httpx.Client`): The httpx.Client objects.
Proxies and not only are here.
"""

def __init__(
self,
addr,
timeout,
proxy_http,
proxy_https
http_client
):

self.addr = addr
@@ -368,29 +360,28 @@ def __init__(
handler_dict = None
if proxy_http is not None:

handler_dict = {'http': proxy_http}
handler_dict = {'http://*': proxy_http}

if proxy_https is not None:

if handler_dict is None:

handler_dict = {'https': proxy_https}
handler_dict = {'https://*': proxy_https}

else:

handler_dict['https'] = proxy_https
handler_dict['https://*'] = proxy_https

if handler_dict is None:

self.opener = None
self.http_client = None
else:

handler = ProxyHandler(handler_dict)
self.opener = build_opener(handler)
self.http_client = Client(proxies=handler_dict)

self.obj = IPWhois(address=self.addr,
timeout=self.timeout,
proxy_opener=self.opener)
http_client=self.http_client)

def generate_output_header(self, query_type='RDAP'):
"""
6 changes: 0 additions & 6 deletions ipwhois/tests/online/test_asn.py
Original file line number Diff line number Diff line change
@@ -24,8 +24,6 @@ def test_lookup(self):
self.assertIsInstance(ipasn.lookup(inc_raw=True), dict)
except ASNRegistryError:
pass
except AssertionError as e:
raise e
except Exception as e:
self.fail('Unexpected exception raised: {0}'.format(e))

@@ -80,10 +78,6 @@ def test_lookup(self):

raise e

except Exception as e:

self.fail('Unexpected exception raised: {0}'.format(e))

net = Net(address='74.125.225.229', timeout=0)
asnorigin = ASNOrigin(net)
self.assertRaises(ASNOriginLookupError, asnorigin.lookup, **dict(
18 changes: 3 additions & 15 deletions ipwhois/tests/online/test_experimental.py
Original file line number Diff line number Diff line change
@@ -2,6 +2,7 @@
from ipwhois.tests import TestCommon
from ipwhois.exceptions import (ASNLookupError)
from ipwhois.experimental import (get_bulk_asn_whois, bulk_lookup_rdap)
from httpx import Client

LOG_FORMAT = ('[%(asctime)s] [%(levelname)s] [%(filename)s:%(lineno)s] '
'[%(funcName)s()] %(message)s')
@@ -32,25 +33,14 @@ def test_get_bulk_asn_whois(self):
self.assertIsInstance(get_bulk_asn_whois(addresses=ips), str)
except ASNLookupError:
pass
except AssertionError as e:
raise e
except Exception as e:
self.fail('Unexpected exception raised: {0}'.format(e))

def test_bulk_lookup_rdap(self):

try:
from urllib.request import (OpenerDirector,
ProxyHandler,
build_opener)
except ImportError:
from urllib2 import (OpenerDirector,
ProxyHandler,
build_opener)
from httpx import Client

handler = ProxyHandler()
opener = build_opener(handler)
bulk_lookup_rdap(addresses=['74.125.225.229'], proxy_openers=[opener])
bulk_lookup_rdap(addresses=['74.125.225.229'], http_client=Client())

ips = [
'74.125.225.229', # ARIN
@@ -86,7 +76,5 @@ def test_bulk_lookup_rdap(self):

except ASNLookupError:
pass
except AssertionError as e:
raise e
except Exception as e:
self.fail('Unexpected exception raised: {0}'.format(e))
20 changes: 3 additions & 17 deletions ipwhois/tests/online/test_ipwhois.py
Original file line number Diff line number Diff line change
@@ -51,8 +51,6 @@ def test_lookup_whois(self):
except (ASNLookupError, ASNRegistryError, WhoisLookupError,
HTTPLookupError):
pass
except AssertionError as e:
raise e
except Exception as e:
self.fail('Unexpected exception raised: {0}'.format(e))

@@ -70,8 +68,6 @@ def test_lookup_whois(self):
inc_raw=True), dict)
except (ASNLookupError, ASNRegistryError, WhoisLookupError):
pass
except AssertionError as e:
raise e
except Exception as e:
self.fail('Unexpected exception raised: {0}'.format(e))

@@ -86,8 +82,6 @@ def test_lookup_whois(self):
extra_blacklist=['rwhois.cogentco.com']), dict)
except (ASNLookupError, ASNRegistryError, WhoisLookupError):
pass
except AssertionError as e:
raise e
except Exception as e:
self.fail('Unexpected exception raised: {0}'.format(e))

@@ -125,10 +119,7 @@ def test_lookup_whois(self):
break

def test_lookup_rdap(self):
try:
from urllib.request import ProxyHandler, build_opener
except ImportError:
from urllib2 import ProxyHandler, build_opener
from httpx import Client

ips = [
'74.125.225.229', # ARIN
@@ -164,13 +155,8 @@ def test_lookup_rdap(self):
except (ASNLookupError, ASNRegistryError, WhoisLookupError,
HTTPLookupError):
pass
except AssertionError as e:
raise e
except Exception as e:
self.fail('Unexpected exception raised: {0}'.format(e))

handler = ProxyHandler({'http': 'http://0.0.0.0:80/'})
opener = build_opener(handler)
http_client = Client(proxies={'http://*': 'http://0.0.0.0:80/'})
result = IPWhois(address='74.125.225.229', timeout=0,
proxy_opener=opener)
http_client=http_client)
self.assertRaises(ASNRegistryError, result.lookup_rdap)
2 changes: 1 addition & 1 deletion ipwhois/tests/stress/test_net.py
Original file line number Diff line number Diff line change
@@ -39,5 +39,5 @@ def test_get_http_json(self):
raise Exception('HTTPLookupError has been raised 5 times. '
'Likely cause is socket connection '
'timeouts. Quitting test to avoid an '
'endless loop.')
'endless loop.') from e
continue
20 changes: 6 additions & 14 deletions ipwhois/tests/test_net.py
Original file line number Diff line number Diff line change
@@ -38,20 +38,12 @@ def test_timeout(self):
result = Net('74.125.225.229')
self.assertIsInstance(result.timeout, int)

def test_proxy_opener(self):
try:
from urllib.request import (OpenerDirector,
ProxyHandler,
build_opener)
except ImportError:
from urllib2 import (OpenerDirector,
ProxyHandler,
build_opener)
def test_http_client(self):
from httpx import Client

result = Net('74.125.225.229')
self.assertIsInstance(result.opener, OpenerDirector)
self.assertIsInstance(result.http_client, Client)

handler = ProxyHandler()
opener = build_opener(handler)
result = Net(address='74.125.225.229', proxy_opener=opener)
self.assertIsInstance(result.opener, OpenerDirector)
client = Client()
result = Net(address='74.125.225.229', http_client=client)
self.assertIsInstance(result.http_client, Client)
4 changes: 0 additions & 4 deletions ipwhois/tests/test_nir.py
Original file line number Diff line number Diff line change
@@ -54,10 +54,6 @@ def test_lookup(self):

raise e

except Exception as e:

self.fail('Unexpected exception raised: {0}'.format(e))

self.assertRaises(NetError, NIRWhois, 'a')
self.assertRaises(KeyError, obj.lookup)
self.assertRaises(KeyError, obj.lookup, **dict(nir='a'))
2 changes: 1 addition & 1 deletion setup.py
Original file line number Diff line number Diff line change
@@ -67,7 +67,7 @@

PACKAGE_DATA = {'ipwhois': ['data/*.xml', 'data/*.csv']}

INSTALL_REQUIRES = ['dnspython<=2.0.0', 'ipaddr==2.2.0;python_version<"3.3"']
INSTALL_REQUIRES = ['dnspython<=2.0.0', 'ipaddr==2.2.0;python_version<"3.3"', 'httpx']

setup(
name=NAME,

0 comments on commit f0a3b4a

Please sign in to comment.