I'm trying to fetch HTTPS pages through a proxy with digest authentication. I use Python 3.4.3 and urllib. Third-party libraries such as requests are not in my consideration. Here are my codes:
import urllib.request
class SimplePasswordManager(object):
def __init__(self, username, password):
self.username = username
self.password = password
def add_password(self, realm, uri, user, passwd):
pass
def find_user_password(self, realm, authuri):
return self.username, self.password
proxy_handler = urllib.request.ProxyHandler({
'http': '<proxy server ip>',
'https': '<proxy server ip>',
})
password_mgr = SimplePasswordManager('<username>', '<password>')
proxy_auth_handler = urllib.request.ProxyDigestAuthHandler(passwd=password_mgr)
opener = urllib.request.build_opener(proxy_auth_handler, proxy_handler)
req = opener.open('http://httpbin.org/ip')
print(req.read().decode('ascii'))
req = opener.open('https://httpbin.org/ip')
print(req.read().decode('ascii'))
And the results:
{
"origin": "<proxy server ip>"
}
Traceback (most recent call last):
File "/usr/lib/python3.4/urllib/request.py", line 1182, in do_open
h.request(req.get_method(), req.selector, req.data, headers)
File "/usr/lib/python3.4/http/client.py", line 1088, in request
self._send_request(method, url, body, headers)
File "/usr/lib/python3.4/http/client.py", line 1126, in _send_request
self.endheaders(body)
File "/usr/lib/python3.4/http/client.py", line 1084, in endheaders
self._send_output(message_body)
File "/usr/lib/python3.4/http/client.py", line 922, in _send_output
self.send(msg)
File "/usr/lib/python3.4/http/client.py", line 857, in send
self.connect()
File "/usr/lib/python3.4/http/client.py", line 1223, in connect
super().connect()
File "/usr/lib/python3.4/http/client.py", line 837, in connect
self._tunnel()
File "/usr/lib/python3.4/http/client.py", line 820, in _tunnel
message.strip()))
OSError: Tunnel connection failed: 407 Proxy Authentication Required
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "proxy_test.py", line 25, in <module>
req = opener.open('https://httpbin.org/ip')
File "/usr/lib/python3.4/urllib/request.py", line 463, in open
response = self._open(req, data)
File "/usr/lib/python3.4/urllib/request.py", line 481, in _open
'_open', req)
File "/usr/lib/python3.4/urllib/request.py", line 441, in _call_chain
result = func(*args)
File "/usr/lib/python3.4/urllib/request.py", line 1225, in https_open
context=self._context, check_hostname=self._check_hostname)
File "/usr/lib/python3.4/urllib/request.py", line 1184, in do_open
raise URLError(err)
urllib.error.URLError: <urlopen error Tunnel connection failed: 407 Proxy Authentication Required>
Seems HTTP connection works while HTTPS not. Are my codes correct? How to handle HTTPS connections correctly?