爬虫代理设置

urllib 方式

def user_proxy(proxy_addr, url):
    import urllib.request
    proxy = urllib.request.ProxyHandler({'http': proxy_addr})
    opener = urllib.request.build_opener(proxy, urllib.request.HTTPHandler)
    urllib.request.install_opener(opener)
    data = urllib.request.urlopen(url).read().decode('utf-8')
    return data

proxy_addr = "114.82.109.134:8118"
data = user_proxy(proxy_addr, "https://www.baidu.com")
print(data)
print(len(data))

如果此时代理服务器地址失效或者填写错了代理服务器，则会发生错误。

requests 方式

import requests

ip, port = ("111.201.81.189", "8118")
url = 'http://www.163.com/'
proxy_url = "http://{0}:{1}".format(ip, port)
print(proxy_url)
proxy_dict = {
    "http": proxy_url
}
response = requests.get(url, proxies=proxy_dict)
html_doc = str(response.content, 'gbk')
print(html_doc)