class UserAgentMiddleware(object):
""" Change User-Agent """
def process_request(self, request, spider):
agent = random.choice(agents)
request.headers["User-Agent"] = agent
class CookiesMiddleware(object):
""" Change Cookie """
def process_request(self, request, spider):
cookie = random.choice(cookies)
request.cookies = cookie
class ProxyMiddleware(object):
"""Everyone Request change proxy"""
def process_request(self, request, spider):
try:
# get ip address
PROXY_POOL_URL = 'xxxxxx'
response = requests.get(url=PROXY_POOL_URL)
if response.status_code == 200:
proxy = response.text.strip()
print('-------use---proxy------: http://',proxy)
request.meta['proxy'] = "http://" + proxy
except Exception as e :
print("-----Error:---Proxy---ERROR-----", e)
url : https://scrapy-chs.readthedocs.io/zh_CN/0.24/topics/downloader-middleware.html