添加代理
方式一:
import os
import scrapy
from scrapy.http import Request
class ChoutiSpider(scrapy.Spider):
name = 'chouti'
allowed_domains = ['chouti.com']
start_urls = ['https://dig.chouti.com/']
def start_requests(self):
os.environ['HTTP_PROXY'] = "http://192.168.11.11"
for url in self.start_urls:
yield Request(url=url,callback=self.parse)
def parse(self, response):
print(response)
方式二
import random
import base64
import six
def to_bytes(text, encoding=None, errors='strict'):
"""Return the binary representation of `text`. If `text`
is already a bytes object, return it as-is."""
if isinstance(text, bytes):
return text
if not isinstance(text, six.string_types):
raise TypeError('to_bytes must receive a unicode, str or bytes '
'object, got %s' % type(text).__name__)
if encoding is None:
encoding = 'utf-8'
return text.encode(encoding, errors)
class MyProxyDownloaderMiddleware(object):
def process_request(self, request, spider):
proxy_list = [
{'ip_port': '111.11.228.75:80', 'user_pass': 'xxx:123'},
{'ip_port': '120.198.243.22:80', 'user_pass': ''},
{'ip_port': '111.8.60.9:8123', 'user_pass': ''},
{'ip_port': '101.71.27.120:80', 'user_pass': ''},
{'ip_port': '122.96.59.104:80', 'user_pass': ''},
{'ip_port': '122.224.249.122:8088', 'user_pass': ''},
]
proxy = random.choice(proxy_list)
if proxy['user_pass'] is not None:
request.meta['proxy'] = to_bytes("http://%s" % proxy['ip_port'])
encoded_user_pass = base64.encodestring(to_bytes(proxy['user_pass']))
request.headers['Proxy-Authorization'] = to_bytes('Basic ' + encoded_user_pass)
else:
request.meta['proxy'] = to_bytes("http://%s" % proxy['ip_port'])
配置:
DOWNLOADER_MIDDLEWARES = {
# 'xiaohan.middlewares.MyProxyDownloaderMiddleware': 543,
}
https
from scrapy.core.downloader.contextfactory import ScrapyClientContextFactory
from twisted.internet.ssl import (optionsForClientTLS, CertificateOptions, PrivateCertificate)
class MySSLFactory(ScrapyClientContextFactory):
def getCertificateOptions(self):
from OpenSSL import crypto
v1 = crypto.load_privatekey(crypto.FILETYPE_PEM, open('/Users/wupeiqi/client.key.unsecure', mode='r').read())
v2 = crypto.load_certificate(crypto.FILETYPE_PEM, open('/Users/wupeiqi/client.pem', mode='r').read())
return CertificateOptions(
privateKey=v1, # pKey对象
certificate=v2, # X509对象
verify=False,
method=getattr(self, 'method', getattr(self, '_ssl_method', None))
)
配置:
DOWNLOADER_HTTPCLIENTFACTORY = "scrapy.core.downloader.webclient.ScrapyHTTPClientFactory"
DOWNLOADER_CLIENTCONTEXTFACTORY = "xiaohan.middlewares.MySSLFactory"
信号
extends.py
from scrapy import signals
class MyExtension(object):
def __init__(self):
pass
@classmethod
def from_crawler(cls, crawler):
obj = cls()
# 在爬虫打开时,触发spider_opened信号相关的所有函数:xxxxxxxxxxx
crawler.signals.connect(obj.xxxxxxxxxxx1, signal=signals.spider_opened)
# 在爬虫关闭时,触发spider_closed信号相关的所有函数:xxxxxxxxxxx
crawler.signals.connect(obj.uuuuuuuuuu, signal=signals.spider_closed)
return obj
def xxxxxxxxxxx1(self, spider):
print('open')
def uuuuuuuuuu(self, spider):
print('close')
return obj
配置:
EXTENSIONS = {
'xiaohan.extends.MyExtension':500,
}
自定制命令
在spiders同级创建任意目录,如:commands
在其中创建 crawlall.py 文件 (此处文件名就是自定义的命令)
from scrapy.commands import ScrapyCommand
from scrapy.utils.project import get_project_settings
class Command(ScrapyCommand):
requires_project = True
def syntax(self):
return '[options]'
def short_desc(self):
return 'Runs all of the spiders'
def run(self, args, opts):
spider_list = self.crawler_process.spiders.list()
for name in spider_list:
self.crawler_process.crawl(name, **opts.__dict__)
self.crawler_process.start()
crawlall.py
在settings.py 中添加配置 COMMANDS_MODULE = ‘项目名称.目录名称’
在项目目录执行命令:scrapy crawlall