class DownLoad: def process_request(self, request, spider): ''' 请求需要被下载,经过所有下载器中间件的process_request调用 :param request: :param spider: :return: None: 后续中间件去下载 Response: 停止process_request的执行,开始执行,process_response Request: 停止中间件的执行,将Request重新发给调度器 raise IgnoreRequest, 停止process_request的执行,开始执行process_exception ''' def process_response(self, request, response, spider): ''' spider处理完成,返回时调用 :param request: :param response: :param spider: :return: response对象: 转交给其他中间件process_response Request对象: 停止中间件,request会被重新调度下载 raise IgnoreRequest异常: 调用Request.errback ''' def process_exception(self, request, exception, spider): ''' 当下载处理器(download handler)或者process_request()(下载中间件)出异常,会自动执行此方法 :param request: :param exception: :param spider: :return: None: 交给后面的中间件继续处理异常,如果都处理不了,那么最后会报错 response: 停止后续process_exception方法 Request: 停止中间件,request将会被重新调用下载 '''
DOWNLOADER_MIDDLEWARES = { 'chouti.middlewaresDownLoad': 543, } #在配置文件当中,将注释取消的,将自己定义的中间件添加进去