from faker import Faker class MySpiderMiddleware(object): def __init__(self): self.fake = Faker() def process_request(self,request,spider): request.headers.setdefault('User-Agent',self.fake.user_agent())
DOWNLOADER_MIDDLEWARES = {
#'middle.middlewares.MyCustomDownloaderMiddleware': 543,
'middle.middlewares.MySpiderMiddleware': 100,
'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware': None,
}