最近在学习关于爬虫方面的知识,因为刚开始接触,还是萌新,所以有什么错误的地方,欢迎大家指出
from multiprocessing import Pool
from urllib.request import Request, urlopen
import re, pymongo
index = 0
class DouBanSpider(object):
client = pymongo.MongoClient('localhost')
db = client['dbmovie'] #进程池无法序列化pymongo对象,因为pymongo数据库中含有线程锁。
def __init__(self):
self.headers = {
'User-Agent': '这里添加自己的浏览器代理'
,'Cookie': '豆瓣需要登录后才能访问爬取信息所以要加上自己的Cookie'
}
self.tool = DataTool()
def get_list_html(self, page_num):
page_num = (page_num - 1) * 25
list_url = 'https://movie.douban.com/top250?start={}'.format(page_num)
request = Request(list_url, headers=self.headers)
try:
response = urlopen(request)
except Exception as e:
print('请求失败:地址{},原因{}'.format(list_url,e))
return None
else:
html = response.read().decode()
return html
def parse_list_html(self, html):
if html:
pattern = re.compile(r'<div class="hd">.*?<a href="(.*?)" class.*?>.*?', re.S)
detail_urls = re.findall(pattern, html)
# for detail_url in detail_urls:
# print(detail_url)
return detail_urls
else:
print('html源代码为None')
return None
def get_detail_html(self, detail_url):
request = Request(detail_url, headers=self.headers)
try:
response = urlopen(request)
except Exception as e:
print('请求失败:地址{},原因{}'.format(detail_url,e))
return None
else:
detail_html = response.read().decode()
return detail_html
def parse_detail_html(self, detail_html):
dic={}
data = re.findall(re.compile(
r'<h1>.*?<span property="v:itemreviewed">(.*?)</span>.*?<div id="info">.*?<span .*?><a href=.*?>(.*?)</a></span>.*?<span class=.*?><a href=.*?>(.*?)</a>.*?<span class="actor">.*?<span class=.*?><a href=.*?>(.*?)</a></span></span><br/>.*?<span class=.*?>.*?<span property=.*?>(.*?)</span><br/>.*?<span class=.*?>.*?</span> (.*?)<br/>.*?<span class=.*?>.*?</span>(.*?)<br/>.*?</span> <span .*?>(.*?)</span><br/>.*?<span class=.*?>.*?</span> <span property=.*? content=.*?>(.*?)</span>.*?<br/>.*?<span class="pl">.*?</span>(.*?)<br/>.*?', re.S), detail_html)[0]
global index
index = index+1
print(index, data)
print('影片名:', data[0])
print('导演:', data[1])
print('编剧:', data[2])
print('主演:', data[3])
print('类型:', data[4])
print('制片国家/地区:', data[5])
print('语言:', data[6])
print('上映日期:', data[7])
print('片长:', data[8])
print('又名:', data[9])
dic['影片名'] = data[0]
dic['导演'] = data[1]
dic['编剧'] = data[2]
dic['主演'] = data[3]
dic['类型'] = data[4]
dic['制片国家/地区'] = data[5]
dic['语言'] = data[6]
dic['上映日期'] = data[7]
dic['片长'] = data[8]
dic['又名'] = data[9]
self.db['movie'].insert_one(dic)
def start_spider(self, num):
i = 0
print('正在请求第{}页'.format(num))
list_html = self.get_list_html(num)
if list_html:
detail_urls = self.parse_list_html(list_html)
if detail_urls:
for detail_url in detail_urls:
i = i+1
if i != 164: # 164 因为二十二那部电影没有又名导致爬不出来所以手动过滤
detail_html = self.get_detail_html(detail_url)
else:
continue
if detail_html:
self.parse_detail_html(detail_html)
if __name__ == '__main__':
obj = DouBanSpider()
pool = Pool(1)
pool.map(obj.start_spider, [x for x in range(1, 10)])
pool.close()
pool.join()
运行结果:
写爬虫一定要有耐心,有时候爬不出来信息很有可能是因为正则表达式写错了!所以要细心。