效果图:
1.工先利其事必先利器,首先我们得下载相应的库:
pip install requests
pip install lxml
pip install xlwt
requests 向网页发送请求
lxml 处理xml文件(xpath)
xlwt 对Excel做写入操作
2.爬取b站热门视频的信息:
打开b站热门视频页面:
按f12
进入开发者选项,然后点击选中你要获取的页面信息,即可找到该信息在该HTML文件中的什么位置(这对我们用xpath获取元素属性和元素值很重要),例如:
代码如下:
# 爬取b站热门视频信息
def spider(video_list):
url = 'https://www.bilibili.com/ranking?spm_id_from=333.851.b_7072696d61727950616765546162.3'
html_data = requests.get(url).text
selector = html.fromstring(html_data)
infolist = selector.xpath('//li[@class="rank-item"]')
for item in infolist:
rank = "".join(item.xpath('./div[@class="num"]/text()'))
videolink = "".join(item.xpath('./div[@class="content"]/div[@class="info"]/a/@href'))
title = "".join(item.xpath('./div[@class="content"]/div[@class="info"]/a/text()'))
playinfo = "".join(item.xpath('./div[@class="content"]/div[@class="info"]/div[@class="detail"]/span/text()')).split("万")
play = playinfo[0] + "万"
comment = playinfo[1]
if comment.isdigit() == False:
comment += "万"
upname = "".join(item.xpath('./div[@class="content"]/div[@class="info"]/div[@class="detail"]/a/span/text()'))
uplink = "http:" + "".join(item.xpath('./div[@class="content"]/div[@class="info"]/div[@class="detail"]/a/@href'))
hot = "".join(item.xpath('./div[@class="content"]/div[@class="info"]/div[@class="pts"]/div/text()'))
video_list.append({
'rank': rank,
'videolink': videolink,
'title': title,
'play': play,
'comment': comment,
'upname': upname,
'uplink': uplink,
'hot': hot
})
return video_list
3.将我们拿到的信息集合(video_list)写入到Excel表格中:
xlwt的基本使用方法:
import xlwt
# 创建一个workbook (并设置编码)
workbook = xlwt.Workbook(encoding = 'utf-8')
# 创建一个worksheet
worksheet = workbook.add_sheet('My Worksheet')
# 写入excel
# 参数对应 行, 列, 值,(格式)
worksheet.write(1,0, label = 'this is test')
# 保存
#参数为你保存该Excel文件的路径
workbook.save('Excel_test.xls')
如果我们想要点击视频名或者up的名字可以跳转,那么我们就要使用Excel表格的HYPERLINK方法:
HYPERLINK(“http://www.baidu.com” ; “百度”)
百度为显示在单元格的信息,而前面的链接为跳转链接。
xlwt.Formula()方法需要传入一个字符串s,s=‘HYPERLINK(“http://www.baidu.com” ; “百度”)’。
代码如下:
# 将爬取到的数据写入Excel表格
def write_Excel(video_list):
print("将b站热门视频信息导入到Excel表格:")
workbook = xlwt.Workbook() # 定义workbook
sheet = workbook.add_sheet('b站热门视频') # 添加sheet
xstyle = xlwt.XFStyle() # 实例化表格样式对象
xstyle.alignment.horz = 0x02 # 字体居中
xstyle.alignment.vert = 0x01 # 字体居中
head = ['视频名', 'up主','排名', '热度','播放量','评论数'] # 表头
for h in range(len(head)):
sheet.write(0, h, head[h],xstyle) # 把表头写到Excel里面去
i = 1
for item in video_list:
# 向单元格(视频名)添加(该视频的)超链接
title_data = 'HYPERLINK("'+item["videolink"]+'";"'+item["title"]+'")' # 设置超链接
sheet.col(0).width = int(256 * len(title_data) * 3/5) # 设置列宽
sheet.write(i, 0, xlwt.Formula(title_data), xstyle)
name_data = 'HYPERLINK("' + item["uplink"] + '";"' + item["upname"] + '")' # 设置超链接
sheet.col(1).width = int(256 * len(title_data) * 3 / 10)
sheet.write(i, 1, xlwt.Formula(name_data), xstyle)
sheet.write(i, 2, item['rank'], xstyle)
sheet.write(i, 3, item['hot'], xstyle)
sheet.write(i, 4, item['play'], xstyle)
sheet.write(i, 5, item['comment'], xstyle)
i += 1
# 如果文件存在,则将其删除
if os.path.exists('D:/Test/b站热门视频信息.xls'):
os.remove('D:/Test/b站热门视频信息.xls')
workbook.save('D:/Test/b站热门视频信息.xls')
print('写入excel成功')
print("文件位置:D:/Test/b站热门视频信息.xls")
4.在入口main中调用上面两个函数
完整代码如下:
import requests
from lxml import html
import xlwt
import os
# 爬取b站热门视频信息
def spider(video_list):
url = 'https://www.bilibili.com/ranking?spm_id_from=333.851.b_7072696d61727950616765546162.3'
html_data = requests.get(url).text
selector = html.fromstring(html_data)
infolist = selector.xpath('//li[@class="rank-item"]')
for item in infolist:
rank = "".join(item.xpath('./div[@class="num"]/text()'))
videolink = "".join(item.xpath('./div[@class="content"]/div[@class="info"]/a/@href'))
title = "".join(item.xpath('./div[@class="content"]/div[@class="info"]/a/text()'))
playinfo = "".join(item.xpath('./div[@class="content"]/div[@class="info"]/div[@class="detail"]/span/text()')).split("万")
play = playinfo[0] + "万"
comment = playinfo[1]
if comment.isdigit() == False:
comment += "万"
upname = "".join(item.xpath('./div[@class="content"]/div[@class="info"]/div[@class="detail"]/a/span/text()'))
uplink = "http:" + "".join(item.xpath('./div[@class="content"]/div[@class="info"]/div[@class="detail"]/a/@href'))
hot = "".join(item.xpath('./div[@class="content"]/div[@class="info"]/div[@class="pts"]/div/text()'))
video_list.append({
'rank': rank,
'videolink': videolink,
'title': title,
'play': play,
'comment': comment,
'upname': upname,
'uplink': uplink,
'hot': hot
})
return video_list
# 将爬取到的数据写入Excel表格
def write_Excel(video_list):
print("将b站热门视频信息导入到Excel表格:")
workbook = xlwt.Workbook() # 定义workbook
sheet = workbook.add_sheet('b站热门视频') # 添加sheet
xstyle = xlwt.XFStyle() # 实例化表格样式对象
xstyle.alignment.horz = 0x02 # 字体居中
xstyle.alignment.vert = 0x01 # 字体居中
head = ['视频名', 'up主','排名', '热度','播放量','评论数'] # 表头
for h in range(len(head)):
sheet.write(0, h, head[h],xstyle) # 把表头写到Excel里面去
i = 1
for item in video_list:
# 向单元格(视频名)添加(该视频的)超链接
title_data = 'HYPERLINK("'+item["videolink"]+'";"'+item["title"]+'")' # 设置超链接
sheet.col(0).width = int(256 * len(title_data) * 3/5) # 设置列宽
sheet.write(i, 0, xlwt.Formula(title_data), xstyle)
name_data = 'HYPERLINK("' + item["uplink"] + '";"' + item["upname"] + '")' # 设置超链接
sheet.col(1).width = int(256 * len(title_data) * 3 / 10)
sheet.write(i, 1, xlwt.Formula(name_data), xstyle)
sheet.write(i, 2, item['rank'], xstyle)
sheet.write(i, 3, item['hot'], xstyle)
sheet.write(i, 4, item['play'], xstyle)
sheet.write(i, 5, item['comment'], xstyle)
i += 1
# 如果文件存在,则将其删除
if os.path.exists('D:/Test/b站热门视频信息.xls'):
os.remove('D:/Test/b站热门视频信息.xls')
workbook.save('D:/Test/b站热门视频信息.xls')
print('写入excel成功')
print("文件位置:D:/Test/b站热门视频信息.xls")
if __name__ == '__main__':
video_list = []
write_Excel(spider(video_list))