首先导入响应的模块:
import requests
from lxml import etree
import json
然后新建一个class类,并创建需要的实例:
class KuwoSpider:
def __init__(self):
self.start_url = "http://www.kuwo.cn/bang/index" # 1.start_url
self.headers = {"User-Agent": "Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A372 Safari/604.1"}
然后写一个run函数来实现主要逻辑:
def run(self): # 实现主要逻辑
# 1.start_url
# 2.发送请求,获取响应
html_str = self.parse_url()
# 3.提取信息
content_list = self.get_content_list(html_str)
# 4.保存
self.save_content_list(content_list)
接下来就可以根据根据主要逻辑来实现相应的步骤:
def parse_url(self): # 2.发送请求,获取响应
response = requests.get(self.start_url, headers=self.headers)
return response.content.decode()
def get_content_list(self, html_str): # 3.提取信息
html = etree.HTML(html_str)
div_list = html.xpath("//ul[@class='listMusic']/li")
print(div_list)
content_list = []
for div in div_list:
item = {}
item["order"] = div.xpath(".//p[@class='num']/text()")[0]
item["song_name"] = div.xpath("./div[@class='name']/a/text()")[0]
item["song_url"] = div.xpath("./div[@class='name']/a/@href")[0]
item["songer"] = div.xpath("./div[@class='artist']/a/text()")[0]
item["heatValue"] = div.xpath(".//div[@class='heatValue']/@style")
item["heatValue"] = item["heatValue"][0].replace("width:", "")
content_list.append(item)
return content_list
def save_content_list(self, content_list): # 4.保存
with open("kuwo.txt", "w", encoding="utf-8") as f:
for content in content_list:
f.write(json.dumps(content, ensure_ascii=False, indent=2))
f.write("\n")
print("保存成功")
最后实例化类并调用类中的run函数:
if __name__ == '__main__':
kuwo = KuwoSpider()
kuwo.run()
这样,运行这个程序便可以将榜单中的你想要的信息爬取出来:
{
"order": "1",
"song_name": "盗将行",
"song_url": "http://www.kuwo.cn/yinyue/53959930?catalog=yueku2016",
"songer": "花僮",
"heatValue": "92%"
}
{
"order": "2",
"song_name": "走马",
"song_url": "http://www.kuwo.cn/yinyue/54002473?catalog=yueku2016",
"songer": "摩登兄弟",
"heatValue": "91%"
}
{
"order": "3",
"song_name": "作曲家",
"song_url": "http://www.kuwo.cn/yinyue/53996611?catalog=yueku2016",
"songer": "刘郡格",
"heatValue": "98%"
}
...
...
上面的数据就是我们所想要爬取的信息,在这里我只截取前三个数据。