书写代码部分
def crawl_pic_urls():
'''
爬取每个选手的百度百科图片,并保存
'''
with open('work/'+ today + '.json', 'r', encoding='UTF-8') as file:
json_array = json.loads(file.read())
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36'
}
import pdb
for star in json_array:
name = star['name']
link = star['link']
#!!!请在以下完成对每个选手图片的爬取,将所有图片url存储在一个列表pic_urls中!!!
# pdb.set_trace()
pic_urls=[]
print(name,link)
try:
response = requests.get(link,headers=headers)
print(response.status_code)
#将一段文档传入BeautifulSoup的构造方法,就能得到一个文档的对象, 可以传入一段字符串
soup = BeautifulSoup(response.text,'lxml')
#返回的是class为table-view log-set-param的<table>所有标签
tables = soup.find_all('div',{'class':'summary-pic'})
# crawl_table_title = "参赛学员"
for table in tables:
#对当前节点前面的标签和字符串进行查找
# table_hrefs = table.select("img")
table_hrefs = table.select("a")
for href in table_hrefs:
# pdb.set_trace()
# data=href.get('src')
# pic_urls.append(data)
data=href.get('href')
# data_list=data.split('/')
url_data='https://baike.baidu.com'+data
# pdb.set_trace()
try:
response = requests.get(url_data,headers=headers)
print(response.status_code)
soup = BeautifulSoup(response.text,'lxml')
pics=soup.select('.pic-list img ')
for pic in pics:
src=pic.get('src')
pic_urls.append(src)
# pics = soup.find_all('div',{'class':'pic-list'})
# for pic in pics:
# pic_urls = pic.select("a")
# for pu in pic_urls:
# imgs=pu.select('img')
# for img in imgs:
# src=img.get('src')
# if src:
# pic_urls.append(src)
except Exception as e:
print(e)
# pic_urls.append(url_data)
except Exception as e:
print(e)
#!!!根据图片链接列表pic_urls, 下载所有图片,保存在以name命名的文件夹中!!!
down_pic(name,pic_urls)