一、问题说明
1、selenium库是爬虫过程中比较讨巧的一个第三方库,它能够跳过js、ajax等交互,上手比较容易。
2、基础代码是根据其他博主参考而来,但携程网站不断变化,除ID等不变的信息外,其余都已发生变化,因此,仔细对比之后,改进并编写了以下代码,发布于2021年10月19日。
3、如果出现报错,请尝试修改下面代码中 time.sleep()
函数参数。
4、要爬取自己想要的数据,只需修改出发地和到达地以及出发时间,另外注意修改浏览器驱动,本人用的是MicroSoft Edge,驱动到对应的网站下载,下载后要更名驱动并修改 driver_path
参数。
4、目前只是一个基础版本,后续可能会发布更新版,如加入直飞、中转、经停等个性化数据爬取。
5、代码只供学习参考,请勿商用!
二、代码
import time
from selenium import webdriver
from bs4 import BeautifulSoup
def page_select_function(driver_path):
driver = webdriver.Edge(executable_path=driver_path)
driver.get('https://www.ctrip.com/')
time.sleep(1)
driver.maximize_window()
input_tag_slect = driver.find_element_by_class_name('s_tab_nocurrent')
input_tag_slect.click()
time.sleep(1)
input_tag_time = driver.find_element_by_id('FD_StartDate')
input_tag_time.send_keys('2021-10-20')
time.sleep(1)
input_tag_depart_city = driver.find_element_by_id('FD_StartCity')
input_tag_depart_city.send_keys('厦门')
time.sleep(1)
input_tag_arrive_city = driver.find_element_by_id('FD_DestCity')
input_tag_arrive_city.send_keys('兰州')
time.sleep(1)
input_tag_search = driver.find_element_by_id('FD_StartSearch')
driver.execute_script("arguments[0].click();", input_tag_search)
time.sleep(3)
tag_close = driver.find_element_by_class_name('close-icon')
tag_close.click()
time.sleep(1)
tag_select = driver.find_element_by_class_name('auto_cursor')
tag_select.click()
time.sleep(2)
driver.find_element_by_xpath('//*[@id="domestic_filter_group_trans_and_train__trans_count"]/li[2]/span').click()
driver.find_element_by_class_name('flight-part').click()
time.sleep(1)
'''
# 只选择直飞
direct_flight_tag_click = driver.find_element_by_class_name('form-label')
direct_flight_tag_click.click()
time.sleep(2)
'''
for i in range(1):
driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
time.sleep(1)
return driver
def data_acquistion(driver):
source = driver.page_source
bs = BeautifulSoup(source, 'html.parser')
divs = bs.find_all('div', class_='flight-item domestic')
return divs
def data_treating(divs):
for div in divs:
try:
airlineName = div.find('div', class_='airline-name').get_text()
flightNumber = div.find_all('span', class_='plane-No')[0].get_text()
departureTime = div.find('div', class_='depart-box').find('div', class_='time').string
arrivalTime = div.find('div', class_='arrive-box').find('div', class_='time').get_text()
lowestPrice = div.find('span', class_='price').get_text()
print(airlineName, '\t', flightNumber, '\t', departureTime, '\t', arrivalTime, '\t', lowestPrice)
time.sleep(1)
except:
print(无该信息)
def main():
driver_path = r'C:\MicrosoftWebDriver.exe'
driver = page_select_function(driver_path)
divs = data_acquistion(driver)
data_treating(divs)
driver.close()
if __name__ == '__main__':
main()