#仅需修改这个地方https://jn.lianjia.com/ershoufang/pg{}rs/ 将jn换成你所在城市的拼写首字母小写 import requests from lxml import etree import time import random import csv class LianjiaSpider(object): def __init__(self): self.url = "https://jn.lianjia.com/ershoufang/pg{}rs/" self.headers = {"User-Agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/14.0.835.163 Safari/535.1"} def get_page(self,url): res = requests.get(url =url ,headers = self.headers) res.encoding ="utf-8" html = res.text #调用解析函数 self.parse_page(html) def parse_page(self,html): parse_html = etree.HTML(html) house_dict = {} house_list = parse_html.xpath("//div//ul[@class='sellListContent']//li") for house in house_list: house_dict["name"] = house.xpath(".//div[@class='title']/a//text()")[0] house_dict["totalprice"] = house.xpath(".//div[@class='totalPrice']/span//text()")[0] house_dict["uniteprice"] = house.xpath(".//div[@class='unitPrice']/span//text()")[0] house_dict["houseInfo"] = house.xpath(".//div[@class='houseInfo']/text()")[0] house_dict["positionInfoShequ"] = house.xpath(".//div[@class='positionInfo']/a[1]//text()")[0] house_dict["positionInfoJiedao"] = house.xpath(".//div[@class='positionInfo']/a[2]//text()")[0] house_dict["followInfo"] = house.xpath(".//div[@class='followInfo']/text()")[0] print(house_dict) with open('F:/top250/lianjia.csv', 'a', newline='', encoding='utf-8')as f: write = csv.writer(f) # write.writerow([house_dict["name"],house_dict["totalprice"],house_dict["uniteprice"],house_dict["info"]]) write.writerow([house_dict["name"], house_dict["totalprice"], house_dict["uniteprice"],house_dict["houseInfo"],house_dict["positionInfoShequ"],house_dict["positionInfoJiedao"],house_dict["followInfo"]]) f.close() def main(self): try: for i in range(1,2000): time.sleep(random.randint(3,5)) url = self.url.format(i) self.get_page(url) except: self.main() if __name__ == '__main__': start = time.time() spider = LianjiaSpider() spider.main() end = time.time() print("执行时间:%.2f" %(end-start))
17python链家济南市全网数据爬取(也可稍加改动用于全国各城市)
猜你喜欢
转载自blog.csdn.net/jidawanghao/article/details/108518064
今日推荐
周排行