写在前面
原创内容,禁止转载,禁止用于商业途径,仅限技术探讨。
最近一位网友写论文需要数据,找我来给他爬一下人人贷的散标用户信息。于是花了5-6个小时,折腾了一番搞定了。
整体思路
先说一下,散标用户信息我是怎么找到的吧。
这是打开的散标用户信息页面,里面关键信息都被隐藏了,提示要登陆。
我先尝试通过request请求登陆接口,但是参数是加密以后才提交的,且实在是找不到破解规律。
然后就打算用selenium模拟浏览器直接抓取。但是也是,死活登陆不上。
本来都有放弃的心了,但不死心,想着数据会不会被藏在其他接口里面。然后我在一堆接口数据里面使劲找,这么一找,果真被我发现了,而且我发现,不用登陆就能看见数据。
这下乐坏了,本以为有多麻烦呢,原来是再简单不过了。哈哈哈
爬取结果
代码参考
# coding=utf-8
from requests.exceptions import RequestException
import requests
import json
import csv
import re
import os
class Spider(object):
def __init__(self):
self.headers = {
'Accept': 'application / json, text / javascript, * / *; q = 0.01',
'Accept - Encoding': 'gzip, deflate, br',
'Accept - Language': 'zh - CN, zh; q = 0.9',
'Connection': 'keep - alive',
'Host': 'www.renrendai.com',
'Referer': 'https: // www.renrendai.com / loan.html',
'User - Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36',
'X - Requested - With': 'XMLHttpRequest'
}
self.count = 0 # 记录成功爬取的条数
# 获取散标信息
def get_sanbiao(self):
# 一共1000条,爬10次,每次100条
for page in range(10):
url = 'https://www.renrendai.com/loan/list/loanList?startNum={}&limit=100'.format(page)
try:
response = requests.get(url, headers=self.headers)
if response.status_code == 200:
self.parse_sanbian(response.text)
except RequestException as e:
print(e)
# 解析散标信息
def parse_sanbian(self, data):
data = json.loads(data)
for item in data['data']['list']:
url = 'https://www.renrendai.com/loan-{}.html'.format(item['loanId'])
self.get_detailinfo(url)
# 获取详细信息
def get_detailinfo(self, url):
try:
response = requests.get(url, headers=self.headers)
if response.status_code == 200:
self.count += 1
print('成功爬取第 {} 条'.format(self.count))
self.parse_detailinfo(response.text)
else:
print('failure: {}'.format(url))
except RequestException as e:
print(e)
# 解析详细信息
def parse_detailinfo(self, data):
data = data.replace(u'\xa9', u'').replace('\\u0022', '"').replace('\\u005C', '\\') # gbk无法对u'\xa9'代表的字符进行编码,在Unicode中u'\xa9'代表的是©。因此直接忽略掉。
data = re.compile("var info = '({.*?})'", re.S).findall(data)
data = json.loads(data[0])
result = {}
# 顶部信息
result['loanId'] = data['loan']['loanId'] # Number
result['borrowType'] = data['loan']['borrowType'] # 贷款类型
result['amount'] = data['loan']['amount'] #标的总额
result['interest'] = data['loan']['interest'] # 年利率
result['months'] = data['loan']['months'] # 还款期限
result['creditLevel'] = data['borrower']['creditLevel'] # 风险等级
result['repayType'] = '按季还款' if int(data['loan']['repayType']) else '按月还款' # 还款方式
result['loanType'] = '等额本息' if data['loan']['loanType'] == 'DEBX' else '付息还本' #借贷方式
result['repaySource'] = data['repaySource'] # 还款来源
# 借贷人信息
result['realName'] = data['borrower']['realName'] # 姓名
result['gender'] = data['borrower']['gender'] # 性别
result['age'] = 2019-int(data['borrower']['birthDay'][:4]) # 年龄
result['marriage'] = '已婚' if data['borrower']['marriage'] else '未婚' # 婚姻
result['graduation'] = data['borrower']['graduation'] # 学历
result['salary'] = data['borrower']['salary'] # 收入
result['houseLoan'] = '有' if data['borrower']['houseLoan'] else '无' # 房贷
result['carLoan'] = '有' if data['borrower']['carLoan'] else '无' # 车贷
result['officeDomain'] = data['borrower']['officeDomain'] # 公司行业
result['hasOthDebt'] =data['hasOthDebt'] # 其他负债
# 信用信息
result['totalCount'] = data['userLoanRecord']['totalCount'] # 申请借款
result['successCount'] = data['userLoanRecord']['totalCount'] # 成功借款
result['alreadyPayCount'] = data['userLoanRecord']['alreadyPayCount'] # 还清笔数
result['availableCredits'] = data['borrower']['availableCredits'] #信用额度
result['borrowAmount'] = data['userLoanRecord']['borrowAmount'] # 借款总额
result['notPayTotalAmount'] = data['userLoanRecord']['notPayTotalAmount'] # 待还本息
result['overdueTotalAmount'] = data['userLoanRecord']['overdueTotalAmount'] # 逾期金额
result['overdueCount'] = data['userLoanRecord']['overdueCount'] # 逾期次数
result['failedCount'] = data['userLoanRecord']['failedCount'] # 严重逾期
self.save_excel(list(result.values()))
# 存到excel
def save_excel(self, data):
out = open('人人贷.csv', 'a', newline='')
write = csv.writer(out, dialect='excel')
write.writerow(data)
def run(self):
if os.path.exists('./人人贷.csv'):
os.remove('./人人贷.csv')
self.save_excel('序号 贷款类型 标的总额 年利率 还款期限 风险等级 还款方式 借贷方式 还款来源'
' 姓名 性别 年龄 婚姻 学历 收入 房贷 车贷 公司行业 其他负债'
' 申请借款 成功借款 还清笔数 信用额度 借款总额 待还本息 逾期金额 逾期次数 严重逾期'.split(' '))
self.get_sanbiao()
if __name__ == '__main__':
spider = Spider()
spider.run()