from urllib import request,parse
import ssl,json,pymysql
class GaoKaoPaiSpider(object):
def __init__(self):
数据库相关
self.client = pymysql.Connect(
host='127.0.0.1',user='root',
password='ljh1314',database='gaokaopai',
port=3306,charset='utf8'
)
# 创建游标
self.cursor = self.client.cursor()
# 添加排行榜排名默认字段
self.rank = 1
def send_request(self,form=None,headers=None):
"""发起请求,获取数据"""
form_data = parse.urlencode(form).encode('utf-8')
if not headers:
headers = {
'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36'
}
url = 'http://www.gaokaopai.com/rank-index.html'
req = request.Request(url=url,headers=headers,data=form_data)
ssl_context = ssl._create_unverified_context()
response = request.urlopen(req,context=ssl_context)
html = response.read().decode('utf-8')
# has_next:判断是否需要继续请求下一页
has_next = self.parse_response(html)
#获取下一页
if has_next:
form['start'] = form['start']+25
self.send_request(form=form)
else:
print('数据获取完毕')
def parse_response(self,html):
#!DOCTYPE HTML 如果!DOCTYPE HTML存在html中
# 说明返回的不是json字符串,而是html,在此案例中
# 表明数据已经获取完毕
if '!DOCTYPE HTML' in html:
return False
# json.loads:参数必须是一个json字符串
json_data = json.loads(html)
jobs = json_data['data']['ranks']
for job in jobs:
job_info = {}
job_info['top'] = self.rank
job_info['title'] =job['uni_name']
job_info['zongfen'] = self.get_default_num(data=job['xiao_total'],isFloat=True)
job_info['class'] = job['uni_type']
job_info['dizhi'] = job['city_code']
job_info['pici'] = '本科一批'
self.rank += 1
print(job_info)
self.save_data_to_mysql(job_info)
# 如果jobs列表数据大于0,说明,可能存在下一页,
# 继续发起请求,所以返回True
if len(jobs) > 0:
return True
elif len(jobs) == 0:
return False
def get_default_num(self,data=None,defalut=0,isFloat=False):
"""
:param data: 传递的参数
:param defalut: 设置默认值
:param isFloat: 是否是浮点型
:return:
"""
if data:
if isFloat:
return float(data)
else:
return int(data)
else:
if isFloat:
return float(defalut)
else:
return defalut
def save_data_to_mysql(self,job_info):
"""插入数据"""
insert_sql = """
INSERT INTO dsa(%s)
VALUES (%s)
"""%(
','.join(job_info.keys()),
','.join(['%s']*len(job_info)),
)
try:
self.cursor.execute(insert_sql,list(job_info.values()))
self.client.commit()
print('插入成功')
except Exception as err:
self.client.rollback()
print(err)
if name == 'main':
spider = GaoKaoPaiSpider()
"""
otype: 2
city: 0
cate: 0
batch_type: 0
start: 25
amount: 25
"""
form = {
'otype': 2,
'city': 0,
'cate': 0,
'batch_type':0,
'start':25,
'amount':25
}
spider.send_request(form=form)
转载于:https://www.jianshu.com/p/179815985d7f