python任务3

爬取小说网小说

coding=utf-8
import re
import requests
from bs4 import BeautifulSoup
import pymysql
db = pymysql.connect(“localhost”,“root”,“zgx675050748”,“RUNOOB”,charset=‘utf8’)
cursor = db.cursor()
url =‘https://www.qu.la/dushixiaoshuo/’#爬取都市小说区
response = requests.get(url)
response = response.content
response = str(response,‘utf-8’)
soup = BeautifulSoup(response,‘html.parser’)
book = soup.select(‘a[target="_blank"]’)
for a in range(0,len(book),2):
url1 = book[a][‘href’]#找到每一本小说，爬取当然小说的第一章网址
url2 = url+str(url1)
response = requests.get(url2)
response = response.content
response = str(response, ‘utf-8’)
soup1 = BeautifulSoup(response, ‘html.parser’)
book1 = soup1.find_all(href=re.compile(str(url1)+"(.*?)"))
name = soup1.select(‘h1’)
textname = name[0].text
sql = “”“CREATE TABLE “”” +str(textname)+ “”"(章节 CHAR(25),内容 text )"""
cursor.execute(sql)
url3 = url+book1[0][‘href’]
for b in book1:
response = requests.get(url3)#利用小说每章网页中的下一章按钮爬取接下来一章的地址，循环往复
response = response.content
response = str(response, ‘utf-8’)
soup1 = BeautifulSoup(response, ‘html.parser’)
content1 = soup1.select(’#pager_next’)
bookname = soup1.select(‘h1’)
content = soup1.select(’#content’)
print(bookname[0].text,content[0].text)
sql = “”“INSERT INTO “”” +textname+"""(章节,内容)VALUES ("%s","%s")""" %(bookname[0].text,content[0].text)
cursor.execute(sql)
db.commit()

    url3 = url2+content1[0]['href']

db.close()