1.准备工作
工具:pyCharm
安装: Python环境变量(网上查找)
可以在git中运行,也可以再cmd中运行,或者直接在工具pycharm中运行
2. 引入需要的模块和定义临时存储变量
# 引入需要的包和定义变量
from urllib import request
import re
from openpyxl import Workbook
wenjian=[] // 临时变量
#爬取功能
def getCont(url):
response = request.urlopen(url)
html = response.read().decode('utf-8')
re_zz = re.compile(r'<h4 class="panel-title">(.*?)</h4>', re.S)
list = re_zz.findall(html)
re_url = re.compile(r'href="(.*?)"')
re_title = re.compile(r'<a .*?>(.*?)</a>')
re_html = re.compile(r'>(.*?)<')
# print(list)
for h in list:
a_url = re_url.findall(h)[0]
a_cont = re_title.findall(h)[0]
wenjian.append([a_url, re_html.findall(a_cont)[1]])
# print(wenjian)
# 存储功能
def cunchu():
wb = Workbook()
sheet = wb.active
sheet.title = '菜鸟教程'
for i in range(1, len(wenjian)):
sheet.cell(1, 1).value = 'URL'
sheet.cell(1, 2).value = 'Content'
for j in range(0, 2):
sheet.cell(row=i + 1, column=j + 1).value = wenjian[i-1][j]
wb.save(r"F:\sample.xlsx")
# 入口函数
if __name__ == '__main__':
for i in range(1, 3):
print('正在爬取第%s页......' % i)
url = 'https://c.runoob.com/examples/page/%s' % i
getCont(url)
print('第%s页爬取完毕!' % i)
print('开始存储')
cunchu()
print('写入完毕')