####config 文件的参数
#mongodb的参数
MONGO_URL = 'localhost'
MONGO_DB = 'weixin'
MONGO_TABLE = 'article_data'
POOL_PROXY_URL = 'http://127.0.0.1:5000/get' # 初始化代理
KEYWORDS= '风景' # 搜索的内容
base_url = 'https://weixin.sogou.com/weixin?'
MAX_COUNT = 5
from pyquery import PyQuery as pq
import requests
from urllib.parse import urlencode
import pymongo
from config import *
# 设置cookies 保证自己处于登陆状态
headers = {
'Cookie': 'SMYUV=1543916558046770; IPLOC=CN3210; SUID=824A933D2E18960A000000005C064C19; usid=xOxPgW3N3IoHtGbK; SUV=00EC55C13D934A825C064C1975830150; ld=ZZllllllll2tx6ZPlllllVZpFj9lllll5B23kkllll9lllllxllll5@@@@@@@@@@; LSTMV=247%2C70; LCLKINT=3969; ABTEST=0|1543916582|v1; weixinIndexVisited=1; SNUID=C008D27C41443EE5E68EBAC74227C258; sct=3; JSESSIONID=aaapI9Xh-AseD_U-Nw_Cw; ppinf=5|1544421555|1545631155|dHJ1c3Q6MToxfGNsaWVudGlkOjQ6MjAxN3x1bmlxbmFtZTo0OlRpbmF8Y3J0OjEwOjE1NDQ0MjE1NTV8cmVmbmljazo0OlRpbmF8dXNlcmlkOjQ0Om85dDJsdUJYcHpaSHVCXzRoMG5rQ2tQdS00bGdAd2VpeGluLnNvaHUuY29tfA; pprdig=NsDEAzGvJ1re-8nnesvi4j3YCyLTG05BToQzQjD1RX5uGMNh3s53s1UDapzy3GV_GmE6xp51ZvYjLa1QquTvsy8WOu3WeqMxzq6cHiQy3YsFnn31696RjyfBmDK_uyA1tbKZu61HU3HCR19g1Og3YUXuisHQqCXg0m2HXy6Iv_4; sgid=11-38266993-AVwOALPb4EmMTeL4sY1JzFg; ppmdig=1544421555000000a5d278737afc6497b87750cd0af731fa',
'Host': 'weixin.sogou.com',
'Upgrade-Insecure-Requests': '1',
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36'
}
client = pymongo.MongoClient(MONGO_URL)
db = client[MONGO_DB]
proxy = None
def get_url(KEYWORDS, page): # 1、构造url,进行微信关键词搜索
base_url = 'https://weixin.sogou.com/weixin?'
data = {
'query': KEYWORDS,
'type': 2,
'page': page
}
url = base_url + urlencode(data)
return url
def get_index_html(url, count=1): # 2、请求url,得到索引页html
print('正在爬取', url)
print('Trying Count', count)
global proxy # 使用global申明全局变量,声明后可在函数内改变proxy的值
if count >= MAX_COUNT:
print('Tried many times')
return None
try:
if proxy:
print('正在使用代理',proxy)
proxies = {
'http':'http://' + proxy
}
# allow_redirects=False 关闭重定向, 默认为True
response = requests.get(url, headers=headers, allow_redirects=False, proxies=proxies)
else:
response = requests.get(url, headers=headers, allow_redirects=False)
if response.status_code == 200:
return response.text
if response.status_code == 302:
# Need proxy
print('302') # 302 状态表示请求网页临时移动到新的位置
proxy = get_proxy() # 添加代理
if proxy:
return get_index_html(url)
else:
print('请求代理失败')
return None
except ConnectionError as e:
print('Error Occurred', e.args)
proxy = get_proxy()
count += 1
return get_index_html(url)
def get_proxy(): # 3、请求url过程中,可能会遇到反爬虫措施,这时候需要开启代理
print('正在请求代理')
try:
response = requests.get(POOL_PROXY_URL)
if response.status_code == 200:
return response.text
else:
print('请求代理失败')
return None
except ConnectionError:
return None
# def delete_proxy(proxy): # 删除频繁出错的代理
# requests.get('http://127.0.0.1:5000/delete/?proxy={}'.format(proxy))
def get_article_url(html): # 4、分析索引页html代码,返回微信详情页url
doc = pq(html)
url_lists = doc('.news-box .news-list li').items()
for item in url_lists:
# print(item.find('h3 a').attr('href'))
yield item.find('h3 a').attr('href')
def get_article_html(article_url): # 5、请求微信详情页url,得到详情页html
try:
response = requests.get(article_url)
if response.status_code == 200:
return response.text
return None
except ConnectionError:
return None
def parse_article_html(article_html): # 6、分析详情页html代码,得到微信标题、公众号、发布日期,文章内容等信息
doc = pq(article_html)
title = doc('#img-content .rich_media_title').text()
content = doc('.rich_media_area_primary_inner').text()
date = doc('.rich_media_meta.rich_media_meta_text').text()
nikname = doc('.rich_media_meta_list .rich_media_meta_nickname').text()
artile_data = {
'title': title,
'content': content,
'date': date,
'nikname': nikname
}
return artile_data
def save_to_mongo(article_data): # 保存到数据库MongoDB
# MongoDB中update()使用'$set'指定一个键的值,如果不存在该值就创建(去重)
# multi:默认是false,只更新找到的第一条记录。如果为true,把按条件查询出来的记录全部更新。
if db[MONGO_TABLE].update({'title':article_data['title']},{'$set': article_data}, True):
print('Saved to MongoDB', article_data['title'])
else:
print('Saved to Mongo Failed', article_data['title'])
# try:
# if db[MONGO_TABLE].insert(article_data):
# print('保存到MONGODB成功',article_data)
# except Exception:
# print('保存到MONGODB失败!',article_data)
def main(): # 调试模块
for page_num in range(1, 4): # 由于我维护的代理池质量不好,所以这里只取前3页的文章内容
index_url = get_url(KEYWORDS, page_num) # 获取微信搜索页面的url
# print(index_url)
index_html = get_index_html(index_url) # 获取搜索页面url的text
if index_html:
article_urls = get_article_url(index_html) # 获取上述html中包含的文章的URL,返回的是一个迭代器
for article_url in article_urls: # 遍历上述的url
# print(article_url)
article_html = get_article_html(article_url) # 得到url的text
if article_html:
article_data = parse_article_html(article_html) # 解析url的text并获取我们想要的内容(content, title, nikname, date...)
print(article_data) # 把上述的data打印出来
save_to_mongo(article_data) # 把上述的data存储到mongodb 数据库中
if __name__ == '__main__':
main()