一.分析
首先简单介绍一下Scrapy的基本流程:
- 引擎从调度器中取出一个链接(URL)用于接下来的抓取
- 引擎把URL封装成一个请求(Request)传给下载器
- 下载器把资源下载下来,并封装成应答包(Response)
- 爬虫解析Response
- 解析出实体(Item),则交给实体管道进行进一步的处理
- 解析出的是链接(URL),则把URL交给调度器等待抓取
在网上找到了接口:http://m.maoyan.com/mmdb/comments/movie/248172.json?_v_=yes&offset=0&startTime=2019-02-05%2020:28:22,可以把offset的值设定为0,通过改变startTime的值来获取更
多的评论信息(把每页评论数据中最后一次评论时间作为新的startTime并构造url重新请求)(startTime=2019-02-05%2020:28:22这里的%20表示空格)
二.主要代码
items.py
import scrapy class MaoyanItem(scrapy.Item): # define the fields for your item here like: # name = scrapy.Field() city = scrapy.Field() # 城市 content = scrapy.Field() # 评论 user_id = scrapy.Field() # 用户id nick_name = scrapy.Field() # 昵称 score = scrapy.Field() # 评分 time = scrapy.Field() # 评论时间 user_level = scrapy.Field() # 用户等级
comment.py
import scrapy import random from scrapy.http import Request import datetime import json from maoyan.items import MaoyanItem class CommentSpider(scrapy.Spider): name = 'comment' allowed_domains = ['maoyan.com'] uapools = [ 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/14.0.835.163 Safari/535.1', 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:6.0) Gecko/20100101 Firefox/6.0', 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50', 'Opera/9.80 (Windows NT 6.1; U; zh-cn) Presto/2.9.168 Version/11.50', 'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; InfoPath.3)', 'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; GTB7.0)', 'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)', 'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1)', 'Mozilla/5.0 (Windows; U; Windows NT 6.1; ) AppleWebKit/534.12 (KHTML, like Gecko) Maxthon/3.0 Safari/534.12', 'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; InfoPath.3; .NET4.0C; .NET4.0E)', 'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; InfoPath.3; .NET4.0C; .NET4.0E; SE 2.X MetaSr 1.0)', 'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/534.3 (KHTML, like Gecko) Chrome/6.0.472.33 Safari/534.3 SE 2.X MetaSr 1.0', 'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; InfoPath.3; .NET4.0C; .NET4.0E)', 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/13.0.782.41 Safari/535.1 QQBrowser/6.9.11079.201', 'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; InfoPath.3; .NET4.0C; .NET4.0E) QQBrowser/6.9.11079.201', 'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0)', 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.80 Safari/537.36', 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:34.0) Gecko/20100101 Firefox/34.0' ] thisua = random.choice(uapools) header = {'User-Agent': thisua} current_time = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') current_time = '2019-04-24 18:50:22' end_time = '2019-04-24 00:05:00' # 电影上映时间 url = 'http://m.maoyan.com/mmdb/comments/movie/248172.json?_v_=yes&offset=0&startTime=' +current_time.replace(' ','%20') def start_requests(self): current_t = str(self.current_time) if current_t > self.end_time: try: yield Request(self.url, headers=self.header, callback=self.parse) except Exception as error: print('请求1出错-----' + str(error)) else: print('全部有关信息已经搜索完毕') def parse(self, response): item = MaoyanItem() data = response.body.decode('utf-8', 'ignore') json_data = json.loads(data)['cmts'] count = 0 for item1 in json_data: if 'cityName' in item1 and 'nickName' in item1 and 'userId' in item1 and 'content' in item1 and 'score' in item1 and 'startTime' in item1 and 'userLevel' in item1: try: city = item1['cityName'] comment = item1['content'] user_id = item1['userId'] nick_name = item1['nickName'] score = item1['score'] time = item1['startTime'] user_level = item1['userLevel'] item['city'] = city item['content'] = comment item['user_id'] = user_id item['nick_name'] = nick_name item['score'] = score item['time'] = time item['user_level'] = user_level yield item count += 1 if count >= 15: temp_time = item['time'] current_t = datetime.datetime.strptime(temp_time, '%Y-%m-%d %H:%M:%S') + datetime.timedelta( seconds=-1) current_t = str(current_t) if current_t > self.end_time: url1 = 'http://m.maoyan.com/mmdb/comments/movie/248172.json?_v_=yes&offset=0&startTime=' + current_t.replace( ' ', '%20') yield Request(url1, headers=self.header, callback=self.parse) else: print('全部有关信息已经搜索完毕') except Exception as error: print('提取信息出错1-----' + str(error)) else: print('信息不全,已滤除')
pipelines文件
import pandas as pd class MaoyanPipeline(object): def process_item(self, item, spider): dict_info = {'city': item['city'], 'content': item['content'], 'user_id': item['user_id'], 'nick_name': item['nick_name'], 'score': item['score'], 'time': item['time'], 'user_level': item['user_level']} try: data = pd.DataFrame(dict_info, index=[0]) # 为data创建一个表格形式 ,注意加index = [0] data.to_csv('G:\info.csv', header=False, index=True, mode='a', encoding='utf_8_sig') # 模式:追加,encoding = 'utf-8-sig' except Exception as error: print('写入文件出错-------->>>' + str(error)) else: print(dict_info['content'] + '---------->>>已经写入文件')
最后爬完的数据12M左右,65000条数据左右
三.数据可视化
1.主要代码
用到的模块:pandas数据处理,matplotlib绘图,jieba分词,wordcloud词云,地图相关模块(echarts-countries-pypkg,echarts-china-provinces-pypkg, echarts-china-cities-pypkg)
#!/usr/bin/env python # -*- coding:utf-8 -*- import pandas as pd from collections import Counter from pyecharts import Geo, Bar, Scatter import jieba import matplotlib.pyplot as plt from wordcloud import WordCloud, STOPWORDS import time #观众地域图中部分注释 #attr:标签名称(地点) #value:数值 #visual_range:可视化范围 #symbol_size:散点的大小 #visual_text_color:标签颜色 #is_visualmap:是否映射(数量与颜色深浅是否挂钩) #maptype:地图类型 #读取csv文件(除了词云,其它图表用的源数据) def read_csv(filename, titles): comments = pd.read_csv(filename, names = titles, low_memory = False) return comments #词云用的源数据(比较小) def read_csv1(filename1, titles): comments = pd.read_csv(filename1, names = titles, low_memory = False) return comments #全国观众地域分布 def draw_map(comments): attr = comments['city_name'].fillna('zero_token') #以'zero_token'代替缺失数据 data = Counter(attr).most_common(300) #Counter统计各个城市出现的次数,返回前300个出现频率较高的城市 # print(data) data.remove(data[data.index([(i,x) for i,x in data if i == 'zero_token'][0])]) #检索城市'zero_token'并移除('zero_token', 578) geo =Geo('《复联4》全国观众地域分布', '数据来源:Mr.W', title_color = '#fff', title_pos = 'center', width = 1000, height = 600, background_color = '#404a59') attr, value = geo.cast(data) #data形式[('合肥',229),('大连',112)] geo.add('', attr, value, visual_range = [0, 4500], maptype = 'china', visual_text_color = '#fff', symbol_size = 10, is_visualmap = True) geo.render('G:\\影评\\观众地域分布-地理坐标图.html') print('全国观众地域分布已完成') #观众地域排行榜单 def draw_bar(comments): data_top20 = Counter(comments['city_name']).most_common(20) #前二十名城市 bar = Bar('《复联4》观众地域排行榜单', '数据来源:Mr.W', title_pos = 'center', width = 1200, height = 600) attr, value = bar.cast(data_top20) bar.add('', attr, value, is_visualmap = True, visual_range = [0, 4500], visual_text_color = '#fff', is_more_utils = True, is_label_show = True) bar.render('G:\\影评\\观众地域排行榜单-柱状图.html') print('观众地域排行榜单已完成') #观众评论数量与日期的关系 #必须统一时间格式,不然时间排序还是乱的 def draw_data_bar(comments): time1 = comments['time'] time_data = [] for t in time1: if pd.isnull(t) == False and 'time' not in t: #如果元素不为空 date1 = t.replace('/', '-') date2 = date1.split(' ')[0] current_time_tuple = time.strptime(date2, '%Y-%m-%d') #把时间字符串转化为时间类型 date = time.strftime('%Y-%m-%d', current_time_tuple) #把时间类型数据转化为字符串类型 time_data.append(date) data = Counter(time_data).most_common() #data形式[('2019/2/10', 44094), ('2019/2/9', 43680)] data = sorted(data, key = lambda data : data[0]) #data1变量相当于('2019/2/10', 44094)各个元组 itemgetter(0) bar =Bar('《复联4》观众评论数量与日期的关系', '数据来源:Mr.W', title_pos = 'center', width = 1200, height = 600) attr, value = bar.cast(data) #['2019/2/10', '2019/2/11', '2019/2/12'][44094, 38238, 32805] bar.add('', attr, value, is_visualmap = True, visual_range = [0, 3500], visual_text_color = '#fff', is_more_utils = True, is_label_show = True) bar.render('G:\\影评\\观众评论日期-柱状图.html') print('观众评论数量与日期的关系已完成') #观众评论数量与时间的关系 #这里data中每个元组的第一个元素要转化为整数型,不然排序还是乱的 def draw_time_bar(comments): time = comments['time'] time_data = [] real_data = [] for t in time: if pd.isnull(t) == False and ':' in t: time = t.split(' ')[1] hour = time.split(':')[0] time_data.append(hour) data = Counter(time_data).most_common() for item in data: temp1 = list(item) temp2 = int(temp1[0]) temp3 = (temp2,temp1[1]) real_data.append(temp3) data = sorted(real_data, key = lambda x : x[0]) bar = Bar('《复联4》观众评论数量与时间的关系', '数据来源:Mr.W', title_pos = 'center', width = 1200, height = 600) attr, value = bar.cast(data) bar.add('', attr, value, is_visualmap = True, visual_range = [0, 3500], visual_text_color = '#fff', is_more_utils = True, is_label_show = True) bar.render('G:\\影评\\观众评论时间-柱状图.html') print('观众评论数量与时间的关系已完成') #词云,用一部分数据生成,不然数据量有些大,会报错MemoryError(64bit的python版本不会) def draw_word_cloud(comments): data = comments['comment'] comment_data = [] print('由于数据量比较大,分词这里有些慢,请耐心等待') for item in data: if pd.isnull(item) == False: comment_data.append(item) comment_after_split = jieba.cut(str(comment_data), cut_all = False) words = ' '.join(comment_after_split) stopwords = STOPWORDS.copy() stopwords.update({'电影', '非常', '这个', '那个', '因为', '没有', '所以', '如果', '演员', '这么', '那么', '最后', '就是', '不过', '这个', '一个', '感觉', '这部', '虽然', '不是', '真的', '觉得', '还是', '但是'}) wc = WordCloud(width = 800, height = 600, background_color = '#000000', font_path = 'simfang', scale = 5, stopwords = stopwords, max_font_size = 200) wc.generate_from_text(words) plt.imshow(wc) plt.axis('off') plt.savefig('G:\\影评\\WordCloud.png') plt.show() #观众评分排行榜单 def draw_score_bar(comments): score_list = [] data_score = Counter(comments['score']).most_common() for item in data_score: if item[0] != 'score': score_list.append(item) data = sorted(score_list, key = lambda x : x[0]) bar = Bar('《复联4》观众评分排行榜单', '数据来源:Mr.W', title_pos = 'center', width = 1200, height = 600) attr, value = bar.cast(data) bar.add('', attr, value, is_visualmap = True, visual_range = [0, 4500], visual_text_color = '#fff', is_more_utils = True, is_label_show = True) bar.render('G:\\影评\\观众评分排行榜单-柱状图.html') print('观众评分排行榜单已完成') #观众用户等级排行榜单 def draw_user_level_bar(comments): level_list = [] data_level = Counter(comments['user_level']).most_common() for item in data_level: if item[0] != 'user_level': level_list.append(item) data = sorted(level_list, key = lambda x : x[0]) bar = Bar('《复联4》观众用户等级排行榜单', '数据来源:Mr.W', title_pos = 'center', width = 1200, height = 600) attr, value = bar.cast(data) # is_more_utils = True 提供更多的实用工具按钮 bar.add('', attr, value, is_visualmap = True, visual_range = [0, 4500], visual_text_color = '#fff', is_more_utils = True, is_label_show = True) bar.render('G:\\影评\\观众用户等级排行榜单-柱状图.html') print('观众用户等级排行榜单已完成') if __name__ == '__main__': filename = 'G:\\info.csv' filename2 = 'G:\\info.csv' titles = ['city_name','comment','user_id','nick_name','score','time','user_level'] comments = read_csv(filename, titles) comments2 = read_csv1(filename2, titles) draw_map(comments) draw_bar(comments) draw_data_bar(comments) draw_time_bar(comments) draw_word_cloud(comments2) draw_score_bar(comments) draw_user_level_bar(comments)
2.效果与分析
01.观众地域分布-地理坐标图
由全国地域热力图可见,观众主要分布在中部,南部,东部以及东北部,各省会城市的观众尤其多(红色代表观众最多),这与实际的经济、文化、消费水平基本相符.(ps:复联4的票价有点贵)
02.《复联4》观众地域排行榜单
北上广深等一线城市,观众粉丝多,消费水平可以。观影数量非常多。
03.《复联4》观众评分排行榜单
可以看到评分满分的用户几乎超过总人数的70%,可见观众看完电影之后很满足,也说明了电影的可看性很高
04.《复联4》观众评论数量与日期的关系
24号上映到现在已经三天,其中观影人数最多的是25号,可能大家觉得首映有点小贵吧,哈哈。
05.《复联4》观众评论数量与时间的关系
从图中可以看出,评论的数量主要集中在16-23点,因为这部电影时长为2小时,所以把评论时间往前移动2小时基本就是看电影时间。可以看出大家都是中午吃完饭(13点左右)和晚上吃完饭(19点左右)后再去看电影的,而且晚上看电影的人更多
06.《复联4》观众用户等级排行榜单
可见用户等级为0,5,6的用户基本没有,而且随着等级的提升,人数急剧变少。新用户可能是以年轻人为主,对科幻电影感兴趣,因而评论数量较多,而老用户主要偏向于现实剧情类的电影,评论数量较少
07.《复联4》词云图
在词云图中可以看到,“好看,可以,完美,精彩,情怀”等字眼,看来影片还是挺好看的。接着就是“钢铁侠,美队,灭霸”看来这几个人在影评中有重要的故事线。