最近帮老师做个项目，需要用到文章摘要和关键词数据集。网上找了一下，没有很适合我的，所以自己写了一个小程序爬取了一些。现在把程序记录起来，供以后学习和需要的朋友使用。

           # coding:utf-8


import urllib2
import re
import time

 #获取大的标签url
def page_link_1(url_1):
    url_list = []
    for i in range(1,500):
        url_i = url_1 + str(i)
        url_list.append(url_i)
    return url_list
    # request = urllib2.Request(url_1)
    # response = urllib2.urlopen(request).read()
    # print response

#获取大的标签下的页面中文章摘要和关键词页面的url
def page_link_2(url_2_list, user_agent):
    paper_title_dict = {}
    headers = {'User_Agent': user_agent}
    i = 1
    for url_i in url_2_list:
        print 'the i_th page:%d' %i
        i += 1
        request = urllib2.Request(url_i, headers=headers)
        html = urllib2.urlopen(request).read()
        title = re.findall(r'<div class="sortby_div">(.*?)<p class="pager_space">',html, re.I|re.S)
        if len(title) != 0:
            title_list = re.findall(r'<li class="title_li">(.*?)</li>', title[0], re.I|re.S)
            for each_title in title_list:
                each_title_1 = re.findall(r'class="abs_img"></a>(.*?)</a>', each_title, re.I|re.S)[0]
                each_title_2 = re.findall(r'''<a href='(.*?)' target="_blank">(.*)''', each_title_1,re.I|re.S|re.L)[0]
                paper_title_dict[each_title_2[1]] = each_title_2[0]
        else:
            continue
        time.sleep(2)
    return paper_title_dict

#找到文章的摘要和关键词
def page_link_3(url_3_dict):
    paper_list = []
    for each_title in url_3_dict:
        abstract_and_keywords = []
        request = urllib2.Request(url_3_dict[each_title])
        response = urllib2.urlopen(request).read()
        abstract_1 = re.findall('<div id="completeAbstract" style="display: none">(.*?)<p id="collapse">', response,
                              re.I | re.S)
        if len(abstract_1) == 0:
            continue
        abstract = abstract_1[0].strip()
        abstract_and_keywords.append(abstract)


        keywords = re.findall('<t>关键词</t>(.*?)</tr>', response, re.I | re.S)[0]
        keywords_list = re.findall('<a href=(.*?)>(.*?)</a>', keywords, re.I | re.S | re.M)
        for each in keywords_list:
            string = re.findall('<(.*)>', each[1], re.I | re.S)
            if len(string) == 0:
                abstract_and_keywords.append(each[1].strip())
        paper_list.append(abstract_and_keywords)
        time.sleep(2)
    return paper_list


if __name__== '__main__':
    user_agent = 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:47.0) Gecko/20100101 Firefox/47.0'

    #这里url连接需要改，我用的是学校图书馆已经购买的万方数据库连接
    url_1 = 'http://s.g.wanfangdata.com.cn/Paper.aspx?q=%E4%B8%93%E4%B8%9A%3a%22%E4%B8%AD%E5%9B%BD%E8%AF%AD%E8%A8%'

    url_1_list = page_link_1(url_1)
    paper_title_dict = page_link_2(url_1_list, user_agent)
    paper_list = page_link_3(paper_title_dict)
    print 'the number of paper is %d' %len(paper_list)
    k = 1

#将文章摘要和关键词写入文件
    with open('paper_abstract_keywords_language.txt', 'a') as f:
        for each_paper in paper_list:
            keywords_length = len(each_paper)
            if keywords_length != 0:
                for i in range(keywords_length):
                    f.writelines(each_paper[i] + '+')
                f.writelines('\n')
                k += 1
            else:
                continue
    print 'the number of writed paper is %d' %k