博客刷访问次数
import java.net.HttpURLConnection;
import java.net.URL;
import java.net.URLConnection;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.PageProcessor;
public class CSDNCrawler implements PageProcessor{
private int k=0;
private Site site=Site.me()
.setDomain("http://www.csdn.net/")
.setTimeOut(5000)
.setRetryTimes(2)
.setUserAgent(
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_2) AppleWebKit/537.31 (KHTML, like Gecko) Chrome/26.0.1410.65 Safari/537.31");
private static final String URL_LIST1="http://blog\\.csdn\\.net/khuangliang/article/list/\\d";
private static Spider sp=Spider.create(new CSDNCrawler());
@Override
public void process(Page page) {
// TODO Auto-generated method stub
if(page.getUrl().regex(URL_LIST1).match()){
System.out.println("分页链接");
//添加分页下所有文章的连接
page.addTargetRequests(page.getHtml().xpath("//h1").links().all());
page.addTargetRequests(page.getHtml().links().regex(URL_LIST1).all());
}else{
String curl=page.getUrl().toString();
System.out.println("文章连接"+curl);
for(int i=0;i<50;i++){
try {
URL url = new URL(curl);
URLConnection URLconnection = url.openConnection();
HttpURLConnection httpConnection = (HttpURLConnection) URLconnection;
httpConnection.setRequestProperty("User-Agent", "Mozilla/4.0 (compatible; MSIE 5.0; Windows NT; DigExt)");
int responseCode = httpConnection.getResponseCode();
if (responseCode == HttpURLConnection.HTTP_OK) {
k++;
System.err.println("成功"+k);
URLconnection.connect();
} else {
System.err.println("失败");
}
} catch (Exception e) {
e.printStackTrace();
}
}
}
}
@Override
public Site getSite() {
// TODO Auto-generated method stub
return site;
}
public static void main(String args[]){
for(int i=0;i<100;i++){
CSDNCrawler crawler=new CSDNCrawler();
crawler.sp.addUrl("http://blog.csdn.net/khuangliang/article/list/1")
.run();
}
}
}
(1) 首先通过爬虫获取所有的文章的连接,使用的是webmagic 国人自己写的爬虫。
(2)对于每个链接设置访问50次,通过爬虫来设置循环爬多少次试过不行,每次都得重新编译运行,看插件源码,是在run()结束后把所有的线程都关闭了,所以在主函数里写100次是没有用的