int isSize = 1;
Integer pageSize = 0;
Integer timeType = 4; // 时间类型 6: 指定时间 5:近半年 0:今日 4:近3月
public String createhttpClient(Integer id, String kw, String startTime, String endTime) {
logger.info("++++++++++++++++++++++++++++++++开始拉取中国政府采购网数据+++++++++++++++++++++++++++++++++");
long start = System.currentTimeMillis();
Integer totalSize = null;
try {
Integer is_homepage = 0;
k = toUtf8String(kw);
s = URLEncoder.encode(startTime, “UTF-8”);
e = URLEncoder.encode(endTime, “UTF-8”);
pageSize = 1;
ur = “http://search.ccgp.gov.cn/bxsearch?searchtype=1&page_index=” + pageSize + “&bidSort=0&buyerName=&projectId=&pinMu=0&bidType=0&dbselect=bidx&kw=” + k + “&start_time=” + s + “&end_time=” + e + “&timeType=” + timeType + “&displayZone=&zoneId=&pppStatus=0&agentName=”;
totalSize = fetchHomePageUrl(ur);
} catch (Exception er) {
logger.error("========================获取中国政府采购网首页数据失败,重新获取中.....=================================");
String message = er.getMessage();
if (message != null) {
fetchException(message, id, pageSize, ur,0) ;
}
}
if (totalSize>0){
while (true){
//遍历每一页的url
/* for (int h = 1; h <= size; h++) {
if (isSize> size){
break;
}
pageSize = h;
// String urlt = "http://search.ccgp.gov.cn/bxsearch?searchtype=1&page_index=" + pageSize+ "&bidSort=0&buyerName=&projectId=&pinMu=0&bidType=0&dbselect=bidx&kw=" + k + "&start_time=" + s + "&end_time=" + e + "&timeType=" + timeType + "&displayZone=&zoneId=&pppStatus=0&agentName=";
fetchingUrl(id, h, ur);
isSize ++ ;
}*/
if (isSize> totalSize){
break;
}
pageSize = isSize;
fetchingPageUrl(id, pageSize, ur);
isSize++;
}
}else {
logger.info("关键词:"+kw + "没有查询到相关数据-"+new Date());
return null;
}
return null;
}
// 首次访问时的url ,需查找当前查询词所在的所总页数
public Integer fetchHomePageUrl(String ur){
Integer size = null;
try {
//maxBodySize : 0 不限制大小, 1: 1m userAgent: 伪装成浏览器
Document doc = Jsoup.connect(ur).header(“Accept-Encoding”, “gzip, deflate”)
// .userAgent(“Mozilla/5.0 (Windows NT 6.1; WOW64; rv:23.0) Gecko/20100101 Firefox/23.0”)
.userAgent(“Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.31 (KHTML, like Gecko) Chrome/26.0.1410.64 Safari/537.31”)
.maxBodySize(0).timeout(600000).get();
Elements eli = doc.select(".vT-srch-result-list-con2 .vT-srch-result-list .vt-srch-result-list-bid>li");
if (eli.size() > 0) {
//获取分页html
Elements allElements = doc.select(".vT_z .pager>script");
if (allElements.size() > 0) {
String pager = allElements.get(0).childNodes().toString();
String startStr = pager.substring(7, pager.length());
String objectStr = startStr.substring(0, startStr.lastIndexOf(")"));
//string转json对象
JSONObject jsonObject = JSONObject.parseObject(objectStr);
// 共有多少页
size = Integer.valueOf((Integer) jsonObject.get(“size”));
}
}
} catch (IOException e1) {
e1.printStackTrace();
}
return size;
}
// 遍历页数,存入数据库
public void fetchingPageUrl(Integer id, Integer pageSize, String fetchUrl) {
Integer is_homePage = 1; // 0 : 首页 1: 否
//存govCCgp表
GovCcgpModel govCcgpModel = new GovCcgpModel();
GovEnclosureModel enclosureModel = new GovEnclosureModel();
GovOutlineModel govOutlineModel = new GovOutlineModel();
GovTextModel govTextModel = new GovTextModel();
String noticeOutline = null;
String noticeText = null;
try {
Document pDoc = Jsoup.connect(fetchUrl).header("Accept-Encoding", "gzip, deflate")
// .userAgent("Mozilla/5.0 (Windows NT 6.1; WOW64; rv:23.0) Gecko/20100101 Firefox/23.0")
.userAgent("Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.31 (KHTML, like Gecko) Chrome/26.0.1410.64 Safari/537.31")
.maxBodySize(0)
.timeout(600000).get();
//根据css样式 获取当前页的所有url
Elements elements = pDoc.select(".vT_z .vT-srch-result-list-bid>li");
// Elements hrefs = doc.select("a[href]");
//遍历当前页的所有url
for (Element element : elements) {
List<String> names = new ArrayList<>();
List<String> enclousureList = new ArrayList<>();
Random rand = new Random();
//生成1-15的随机数,睡眠
Integer r = rand.nextInt(16) + 1;
Thread.sleep(r);
Elements childs = element.children();
//获取a标签的href 网址, select获取到当前a标签attr href获取到网址;
String url = childs.select("a").attr("href");
String str = url.substring(0, 10);
String govId = url.substring(22, url.length());
//判断a标签的网址是否为“http://www.”开头
// if (!str.equals("http://www")) return null;
Document document = Jsoup.connect(url).timeout(6000).get();
//取正文标题 div //进入url,将里面的内容处理存入数据库
// Elements eTitle = document.select(".vF_detail_header>h2");
// Elements eTime = document.select(".vF_detail_header>p");
//招标公告名称
String title = document.title();
//采购人信息
String purInformation = childs.get(2).text();
// 采购人信息 “|” 每一次出现的位置
int p = purInformation.indexOf("|");
SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy.MM.dd hh:mm:ss");
//公告标书时间
Date createTime = simpleDateFormat.parse(purInformation.substring(0, p));
//采购人
String purchaser = purInformation.substring(p + 1, purInformation.length());
//公告概要 表信息html
Elements outLine = document.select(".main .main_container .vF_deail_maincontent .vF_detail_main .table>table");
//解析表信息
Document outlineDoc = Jsoup.parse(String.valueOf(outLine));
//遍历每一行表格信息,判断 是否包含附件
Elements trs = outlineDoc.select("table").select("tr");
String texTtable = outlineDoc.select("table").html();
for (int i = 0; i < trs.size(); i++) {
//判断表格内是否存在附件
Elements tds = trs.get(i).select("td");
String enclosureStr = tds.get(0).text();
if ("附件:".equals(enclosureStr)) {
//存入附件表
for (int j = i + 1; j < trs.size(); j++) {
Elements tdsEnclo = trs.get(j).select("td");
String encloDoc = tdsEnclo.get(1).html();
//获取a标签的id
String uuid = tdsEnclo.get(1).select("a").attr("id");
if (uuid != null && uuid.length() > 0) {
//组合路径 下载附件
String enUrl = "http://www.ccgp.gov.cn/oss/download?uuid=" + uuid;
String fileName = tdsEnclo.get(1).text();
govCcgpModel.setIsEnclosure(0);
enclosureModel.setGovId(govId);
enclosureModel.setName(String.valueOf(fileName));
//指定下载到本地的路径
download(enUrl, fileName, PATH, enclosureModel);
}
}
break;
}
//存放在noticeountline 中每行信息以&/n&进行分隔;
if (noticeOutline != null && noticeOutline.length() != 0) {
noticeOutline = noticeOutline + "" + trs.get(i).text() + "&/n&";
} else {
noticeOutline = trs.get(i).text() + "&/n&";
}
}
if (noticeOutline != null && noticeOutline.length() != 0) {
//存入概要表
govOutlineModel.setGovId(govId);
govOutlineModel.setOutlineText(noticeOutline);
govOutlineModel.setOutlineTable(texTtable);
govOutlineDao.insertOutLine(govOutlineModel);
}
//存govCCgp表
Integer en_id = govCcgpModel.getIsEnclosure();
if (en_id == null ) {
govCcgpModel.setIsEnclosure(1);
}
govCcgpModel.setGovId(govId);
govCcgpModel.setPurchaser(purchaser);
govCcgpModel.setTitle(title);
govCcgpModel.setGov_keyword_id(id);
govCcgpModel.setIs_enable(0);
govCcgpModel.setSource_site("ccgp.gov.cn");//来源网站 中国政务采购网
govCcgpModel.setCreateTime(createTime);
govCcgpModel.setCreateOn(new Date());
//govId 是否存在, 存在 则更新
List<HashMap<String, String>> govidList = govCcgpDao.findGovId(govId);
if (govidList.size() > 0) {
govCcgpDao.updateGov(govCcgpModel);
} else {
govCcgpDao.insertGov(govCcgpModel);
}
govCcgpModel.setIsEnclosure(null);
//公告正文
Elements textEle = document.select(".main .main_container .vF_detail_main");
noticeText = textEle.text();
if (noticeText != null && noticeOutline.length() != 0) {
//存入公告正文表
govTextModel.setGovId(govId);
govTextModel.setText(noticeText);
govTextDao.insertGovText(govTextModel);
}
}
} catch (Exception e) {
System.out.println(e.getMessage());
String message = e.getMessage();
if (message != null) {
fetchException(message, id, pageSize, fetchUrl,is_homePage);
}
}
}
public void fetchException(String message, Integer id, Integer pageSize, String fetchUrl,Integer homePage) {
try {
Thread.sleep(60 * 1000);
if (message.equals(“HTTP error fetching URL”)) {
if(homePage == 0){
fetchHomePageUrl(fetchUrl);
}else {
fetchingPageUrl(id, pageSize, fetchUrl);
}
}
} catch (InterruptedException e) {
e.printStackTrace();
}
}