/**
* 网络爬虫测试
*
* @author Tunny
* @2018年8月11日
*/
public class WebCrawler {
public static void main(String[] args) {
Scanner can = new Scanner(System.in);
String url = "http://www.zysj.com.cn/lilunshuji/jichulilun/index.html";
url = can.nextLine();
CloseableHttpClient httpClient = HttpClients.createDefault();
HttpGet httpGet = new HttpGet(url);
httpGet.addHeader("Content-Type","application/x-www-form-urlencoded;charset=utf-8");
// 执行请求
HttpResponse response;
String line;
try {
response = httpClient.execute(httpGet);
HttpEntity httpEntity = response.getEntity();
BufferedReader bufferedReader = null;
bufferedReader = new BufferedReader(new InputStreamReader(
httpEntity.getContent(), "utf-8"), 8 * 1024);
StringBuilder entityStringBuilder = new StringBuilder();
while ((line = bufferedReader.readLine()) != null) {
entityStringBuilder.append(line + "\n");
}
// System.out.println(entityStringBuilder.toString());
// appendMethodB("f:/中医基础理论.html",entityStringBuilder.toString());
savaFile("C:/Users/tunny/Desktop/index.html",entityStringBuilder.toString(),"UTF-8");
} catch (ClientProtocolException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
}
}
/**
* 保存文件
* @param fileName 文件名称:绝对路径
* @param content 要保存的内容
* @param format 以某种格式保存文件
*/
public static void savaFile(String fileName, String content,String format) {
BufferedWriter rd=null;
OutputStream out=null;
File file = new File(fileName);
try {
out = new FileOutputStream(file);
rd = new BufferedWriter(new OutputStreamWriter(out,format));
rd.write(content);
} catch (IOException e) {
e.printStackTrace();
}finally{
if(null!=rd){
try {
rd.close();
} catch (IOException e) {
e.printStackTrace();
}
}
if(null!=out){
try {
out.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
}
/**
*
* @param fileName
* @param content
*/
public static void appendMethodB(String fileName, String content) {
FileWriter writer=null;
try {
writer = new FileWriter(fileName, false);
// 打开一个写文件器,构造函数中的第二个参数true表示以追加形式写文件
writer.write(content);
} catch (IOException e) {
e.printStackTrace();
}finally{
try {
if(null!=writer){
writer.close();
}
} catch (IOException e) {
e.printStackTrace();
}
}
}
}
网络爬虫测试
猜你喜欢
转载自blog.csdn.net/tun52099/article/details/81706612
今日推荐
周排行