jsoup的使用
使用Jsoup来爬取学校公告,配合Recyclerview显示出来(这个测试最好在白天,晚上官网有时候不能访问)
github源码地址:点击查看
效果如图:
首先添加依赖
implementation 'org.jsoup:jsoup:1.11.3'
implementation 'com.android.support:appcompat-v7:28.0.0'
implementation 'com.android.support:recyclerview-v7:28.0.0'
打开我们要爬的目标网址,我这里选择了我们学校图书馆的公告
http://www.lib.wust.edu.cn/bullet/bullet.aspx
点击右键查看网页源代码:
找到一个id为content的table
可以看到通知列表在其中的 第一个tr -> 第二个td -> 第四个div中
这个一定要看清楚,别数错了,一会爬取数据时要用
activity_main:
扫描二维码关注公众号,回复:
9392049 查看本文章
<?xml version="1.0" encoding="utf-8"?>
<LinearLayout xmlns:android="http://schemas.android.com/apk/res/android"
xmlns:app="http://schemas.android.com/apk/res-auto"
xmlns:tools="http://schemas.android.com/tools"
android:layout_width="match_parent"
android:layout_height="match_parent"
android:orientation="vertical"
tools:context=".MainActivity">
<androidx.appcompat.widget.Toolbar
android:id="@+id/toolbar"
android:layout_width="match_parent"
android:layout_height="?attr/actionBarSize"
android:background="@color/colorPrimary"
app:title="校园公告"
app:titleTextColor="#fff" />
<androidx.recyclerview.widget.RecyclerView
android:id="@+id/student_list"
android:layout_width="match_parent"
android:layout_height="match_parent">
</androidx.recyclerview.widget.RecyclerView>
</LinearLayout>
News:
package com.example.jsouptest;
public class News {
private String title;
private String date;
public String getDate() {
return date;
}
public void setDate(String date) {
this.date = date;
}
public String getTitle() {
return title;
}
public void setTitle(String title) {
this.title = title;
}
}
newsitem:
<?xml version="1.0" encoding="utf-8"?>
<LinearLayout xmlns:android="http://schemas.android.com/apk/res/android"
xmlns:app="http://schemas.android.com/apk/res-auto"
android:layout_width="match_parent"
android:layout_height="wrap_content"
android:elevation="8dp"
android:orientation="vertical">
<RelativeLayout
android:layout_width="match_parent"
android:layout_height="wrap_content">
<ImageView
android:id="@+id/ad"
android:layout_alignParentLeft="true"
android:layout_marginTop="5dp"
android:layout_marginLeft="10dp"
android:background="@drawable/ad"
android:layout_width="25dp"
android:layout_height="25dp"/>
<TextView
android:text="公告1"
android:id="@+id/head"
android:layout_toRightOf="@id/ad"
android:textSize="18sp"
android:layout_marginTop="5dp"
android:layout_marginLeft="10dp"
android:layout_width="wrap_content"
android:layout_height="wrap_content"/>
<androidx.appcompat.widget.AppCompatTextView
android:id="@+id/title_text"
android:layout_below="@id/head"
android:layout_width="match_parent"
android:layout_marginTop="5dp"
android:layout_centerVertical="true"
android:layout_height="wrap_content"
android:lines="2"
android:textColor="#000"
android:layout_marginLeft="10dp"
android:text="关于一次大型会议的记录关于一次大型会议的记录..."
android:textSize="20dp" />
<androidx.appcompat.widget.AppCompatTextView
android:layout_below="@id/title_text"
android:gravity="end"
android:id="@+id/date_text"
android:layout_width="match_parent"
android:layout_height="wrap_content"
android:layout_marginEnd="15dp"
android:layout_marginRight="15dp"
android:layout_marginBottom="5dp"
android:textColor="#000"
android:text="2017-08-15"
android:textSize="16sp" />
</RelativeLayout>
<View
android:layout_width="match_parent"
android:layout_height="0.5dp"
android:layout_marginLeft="5dp"
android:layout_marginTop="5dp"
android:layout_marginRight="5dp"
android:layout_marginBottom="5dp"
android:background="#ACACA9" />
</LinearLayout>
NewsAdapter:
package com.example.jsouptest;
import android.content.Context;
import android.view.LayoutInflater;
import android.view.View;
import android.view.ViewGroup;
import android.widget.TextView;
import java.util.ArrayList;
import java.util.List;
import androidx.annotation.NonNull;
import androidx.recyclerview.widget.RecyclerView;
public class NewsAdapter extends RecyclerView.Adapter<NewsAdapter.ViewHolder> {
List<News> newslist = new ArrayList<>();
LayoutInflater inflater;
Context context;
@Override
public NewsAdapter.ViewHolder onCreateViewHolder(@NonNull ViewGroup parent, int viewType) {
View view = inflater.inflate(R.layout.newsitem,null);
ViewHolder viewHolder = new ViewHolder(view);
return viewHolder;
}
@Override
public void onBindViewHolder(@NonNull NewsAdapter.ViewHolder holder, int position) {
News news = newslist.get(position);
int pos = position+1;
holder.title.setText(news.getTitle());
holder.date.setText(news.getDate());
holder.head.setText("公告"+pos);
}
@Override
public int getItemCount() {
return newslist.size();
}
public class ViewHolder extends RecyclerView.ViewHolder {
TextView head;
TextView title;
TextView date;
public ViewHolder(@NonNull View itemView) {
super(itemView);
head = itemView.findViewById(R.id.head);
date =itemView.findViewById(R.id.date_text);
title = itemView.findViewById(R.id.title_text);
}
}
public NewsAdapter(Context context,List<News> newslist){
this.newslist = newslist;
this.context = context;
inflater =LayoutInflater.from(context);
}
}
NewsJsoup(在这个中去爬取数据)(按照我们上面分析的位置去爬取,具体用法就按下面去做)
package com.example.jsouptest;
import android.util.Log;
import org.jsoup.Connection;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
public class NewsJsoup {
private String api = "http://www.lib.wust.edu.cn/bullet/bullet.aspx";
private static final String TAG = "NewsJsoup";
List<News> newslist = new ArrayList<>();
public List<News> GetNews() throws IOException {
Connection connection = Jsoup.connect(api);
connection.timeout(5000);
Document document = connection.get();
Element element = document.getElementById("content").select("tr").get(0)
.select("td").get(1).select("div").get(3);
Elements elements = element.children();
for (int i=0;i<elements.size();i++){
Element element1 = elements.get(i);
String text = element1.select("div").get(1).text().replace("»","");
String date = element1.select("div").get(2).text();
News news = new News();
news.setTitle(text);
news.setDate(date);
newslist.add(news);
}
return newslist;
}
}
MainActivity:(注意网络请求要放在子线程中,ui显示又要返回主线程中)
package com.example.jsouptest;
import androidx.appcompat.app.AppCompatActivity;
import androidx.appcompat.widget.Toolbar;
import androidx.recyclerview.widget.LinearLayoutManager;
import androidx.recyclerview.widget.RecyclerView;
import android.content.Context;
import android.os.Bundle;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
public class MainActivity extends AppCompatActivity {
private RecyclerView recyclerView;
Toolbar toolbar;
private List<News> news = new ArrayList<>();
Context context = MainActivity.this;
@Override
protected void onCreate(Bundle savedInstanceState) {
super.onCreate(savedInstanceState);
setContentView(R.layout.activity_main);
recyclerView = findViewById(R.id.student_list);
toolbar = findViewById(R.id.toolbar);
InitData();
}
private void InitData() {
new Thread(new Runnable() {
@Override
public void run() {
try {
NewsJsoup newsJsoup = new NewsJsoup();
news = newsJsoup.GetNews();
runOnUiThread(new Runnable() {
@Override
public void run() {
InitRecyclerView();
}
});
} catch (IOException e) {
e.printStackTrace();
}
}
}).start();
}
private void InitRecyclerView(){
LinearLayoutManager linearLayoutManager = new LinearLayoutManager(this);
recyclerView.setLayoutManager(linearLayoutManager);
NewsAdapter newsAdapter = new NewsAdapter(context,news);
recyclerView.setAdapter(newsAdapter);
}
}
别忘了对于网络的权限设置(这里是安卓9的网络权限)
新建一个叫network_security_config.xml的actions xml
<?xml version="1.0" encoding="utf-8"?>
<network-security-config>
<base-config cleartextTrafficPermitted="true">
<trust-anchors>
<certificates src="system" overridePins="true" />
<certificates src="user" overridePins="true" />
</trust-anchors>
</base-config>
</network-security-config>
然后在manifests中
<uses-permission android:name="android.permission.INTERNET" />
在applications中:
android:networkSecurityConfig="@xml/network_security_config"