ELK的数据处理流程:
第一步:产生日志数据
spring-boot项目基本配置:
新建spring-boot项目2.1.14,勾选好下图两个依赖,开始新建项目
导入guava依赖
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>18.0</version>
</dependency>
<dependency>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
<version>1.2.17</version>
</dependency>
<!-- https://mvnrepository.com/artifact/com.fasterxml.jackson.core/jackson-core -->
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-core</artifactId>
<version>2.9.9</version>
</dependency>
<!-- https://mvnrepository.com/artifact/com.fasterxml.jackson.core/jackson-databind -->
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
<version>2.9.9</version>
</dependency>
<dependency>
<groupId>net.logstash.logback</groupId>
<artifactId>logstash-logback-encoder</artifactId>
<version>4.11</version>
</dependency>
并修改application.properties为application.yml,添加es的集群配置和dao接口配置
spring:
data:
elasticsearch:
cluster-nodes: niceday-es
cluster-name: 192.168.56.101:9300
repositories:
enabled: true
创建controller包
对线程生产的封装,我以将代码开源到github上,可以去star项目慢慢看
第二步:清洗日志数据格式
第三步:存储日志数据并分析
第四步:显示日志数据
control层
@RestController
public class UserActionCtl {
@Autowired
private UserActionServices uas;
@RequestMapping("/fenzu")
public List<VUserModelPie> groupAndCount(){
return uas.aggsSearch();
}
}
dao层
public interface UserActionDAO extends ElasticsearchRepository<UserAction,String> {
}
model层
browserInfos
public class BrowserInfos {
private String browserVersion;
private int custid;
private Events events;
private static BrowserInfos browserInfos;
public static BrowserInfos getBrowserInstance(){
if (browserInfos == null){
synchronized (BrowserInfos.class){
if (browserInfos == null){
browserInfos = new BrowserInfos();
}
}
}
return browserInfos;
}
private BrowserInfos(){}
public Events getEvents() {
return events;
}
public void setEvents(Events events) {
this.events = events;
}
public int getCustid() {
return custid;
}
public void setCustid(int custid) {
this.custid = custid;
}
public String getBrowserVersion() {
return browserVersion;
}
public void setBrowserVersion(String browserVersion) {
this.browserVersion = browserVersion;
}
}
events
public class Events {
private String eventCateGory;
private String position;
private String time;
private String pageName; // 哪一个页面
private String msg;
private static volatile Events events;
private Events() {
}
public static Events getEventInstance(){
if (events == null){
synchronized (Events.class){
if (events == null){
events = new Events();
}
}
}
return events;
}
public Events(String eventCateGory, String position, String time, String pageName, String msg) {
this.eventCateGory = eventCateGory;
this.position = position;
this.time = time;
this.pageName = pageName;
this.msg = msg;
}
public String getEventCateGory() {
return eventCateGory;
}
public void setEventCateGory(String eventCateGory) {
this.eventCateGory = eventCateGory;
}
public String getPosition() {
return position;
}
public void setPosition(String position) {
this.position = position;
}
public String getTime() {
return time;
}
public void setTime(String time) {
this.time = time;
}
public String getPageName() {
return pageName;
}
public void setPageName(String pageName) {
this.pageName = pageName;
}
public String getMsg() {
return msg;
}
public void setMsg(String msg) {
this.msg = msg;
}
}
useraction类
@Document(indexName = "users" ,type = "useraction")
public class UserAction implements Serializable {
@JsonProperty("id")
private String id;
@JsonProperty("pageName")
private String pageName;
@JsonProperty("time")
private String time;
@JsonProperty("position")
private String position;
@JsonProperty("msg")
private String msg;
@JsonProperty("custid")
private long custid;
@JsonProperty("browserVersion")
private String browserVersion;
@JsonProperty("eventCateGory")
private String eventCateGory;
public String getId() {
return id;
}
public void setId(String id) {
this.id = id;
}
public long getCustid() {
return custid;
}
public void setCustid(long custid) {
this.custid = custid;
}
public String getPageName() {
return pageName;
}
public void setPageName(String pageName) {
this.pageName = pageName;
}
public String getTime() {
return time;
}
public void setTime(String time) {
this.time = time;
}
public String getPosition() {
return position;
}
public void setPosition(String position) {
this.position = position;
}
public String getMsg() {
return msg;
}
public void setMsg(String msg) {
this.msg = msg;
}
public String getBrowserVersion() {
return browserVersion;
}
public void setBrowserVersion(String browserVersion) {
this.browserVersion = browserVersion;
}
public String getEventCateGory() {
return eventCateGory;
}
public void setEventCateGory(String eventCateGory) {
this.eventCateGory = eventCateGory;
}
}
vusermodepie类
public class VUserModelPie {
private String name;
private Long value;
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
public Long getValue() {
return value;
}
public void setValue(Long value) {
this.value = value;
}
}
service层
UserActionServices
@Service
public class UserActionServices {
@Autowired
private UserActionDAO uad;
// 分组查询
public List<VUserModelPie> aggsSearch() {
//第一个字段进行分组 聚合name之后查找自己起的叫abc名字的
TermsAggregationBuilder builder = AggregationBuilders.terms("niceday").field("eventCateGory");
NativeSearchQueryBuilder nsqb = new NativeSearchQueryBuilder();
// withSourceFilter通过拿到的数据源进行处理,FetchSourceFilter(想要的数组,不想要的数组)过滤后拿到一个数据源
// nsqb.withSourceFilter(new FetchSourceFilter(new String[]{""},null));
nsqb.addAggregation(builder);
AggregatedPage ap = (AggregatedPage) uad.search(nsqb.build());
// 拿到聚合后的表名
StringTerms abc = (StringTerms) ap.getAggregation("niceday");
//拿表名的捅字段
List<StringTerms.Bucket> buckets = abc.getBuckets();
List<VUserModelPie> vump = new ArrayList<>();
for (StringTerms.Bucket bucket : buckets) {
VUserModelPie vp = new VUserModelPie();
vp.setName(bucket.getKeyAsString());
vp.setValue((bucket.getDocCount()));
vump.add(vp);
}
return vump;
}
public Map<String,List<TempBIObject>> findCount(){
Script script = new Script("if(doc['time'].values.length == 6) if(doc['time'].values[5].length() == 4) doc['time'].values[5] + doc['time'].values[2] + '#' + doc['eventCateGory'].values");
TermsAggregationBuilder group = AggregationBuilders.terms("abc").script(script).size(1000);
NativeSearchQueryBuilder nqb = new NativeSearchQueryBuilder();
nqb.addAggregation(group);
AggregatedPage<Action> acts = (AggregatedPage<Action>)actionDAO.search(nqb.build());
StringTerms terms = (StringTerms)acts.getAggregation("abc");
List<StringTerms.Bucket> buckets = terms.getBuckets();
Map<String, List<TempBIObject>> change = change(buckets);
for (String s : change.keySet()) {
System.out.println(s + ":" );
}
return change;
}
private Map<String,List<TempBIObject>> change(List<StringTerms.Bucket> bck){
Map<String, List<TempBIObject>> groups = new HashMap<>();
// 将所有数据封装成对象中
for (StringTerms.Bucket bucket : bck) {
String[] split = bucket.getKeyAsString().split("#");
// 通过事件的名称到分组集合中获取事件组集合
List<TempBIObject> tbo = null;
//填充map的链表
if(groups.containsKey(split[1])){
tbo = groups.get(split[1]);
}else {
tbo = new ArrayList<>();
groups.put(split[1],tbo);+
}
// 塞成排序对象,将对象塞到链表里
TempBIObject tempBIObject = new TempBIObject();
tempBIObject.setTime(Long.parseLong(split[0]));
tempBIObject.setEvent(split[1]);
tempBIObject.setClickNum(bucket.getDocCount());
tbo.add(tempBIObject);
}
// 对分组数组中的对象进行排序
for (List<TempBIObject> val : groups.values() ){
// 给全局临时变量赋值
temp = val;
// 调用排序完成日期排序
quick(0,temp.size() - 1);
}
return groups;
}
private void quick(int low,int high){
int lo = low, hi = high;
if (lo >= hi)return;
boolean flag = false;
while(lo < hi){
// hi < lo 时
if (temp.get(lo).compareTo(temp.get(hi)) > 0){
TempBIObject tmTbo = temp.get(lo);
temp.set(lo,temp.get(hi));
temp.set(hi,tmTbo);
flag = !flag;
}
if (flag)lo++;else hi--;
}
lo--;hi++;
quick(low,lo);
quick(hi,high);
}
}
logFactory包:
logFactory
public interface LogFactory{
UserLog createUserLogThread();
ServerLog createServerLogThread();
}
logProduct
public class LogProduct implements LogFactory {
@Override
public UserLog createUserLogThread() {
return UserLog.getInstance("action");
}
@Override
public ServerLog createServerLogThread() {
return null;
}
}
serverlog类
public class ServerLog {
private ServerLog() {}
private static volatile ServerLog serverLog;
public static ServerLog getInstance(){
if (serverLog == null){
synchronized (UserLog.class){
if (serverLog == null){
serverLog = new ServerLog();
}
}
}
return serverLog;
}
public String makeServiceLog(){
try {
Properties properties = Properties.class.newInstance();
} catch (Exception e) {
e.printStackTrace();
}
return null;
}
}
userlog类
public class UserLog extends Thread {
private static volatile UserLog userlog;
private UserLog(){}
private UserLog(String threadName) {
setName(threadName);
}
// 单例
public static UserLog getInstance(String threadName){
if (userlog == null){
synchronized (UserLog.class){
if (userlog == null){
userlog = new UserLog(threadName);
}
}
}
return userlog;
}
@Override
public void run(){
InfosWriter infosWriter = new InfosWriter();
String name = Thread.currentThread().getName();
int circle = 300;
if (name.equalsIgnoreCase("action")){
circle = 100000;
}
for (int i = 0; i < circle; i++) {
infosWriter.writeLog(name);
}
}
// 用时1分钟
public static void main(String[] args) {
LogFactory uselog = new LogProduct();
uselog.createUserLogThread().start();
}
}
echarts前端ajax页面
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<title>Title</title>
</head>
<body>
<div id="phone" style="width: 600px;height: 400px"></div>
<script type="text/javascript" src="frame/jquery/jquery-3.5.1.js"></script>
<script type="text/javascript" src="frame/echarts/echarts.min.js"></script>
<script type="text/javascript">
$.ajax({
type : 'get',
url : 'fenzu',
dataType : 'json',
success : function (res) {
var mc = echarts.init(document.getElementById("phone"));
var option = {
title : {
text : "nicedays"
},
color : ["#ff8400","#FFFF00","#eee564"],
legend : {
data : ["销量"]
},
series : [
{
name : "销量",
type : "pie",
radius : '25%',
center : ['50%','50%'],
data : res,
animation : true
}
]
};
mc.setOption(option);
}
})
</script>
</body>
</html>