ELK日志分析系统
一、安装Logstash
1.安装jdk
Logstash的运行依赖于Java环境
[root@localhost ~]# yum -y install java-1.8.0
[root@localhost ~]# java -version
openjdk version "1.8.0_131"
OpenJDK Runtime Environment (build 1.8.0_131-b11)
OpenJDK 64-Bit Server VM (build 25.131-b11, mixed mode)
2.安装Logstash
[root@localhost ~]# cd /opt/software/
[root@localhost software]# wget https://download.elastic.co/logstash/logstash/logstash-1.5.4.tar.gz
[root@localhost software]# tar zxvf logstash-1.5.4.tar.gz -C /usr/local/
3.配置Logstash的环境变量
[root@localhost software]# echo "export PATH=\$PATH:/usr/local/logstash-1.5.4/bin" > /etc/profile.d/logstash.sh
[root@localhost software]# . /etc/profile
[root@localhost software]#
4.启动Logstash(注:-e:指定Logstash的配置信息,可以用于快速测试;-f:指定Logstash的配置文件,可以用于生产环境。)
4.1 -e参数指定logstash的配置信息,用于快速测试,直接输出到屏幕
[root@localhost software]# logstash -e "input {stdin{}} output {stdout{}}"
4.2通过-e参数指定logstash的配置信息,用于快速测试,以json格式输出到屏幕
[root@localhost software]# logstash -e 'input{stdin{}}output{stdout{codec=>rubydebug}}'
5.logstash以配置文件方式启动
5.1输出信息到屏幕
[root@localhost software]# vim logstash-simple.conf
[root@localhost software]# cat logstash-simple.conf
input { stdin {} }
output {
stdout { codec=> rubydebug }
}
普通方式启动
[root@localhost software]# logstash -f logstash-simple.conf
以debug模式开启
[root@localhost software]# logstash agent -f logstash-simple.conf --verbose
5.2 logstash输出信息存储到redis数据库中
[root@localhost software]# vim logstash_to_redis.conf
[root@localhost software]# cat logstash_to_redis.conf
input { stdin { } }
output {
stdout { codec => rubydebug }
redis {
host => '192.168.11.43'
data_type => 'list'
key => 'logstash:redis'
}
}
二、Redis
1.安装Redis
[root@localhost software]# wget http://download.redis.io/releases/redis-2.8.19.tar.gz
[root@localhost software]# yum -y install tcl
[root@localhost software]# tar zxvf redis-2.8.19.tar.gz
[root@localhost software]# cd redis-2.8.19
[root@localhost redis-2.8.19]# make MALLOC=libc
安装gcc
[root@localhost software]# yum -y install gcc
[root@localhost software]# cd redis-2.8.19
[root@localhost redis-2.8.19]# make MALLOC=libc
[root@localhost redis-2.8.19]# make test
[root@localhost redis-2.8.19]# make install
[root@localhost redis-2.8.19]# cd utils/
[root@localhost utils]# ./install_server.sh
2.查看redis的监控端口
[root@localhost utils]# netstat -tlnp | grep redis
tcp 0 0 0.0.0.0:6379 0.0.0.0:* LISTEN 30344/redis-server
tcp 0 0 :::6379 :::* LISTEN 30344/redis-server
3.测试redis是否正常工作
[root@localhost utils]# cd ..
[root@localhost redis-2.8.19]# cd src/
[root@localhost src]# ./redis-cli -h 192.168.11.43 -p 6379
192.168.11.43:6379> ping
PONG
4.redis服务启动
[root@localhost ~]# cd /opt/software/
[root@localhost software]# ps -ef | grep redis
root 30344 1 0 10:43 ? 00:00:00 /usr/local/bin/redis-server *:6379
root 30367 1628 0 10:51 pts/0 00:00:00 ./redis-cli -h 192.168.11.43 -p 6379
root 30370 28369 0 10:52 pts/2 00:00:00 grep redis
5.Redis的动态监控
[root@localhost software]# cd redis-2.8.19/src/
[root@localhost src]# ./redis-cli monitor
OK
6.Logstash结合redis工作
6.1确认redis服务是启动的
[root@localhost src]# ps -ef | grep redis
root 30344 1 0 10:43 ? 00:00:01 /usr/local/bin/redis-server *:6379
root 30367 1628 0 10:51 pts/0 00:00:00 ./redis-cli -h 192.168.11.43 -p 6379
root 30379 28369 0 10:56 pts/2 00:00:00 grep redis
6.2启动redis动态监控
[root@localhost src]# ./redis-cli monitor
OK
6.3基于入口redis启动logstash
[root@localhost software]# vim logstash_to_redis.conf
[root@localhost software]# cat logstash_to_redis.conf
input { stdin { } }
output {
stdout { codec => rubydebug }
redis {
host => '192.168.11.43'
data_type => 'list'
key => 'logstash:redis'
}
}
[root@localhost software]# logstash agent -f logstash_to_redis.conf --verbose
如果redis的监控上也有以上信息输出,表示Logstash和redis的结合是正常的。
三、Elasticsearch
1.安装Elasticsearch
[root@localhost software]# wget https://download.elastic.co/elasticsearch/elasticsearch/elasticsearch-1.7.2.tar.gz
[root@localhost software]# tar zxvf elasticsearch-1.7.2.tar.gz -C /usr/local
2.修改Elasticsearch配置文件elasticsearch.yml并且做以下修改:
[root@localhost software]# vim /usr/local/elasticsearch-1.7.2/config/elasticsearch.yml
discovery.zen.ping.multicast.enabled: false(关闭广播,如果局域网有机器开9300端口,服务会启动不了)
network.host: 192.168.11.43 (指定主机地址)
追加2行
http.cors.allow-origin: "/.*/"
http.cors.enabled: true
3.启动Elasticsearch服务
[root@localhost software]# /usr/local/elasticsearch-1.7.2/bin/elasticsearch
注:日志会输出到stdout
[root@localhost software]# /usr/local/elasticsearch-1.7.2/bin/elasticsearch -d
注:表示以daemon方式启动
[root@localhost software]# nohup /usr/local/elasticsearch-1.7.2/bin/elasticsearch > /var/log/logstash.log 2>&1 &
4.查看Elasticsearch的监听端口
[root@localhost software]# netstat -tnlp| grep java
tcp 0 0 ::ffff:192.168.11.43:9200 :::* LISTEN 30514/java
tcp 0 0 ::ffff:192.168.11.43:9201 :::* LISTEN 30588/java
tcp 0 0 ::ffff:192.168.11.43:9202 :::* LISTEN 30645/java
tcp 0 0 ::ffff:192.168.11.43:9300 :::* LISTEN 30514/java
tcp 0 0 ::ffff:192.168.11.43:9301 :::* LISTEN 30588/java
tcp 0 0 ::ffff:192.168.11.43:9302 :::* LISTEN 30645/java
5.Elasticsearch和Logstash结合
将Logstash的信息输出到Elasticsearch中
[root@localhost software]# vim logstash-elasticsearch.conf
[root@localhost software]# cat logstash-elasticsearch.conf
input { stdin {} }
output {
elasticsearch { host => "192.168.11.43" }
stdout { codec=> rubydebug }
}
6.基于配置文件启动Logstash
[root@localhost software]# /usr/local/logstash-1.5.4/bin/logstash agent -f logstash-elasticsearch.conf
7.curl命令发送请求来查看Elasticsearch是否接收了数据
[root@localhost software]# curl http://192.168.11.43:9200/_search?pretty
{
"took" : 161,
"timed_out" : false,
"_shards" : {
"total" : 0,
"successful" : 0,
"failed" : 0
},
"hits" : {
"total" : 0,
"max_score" : 0.0,
"hits" : [ ]
}
}
8.安装Elasticsearch插件
Elasticsearch-kopf插件可以查询Elasticsearch中的数据,安装elasticsearch-kopf,只要在你安装Elasticsearch的目录中执行以下命令即可:
[root@localhost software]# cd /usr/local/elasticsearch-1.7.2/bin/
[root@localhost bin]# ./plugin install lmenezes/elasticsearch-kopf
注:附手动下载
手动下载该软件,不通过插件安装命令...
cd /usr/local/elasticsearch-1.7.2/plugins
wget https://github.com/lmenezes/elasticsearch-kopf/archive/master.zip
unzip master.zip
mv elasticsearch-kopf-master kopf
以上操作就完全等价于插件的安装命令
9.浏览器访问kopf页面访问elasticsearch保存的数据
[root@localhost bin]# netstat -tnlp | grep java
tcp 0 0 ::ffff:192.168.11.43:9200 :::* LISTEN 30514/java
tcp 0 0 ::ffff:192.168.11.43:9201 :::* LISTEN 30588/java
tcp 0 0 ::ffff:192.168.11.43:9202 :::* LISTEN 30645/java
tcp 0 0 ::ffff:192.168.11.43:9300 :::* LISTEN 30514/java
tcp 0 0 ::ffff:192.168.11.43:9301 :::* LISTEN 30588/java
tcp 0 0 ::ffff:192.168.11.43:9302 :::* LISTEN 30645/java
tcp 0 0 :::9303 :::* LISTEN 30698/java
10.从redis数据库中读取然后输出到elasticsearch中
[root@localhost software]# vim logstash-redis.conf
[root@localhost software]# cat logstash-redis.conf
input {
redis {
host => '192.168.11.43'
data_type => 'list'
port => "6379"
key => 'logstash:redis'
type => 'redis-input'
}
}
output {
elasticsearch {
host => "192.168.11.43"
codec => "json"
protocol => "http"
}
}
四、Kinaba
1.安装Kinaba
[root@localhost software]# wget https://download.elastic.co/kibana/kibana/kibana-4.1.2-linux-x64.tar.gz
[root@localhost software]# tar zxvf kibana-4.1.2-linux-x64.tar.gz -C /usr/local
2.修改kinaba配置文件kinaba.yml
[root@localhost software]# vim /usr/local/kibana-4.1.2-linux-x64/config/kibana.yml
elasticsearch_url: "http://192.168.11.43:9200"
3.启动kinaba
[root@localhost software]# /usr/local/kibana-4.1.2-linux-x64/bin/kibana
输出以上信息,表明kinaba成功。
kinaba默认监听在本地的5601端口上 。
4.浏览器访问kinaba
4.1 使用默认的logstash-*的索引名称,并且是基于时间的,点击“Create”即可
看到如下界面说明索引创建完成。
点击“Discover”,可以搜索和浏览Elasticsearch中的数据。