一、ELK简介
Elk是指logstash,elasticsearch,kibana三件套,我们一般使用它们做日志分析。
ELK工作原理图:
简单来讲ELK具体的工作流程就是客户端的logstash agent(shipper)从日志里取出数据推送到服务端的redis里,服务端的logstash从redis里取出数据推送到elasticsearch并产生索引,然后使用Kibana进行页面展示。
二.ELK准备环境配置
1.搭建环境(都是在Centos6.8系统下操作完成):
Ip地址 节点 部署服务
192.168.100.10 ELK-node1 elasticsearch + logstrsh
192.168.100.20 ELK-node2 elasticsearch + redis + kibana
192.168.100.30 nginx-agent nginx + logstash
2.软件下载地址及版本:
cd /usr/local/src/
wget https:
//download.elasticsearch.org/elasticsearch/release/org/elasticsearch/distribution/tar/elasticsearch/2.3.0/elasticsearch-2.3.0.tar.gz
wget https:
//download.elastic.co/logstash/logstash/logstash-1.5.4.tar.gz
curl -L -O https:
//download.elastic.co/kibana/kibana/kibana-4.5.1-linux-x64.tar.gz
wget http:
//download.redis.io/releases/redis-3.0.7.tar.gz
3.各服务器安装jdk,jdk要1.7以上的版本.
rpm -ivh jre-8u91-linux-x64.rpm
node1 node2 节点部署elasticsearch + logstrsh.
1.安装elasticsearch
cd /usr/local/src/
wget https:
//download.elasticsearch.org/elasticsearch/release/org/elasticsearch/distribution/tar/elasticsearch/2.3.0/elasticsearch-2.3.0.tar.gz
tar -zxvf elasticsearch-2.3.0.tar.gz -C /usr/local/
ln -s elasticsearch-2.3.0 elasticsearch
mkdir -pv /usr/local/elasticsearch/data
mkdir -pv /usr/local/elasticsearch/logs
//日志文件目录
#主节点配置如下:
grep
'^[a-z]'
elasticsearch.yml
cluster.name: ELK-elasticsearch
node.name: node-1
path.data: /usr/local/elasticsearch/data
path.logs: /usr/local/elasticsearch/logs
bootstrap.mlockall:
true
network.host: 192.168.100.10
http.port: 9200
node.master:
true
node.data:
true
#集群节点参数详解:
https:
//my.oschina.net/liuxundemo/blog/688736?p={{page}}
启动elasticsearch:
elasticsearch只能以普通用户运行
创建elasticsearch运行用户,es只能用普通用户启动.
groupadd elasticsearch
useradd -g elasticsearch elasticsearch
chown -R elasticsearch:elasticsearch /usr/local/elasticsearch
/usr/local/elasticsearch/bin/elasticsearch >/dev/
null
2>&1 &
服务管理的插件:
elasticsearch的插件应用
http:
//www.cnblogs.com/xing901022/p/5962722.html
安装集群管理插件:
/usr/local/elasticsearch/bin/plugin install mobz/elasticsearch-head
http:
//192.168.100.10:9200/_plugin/head/
health状况:
curl
'192.168.100.10:9200/_cat/health?v'
epoch timestamp cluster status node.total node.data shards pri relo init unassign pending_tasks max_task_wait_time active_shards_percent
1488389751 01:35:51 elasticsearch green 1 1 0 0 0 0 0 0 - 100.0%
curl -X GET http:
//192.168.100.10:9200 //获取网页内容
curl -I GET http:
//192.168.100.10:9200 //获取网页头部信息,200正常
从节点192.168.100.20 node2 elasticsearch配置如下:
#从节点配置如下:
cluster.name: ELK-elasticsearch
node.name: node-2
path.data: /usr/local/elasticsearch/data
path.logs: /usr/local/elasticsearch/logs
bootstrap.mlockall:
true
network.host: 192.168.100.20
http.port: 9200
discovery.zen.ping.unicast.hosts: [
"192.168.100.10:9300"
]
LogStash部署和使用(主节点和从节点一样,只要服务启动正常即可):
cd /usr/local/src/
wget https:
//download.elastic.co/logstash/logstash/logstash-1.5.4.tar.gz
[root@localhost local]# tar -zxvf logstash-1.5.4.tar.gz -C /usr/local/
[root@localhost local]#ln -s logstash-1.5.4 logstash
标准输入输出:
/usr/local/logstash/bin/logstash -e
'input { stdin{} } output { stdout{}}'
#加东西可以改变输出:
[root@localhost ~]# /usr/local/logstash/bin/logstash -e
'input { stdin{} } output { stdout{codec => rubydebug}}'
#标准输出到elasticsearch中,定义host和协议
/usr/local/logstash/bin/logstash -e
'input { stdin{} } output { elasticsearch { host => "192.168.100.10" protocol => "http"}}'
#多重输出
/usr/local/logstash/bin/logstash -e
'input { stdin{} } output { elasticsearch { host => "192.168.100.10" protocol => "http"} stdout{ codec => rubydebug }}'
启动logstrash
/usr/local/logstash/bin/logstash -f /usr/local/logstash/config/logstrash.conf
#logstash解析nginx时间字段
http:
//blog.csdn.net/jianblog/article/details/54585043
logstash配置文件编写:
#文件输入到文件输出.
logstash.conf
input {
file{
path =>
"/var/log/messages"
}
}
output {
file {
path =>
"/tmp/%{+YYYY-MM-dd}.messages.gz"
gzip =>
true
}
}
#文件输入到文件和es中
input {
file{
path =>
"/var/log/messages"
}
}
output {
file {
path =>
"/tmp/%{+YYYY-MM-dd}.messages.gz"
gzip =>
true
}
elasticsearch {
host =>
"192.168.100.10"
protocol =>
"http"
index =>
"sysem-messages-%{+YYYY.MM.dd}"
}
}
启动logstrash
/usr/local/logstash/bin/logstash -f /usr/local/logstash/config/logstrash.conf
安装redis保证服务启动正常即可:
wget http:
//download.redis.io/releases/redis-3.0.7.tar.gz
tar -zxvf redis-3.0.7
make && make install
[root@localhost ~]#nohup redis-server 2>&1 &
[root@localhost ~]# redis-cli -h 192.168.100.20 -p 6379
192.168.21.128:6379>
select
1
OK
192.168.100.20:6379[1]> keys *
1)
"sys-messages"
192.168.100.20:6379[1]> LLEN sys-messages #查看redis 列表的长度
(integer) 42120
192.168.100.20:6379[1]> LINDEX sys-messages -1 #查看列表最后的一行
"{\"message\":\"hello logstrash to redis sucess\",\"@version\":\"1\",\"@timestamp\":\"2017-02-19T02:35:44.082Z\",\"host\":\"localhost.localdomain\",\"path\":\"/var/log/messages\"}"
192.168.100.20:6379[1]>
安装kibana:
安装kibana
curl -L -O https:
//download.elastic.co/kibana/kibana/kibana-4.5.1-linux-x64.tar.gz
tar -zxvf kibana-4.5.1-linux-x64.tar.gz -C /usr/local
ln -s kibana-4.5.1-linux-x64 kibana
vim kibana.yml
elasticsearch_url:
"http://192.168.100.10:9200"
#es主节点的ip地址
启动kibana服务
/usr/local/kibana/bin/kibana &
访问地址:
http:
//192.168.100.20:5601/app/kibana
在192.168.100.30 上安装nginx和logstash作为nginx日志输出端.
ELK收集Nginx日志有两种方式:
1.使用codec的json插件将日志的域进行分段,使用key-value的方式,使日志格式更清晰,易于搜索,还可以降低cpu的负载 ,更改nginx的配置文件的日志格式,使用json
2.不用修改Nginx的日志格式,通过filter过滤器来改变日志的格式.
我们这里先讲第一种方式:Nginx日志改成json输出。
logstash_nginx端的日志格式如下:
[root@logstash_nginx ~]#sed -n
'15,33p'
/etc/nginx/nginx.conf
log_format logstash_json
'{ "@timestamp": "$time_local", '
'"@fields": { '
'"remote_addr": "$remote_addr", '
'"remote_user": "$remote_user", '
'"body_bytes_sent": "$body_bytes_sent", '
'"request_time": "$request_time", '
'"status": "$status", '
'"request": "$request", '
'"request_method": "$request_method", '
'"http_referrer": "$http_referer", '
'"body_bytes_sent":"$body_bytes_sent", '
'"http_user_agent": "$http_user_agent" , '
'"http_x_forwarded_for": "$http_x_forwarded_for"} }'
;
access_log /
var
/log/nginx/access.log logstash_json;
启动nginx
[root@logstash_nginx ~]# /usr/sbin/nginx -t
nginx: the configuration file /etc/nginx/nginx.conf syntax
is
ok
nginx: configuration file /etc/nginx/nginx.conf test
is
successful
[root@logstash_nginx ~]# /usr/sbin/nginx
logstash配置如下:
#使用logstash收集nginx的访问日志
#vim /usr/local/logstash/logstash-agent.conf
input {
file {
type =>
"nginx-access-log"
path =>
"/var/log/nginx/access.log"
codec =>
"json"
}
}
filter {
}
output {
redis {
data_type =>
"list"
key =>
"nginx-access-log"
host =>
"192.168.100.20"
port =>
"6379"
db =>
"1"
}
}
#logstash indexer端文件,从redis里面读取数据然后在es中
vim /usr/local/logstash/logstash_indexer.conf
input{
redis{
data_type =>
"list"
key =>
"nginx-access-log"
host =>
"redis-ip"
port =>
"6379"
db =>
"1"
}
}
output {
elasticsearch {
host =>
"192.168.100.10"
protocol =>
"http"
index =>
"logstash-nginx-access-log-%{+YYYY.MM.dd}"
}
}
确定没有问题后,重新启动logstash
/usr/local/logstash/bin/logstash -f /usr/local/logstash/bin/logstash-agent.conf
/usr/local/logstash/bin/logstash -f /usr/local/logstash/bin/logstash_indexer.conf
我们可以访问http://192.168.100.10:9200/_plugin/head/,如果配置无误可以看到nginx-access-log这个索引,说明logstash已经将日志发送到es端了.
然后在kibana里面创建你的索引即可.
这样就可以收集到日志了。
elasticsearch的插件应用:
http://www.cnblogs.com/xing901022/p/5962722.html
使用Kibana 分析Nginx 日志并在 Dashboard上展示
http://www.cnblogs.com/hanyifeng/p/5860731.html
参考文档:
kibana画图参考文档:
http://blog.csdn.net/ming_311/article/details/50619859
http://www.cnblogs.com/hanyifeng/p/5857875.html
http://blog.oldboyedu.com/elk/
http://www.cnblogs.com/galengao/p/5780588.html
http://blog.csdn.net/wanglipo/article/details/50739820
http://www.jianshu.com/p/66e03eb6d95a
http://www.cnblogs.com/skyblue/p/5504595.html
Kibana反向代理配置:
1.Nginx安装(略):
2.Nginx配置 (kibana_proxy.conf)
#The default server
upstream kibana_proxy {
server kibana-ip-address:5601;
}
server {
listen 80;
server_name elk.xxx.com;
location / {
index index.html index.htm;
auth_basic
"welcome to kibana"
;
auth_basic_user_file /etc/nginx/passwd.db;
proxy_pass http:
//kibana_proxy;
}
location /status {
stub_status
on
;
access_log /
var
/log/nginx/kibana_status.log;
auth_basic
"NginxStatus"
;
}
}
#生成认证登录的用户名和密码:
[root@elk-node conf.d]# htpasswd -c /etc/nginx/passwd.db admin
New password:
Re-type
new
password:
Adding password
for
user admin
# chmod 400 /etc/nginx/passwd.db //修改网站认证数据库权限
# chown nginx. /etc/nginx/passwd.db //修改网站认证数据库属主和属组
# cat /etc/nginx/passwd.db //可以看到通过htpasswd生成的密码为加密格式
admin:8eZAz7BqcrXmY