ELK采集nginx错误日志
一、filebeat采集配置
1、在nginx服务器上安装filebeat
wget https://artifacts.elastic.co/downloads/beats/filebeat/filebeat-6.3.1-x86_64.rpm
yum localinstall filebeat-6.3.1-x86_64.rpm
2、配置filebeat采集文件
vim /etc/filebeat/filebeat.yml
logging.level: info
logging.to_files: true
logging.files:
path: /data/logs/filebeat
name: filebeat.log
keepfiles: 7
permissions: 0644
filebeat.inputs:
- type: log
enabled: true
exclude_lines: ['\\x']
fields:
log-type: nginx-access-logs
paths:
- /data/logs_nginx/*.json.log
- type: log
enabled: true
fields:
log-type: nginx-error-logs
paths:
- /data/logs_nginx/error.log
output.kafka:
# initial brokers for reading cluster metadata
hosts: ["kafka1:9092", "kafka2:9092", "kafka3:9092"]
# message topic selection + partitioning
topic: '%{[fields][log-type]}'
partition.hash:
reachable_only: false
required_acks: 1
compression: snappy
max_message_bytes: 1000000
4、启动filebeat
systemctl start filebeat
二、配置logstash过滤规则并存储到elasticsearch
1、添加nginx错误日志grok表达式
cd /usr/share/logstash/patterns/
vim nginx
NGINX_ERROR_LOG (?<timestamp>%{YEAR}[./-]%{MONTHNUM}[./-]%{MONTHDAY}[- ]%{TIME}) \[%{LOGLEVEL:severity}\] %{POSINT:pid}#%{NUMBER}: %{GREEDYDATA:errormessage}(?:, client: (?<clientip>%{IP}|%{HOSTNAME}))(?:, server: %{IPORHOST:server}?)(?:, request: %{QS:request})?(?:, upstream: (?<upstream>\"%{URI}\"|%{QS}))?(?:, host: %{QS:request_host})?(?:, referrer: \"%{URI:referrer}\")?
2、配置logstash过滤nginx日志规则
cd /etc/logstash/conf.d
vim nginx-error.conf
input{
kafka{
bootstrap_servers => ["kafka1:9092,kafka2:9092,kafka3:9092"]
client_id => "nginx-error-logs"
group_id => "logstash"
auto_offset_reset => "latest"
consumer_threads => 10
decorate_events => true
topics => ["nginx-error-logs"]
type => "nginx-error-logs"
codec => json {charset => "UTF-8"}
}
}
filter {
if [fields][log-type] == "nginx-error-logs" {
grok {
match => [ "message" , "%{NGINX_ERROR_LOG}"]
}
geoip {
database =>"/usr/share/logstash/GeoLite2-City/GeoLite2-City.mmdb"
source => "clientip"
}
date {
timezone => "Asia/Shanghai"
match => ["timestamp","yyyy/MM/dd HH:mm:ss"]
}
}
}
output {
if [fields][log-type] == "nginx-error-logs" {
elasticsearch {
hosts => ["http://es1:9200","http://es2:9200","http://es3:9200"]
index => "nginx-error-%{+YYYY.MM.dd}"
}
}
}
3、重启logstash
systemctl restart logstash