获取nginx日志要写grok 还有很多正则来做
那么很多像我一样的新手不知道该如何操作
下面我们来个简单的
第一种 :
重点是: 把nginx的access.log日志格式改成json类型
更重要的是下面两行
log_format json '{"@timestamp":"$time_iso8601",'
'"host":"$server_addr",'
'"clientip":"$remote_addr",'
'"request":"$request",'
'"size":$body_bytes_sent,'
'"responsetime":$request_time,'
'"upstreamtime":"$upstream_response_time",'
'"upstreamhost":"$upstream_addr",'
'"http_host":"$host",'
'"url":"$uri",'
'"referer":"$http_referer",'
'"agent":"$http_user_agent",'
'"status":"$status"}';
access_log /data/nginx/logs/access_json.log json;
上面字体 颜色 一种颜色是一行
把这两行加到nginx.conf的http里面
如下代码:
http {
include mime.types;
default_type application/octet-stream;
log_format json '{"@timestamp":"$time_iso8601",'
'"host":"$server_addr",'
'"clientip":"$remote_addr",'
'"request":"$request",'
'"size":$body_bytes_sent,'
'"responsetime":$request_time,'
'"upstreamtime":"$upstream_response_time",'
'"upstreamhost":"$upstream_addr",'
'"http_host":"$host",'
'"url":"$uri",'
'"referer":"$http_referer",'
'"agent":"$http_user_agent",'
'"status":"$status"}';
access_log /data/nginx/logs/access_json.log json;
server_names_hash_bucket_size 128;
client_header_buffer_size 32K;
large_client_header_buffers 4 32k;
--------------------------以下省略
重启nginx 则在/data/nginx/logs/看到access_json.log的日志文件
下面我们写logstash的配置
我们配置文件是输出到redis里面,如果是直接写到es里面。需要改动
input {
file {
path => ['/data/nginx/logs/access_json.log']
start_position => "beginning"
codec => "json"
tags => ['user']
type => "nginx"
}
}
output {
if [type] == "nginx" {
redis {
host => "172.17.0.90"
port => "6379"
key => "nginx"
db => "10"
data_type => "list"
}
}
}
上面的配置文件就不做多解释 其它文章里面会介绍到
接下来就可以操作kibana了加索引了。会看到更多的列了
第二种 是后面发现的
https://grafana.com/dashboards/2292
参考grafana.com的
这个比第一种的全面
定义日志类型
log_format main '{"@timestamp":"$time_iso8601",'
'"@source":"$server_addr",'
'"hostname":"$hostname",'
'"ip":"$http_x_forwarded_for",'
'"client":"$remote_addr",'
'"request_method":"$request_method",'
'"scheme":"$scheme",'
'"domain":"$server_name",'
'"referer":"$http_referer",'
'"request":"$request_uri",'
'"args":"$args",'
'"size":$body_bytes_sent,'
'"status": $status,'
'"responsetime":$request_time,'
'"upstreamtime":"$upstream_response_time",'
'"upstreamaddr":"$upstream_addr",'
'"http_user_agent":"$http_user_agent",'
'"https":"$https"'
'}';
logstash的配置文件
input {
file {
#这里根据自己日志命名使用正则匹配所有域名访问日志
#path => [ "/usr/local/nginx/logs/*_access.log" ]
path => ['/data/nginx/logs/access_json.log']
start_position => "beginning"
codec => "json"
tags => ['user']
type => "nginx"
}
}
filter {
mutate {
convert => [ "status","integer" ]
convert => [ "size","integer" ]
convert => [ "upstreatime","float" ]
remove_field => "message"
}
geoip {
source => "ip"
}
}
output {
if [type] == "nginx" {
redis {
host => "172.17.0.90"
port => "6379"
key => "nginx"
db => "10"
data_type => "list"
}
}
}