Logstash filter{} 插件 grok,收集apache访问日志
- grok非常影响性能
- 不够灵活,除非很懂ruby
标准输入标准输出执行
[root@node1 conf.d]# cat grok.conf
input{
stdin {}
}
filter{
grok{
match => { "message" => "%{IP:client} %{WORD:method} %{URIPATHPARAM:request}%{NUMBER:bytes} %{NUMBER:duration}" }
}
}
output{
stdout{
codec => rubydebug
}
}
[root@node1 conf.d]# /opt/logstash/bin/logstash -f grok.conf
[root@node1 conf.d]# /opt/logstash/bin/logstash -f grok.conf
Settings: Default pipeline workers: 2
Pipeline main started
192.168.79.103 GET /index.html 15000 0.041
{
"message" => "192.168.79.103 GET /index.html 15000 0.041",
"@version" => "1",
"@timestamp" => "2018-08-24T07:55:10.386Z",
"host" => "node1",
"client" => "192.168.79.103",
"method" => "GET",
"request" => "/index.html",
"bytes" => "15000",
"duration" => "0.041"
}
标准输出,收集apache日志,使用grok插件
[root@node1 conf.d]# cat apache_grok.conf
input{
file{
path => "/var/log/httpd/access_log"
start_position => "beginning"
type => "apache_log"
}
}
filter{
grok {
match => { "message" => "%{COMBINEDAPACHELOG}" }
}
}
output{
stdout {
codec => rubydebug
}
}
[root@node1 conf.d]# /opt/logstash/bin/logstash -f apache_grok.conf
Settings: Default pipeline workers: 2
Pipeline main started
{
"message" => "192.168.79.1 - - [24/Aug/2018:16:14:59 +0800] \"GET /noindex/css/fonts/Bold/OpenSans-Bold.ttf HTTP/1.1\" 404 238 \"http://192.168.79.103/noindex/css/open-sans.css\" \"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36\"",
"@version" => "1",
"@timestamp" => "2018-08-24T08:14:59.771Z",
"path" => "/var/log/httpd/access_log",
"host" => "node1",
"type" => "apache_log",
"clientip" => "192.168.79.1",
"ident" => "-",
"auth" => "-",
"timestamp" => "24/Aug/2018:16:14:59 +0800",
"verb" => "GET",
"request" => "/noindex/css/fonts/Bold/OpenSans-Bold.ttf",
"httpversion" => "1.1",
"response" => "404",
"bytes" => "238",
"referrer" => "\"http://192.168.79.103/noindex/css/open-sans.css\"",
"agent" => "\"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36\""
}
收集至es
[root@node1 conf.d]# cat apache_grok.conf
input{
file{
path => "/var/log/httpd/access_log"
start_position => "beginning"
type => "apache_log"
}
}
filter{
grok {
match => { "message" => "%{COMBINEDAPACHELOG}" }
}
}
output{
elasticsearch{
hosts => ["192.168.79.103:9200"]
index => "apache_accesslog-%{+YYYY.MM.dd}"
}
}
[root@node1 conf.d]# /opt/logstash/bin/logstash -f apache_grok.conf