目录
1【elk】
Linux MySQL数据库数据同步Windows ES。
logstash配置文件,索引名必须小写。
2【Linux启动elk】
安装好了jdk11、elasticsearch、logstash、kibana、node.js、esHead插件,并在Windows浏览器上通过kibana看到了logstash同步mysql数据的效果。
2.1【启动elasticsearch】
[root@hadoop100 ~]# su es
[es@hadoop100 root]$ cd /opt/module/elasticsearch-8.5.1/bin
[es@hadoop100 bin]$ ./elasticsearch
2.2【启动kibana】
[root@hadoop100 ~]# jps
2992 CliToolLauncher
5233 Jps
3174 Elasticsearch
[root@hadoop100 ~]# cd /opt/module/kibana-8.5.1/bin
[root@hadoop100 bin]# ./kibana
Kibana should not be run as root. Use --allow-root to continue.
[root@hadoop100 bin]# su es
[es@hadoop100 bin]$ ./kibana
[2023-03-16T16:24:39.182+08:00][INFO ][node] Kibana process configured with roles: [background_tasks, ui]
2.3【启动logstash】
连接成功
Last login: Thu Mar 16 16:24:51 2023 from 192.168.88.1
[root@hadoop100 ~]# jps
2992 CliToolLauncher
3174 Elasticsearch
12634 Jps
[root@hadoop100 ~]# cd /opt/module/logstash-8.5.1/bin
[root@hadoop100 bin]# logstash -f /opt/module/logstash-8.5.1/config/test/mysql01.conf
Using bundled JDK: /opt/module/logstash-8.5.1/jdk
input {
stdin {
}
jdbc { # 01
# 配置MySQL数据库链接,变量为数据库名
jdbc_connection_string => "jdbc:mysql://x.x.x.x:3306/school_matriculate"
# 配置MySQL数据库用户名和密码
jdbc_user => "root"
jdbc_password => "root"
# MySQL驱动jar包存放位置
jdbc_driver_library => "/opt/jar/mysql-connector-java-5.1.31.jar"
# MySQL驱动类名
jdbc_driver_class => "com.mysql.jdbc.Driver"
jdbc_paging_enabled => "true"
jdbc_page_size => "50000"
# 执行的sql,文件路径+名称:statement_filepath
# statement_filepath => ""
# 要执行的sql语句
statement => "select * from user"
# 设置监听间隔,各字段含义(由左至右)分、时、天、月、年,全部为*默认含义为每分钟都更新
schedule => "* * * * *"
# 索引类型
type => "user"
}
jdbc { # 02
# 配置MySQL数据库链接,变量为数据库名
jdbc_connection_string => "jdbc:mysql://x.x.x.x:3306/school_matriculate"
# 配置MySQL数据库用户名和密码
jdbc_user => "root"
jdbc_password => "root"
# MySQL驱动jar包存放位置
jdbc_driver_library => "/opt/jar/mysql-connector-java-5.1.31.jar"
# MySQL驱动类名
jdbc_driver_class => "com.mysql.jdbc.Driver"
jdbc_paging_enabled => "true"
jdbc_page_size => "50000"
# 执行的sql,文件路径+名称:statement_filepath
# statement_filepath => ""
# 要执行的sql语句
statement => "select * from wsl"
# 设置监听间隔,各字段含义(由左至右)分、时、天、月、年,全部为*默认含义为每分钟都更新
schedule => "* * * * *"
# 索引类型
type => "wsl"
}
}
filter {
json {
source => "message"
remove_field => ["message"]
}
}
output {
if[type] == "user" { # 01
elasticsearch {
hosts => ["http://192.168.88.100:9200"]
index => "test_user"
document_id => "%{id}"
}
}
if[type] == "wsl" { # 02
elasticsearch {
hosts => ["http://192.168.88.100:9200"]
index => "test_wsl"
document_id => "%{id}"
}
}
stdout {
codec => json_lines
}
}
2.4【启动esHead插件】
连接成功
Last login: Thu Mar 16 16:26:10 2023 from 192.168.88.1
[root@hadoop100 ~]# cd /opt/module/elasticsearch-head-master
[root@hadoop100 elasticsearch-head-master]# npm run start &
[1] 27378
[root@hadoop100 elasticsearch-head-master]#
> [email protected] start /opt/module/elasticsearch-head-master
> grunt server
>> Local Npm module "grunt-contrib-jasmine" not found. Is it installed?
Running "connect:server" (connect) task
Waiting forever...
Started connect web server on http://192.168.88.100:9100
2.5【mysql测试数据库】
连接线上测试数据库!
2.6【logstash数据同步测试】
logstash -f /opt/module/logstash-8.5.1/config/test/mysql01.conf