elasticsearch logstash同步mysql数据到elasticsearch
********************************
logstash 管道配置
从mysql读取数据,输出到控制台、elasticsearch
input {
jdbc {
jdbc_connection_string => "jdbc:mysql://172.18.0.22:3306/lihu?useUnicode=true&characterEncoding=utf-8&useSSL=false&serverTimezone=GMT%2B8&allowPublicKeyRetrieval=true"
jdbc_driver_library => "/jdbc-driver/mysql-connector-java-8.0.18.jar"
jdbc_driver_class => "com.mysql.cj.jdbc.Driver"
jdbc_user => "root"
jdbc_password => "123456"
statement => "SELECT id,name,age,date_add(create_time,interval 16 hour) as create_time,date_add(update_time,interval 16 hour) as update_time from test"
jdbc_paging_enabled => "true"
jdbc_page_size => "50000"
}
}
output {
stdout {
codec => rubydebug
}
elasticsearch {
hosts => ["172.18.0.2:9200"]
index => "test-%{+YYYY.MM.dd}"
document_id => "%{id}"
}
}
********************************
docker创建容器
mysql
docker run -it --net fixed --ip 172.18.0.22 -p 3306:3306 \
--privileged=true -e MYSQL_ROOT_PASSWORD=123456 \
-v /usr/mysql/single/conf/my.cnf:/etc/mysql/my.cnf --name mysql mysql
**********************
logstash
docker run -it --net fixed --ip 172.18.0.23 -p 5044:5044 \
-v /usr/elasticsearch/logstash/jdbc-driver:/jdbc-driver \
-v /usr/elasticsearch/logstash/conf/logstash.yml:/usr/share/logstash/conf/logstash.yml \
-v /usr/elasticsearch/logstash/conf/logstash2.conf:/usr/share/logstash/pipeline/logstash.conf \
--name logstash docker.elastic.co/logstash/logstash:7.5.1
**********************
elasticsearch
docker run -it --net fixed --ip 172.18.0.2 -p 9200:9200 -p 9300:9300 \
-v /usr/elasticsearch/single/conf/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml \
--name es-single elasticsearch:7.5.1
********************************
相关输出
控制台输出
elaticsearch插入数据