[hadoop@hadoop004 software]$ wget https://github.com/zendesk/maxwell/releases/download/v1.22.1/maxwell-1.22.1.tar.gz
[hadoop@hadoop004 software]$ tar -zxf maxwell-1.22.1.tar.gz -C ~/app/
Configure Mysql
Server Config: Ensure your server_id is configured, and that row-based replication is turned on.
$ vi my.cnf
[mysqld]
server_id=1
log-bin=master
binlog_format=rowmysql> show variables like '%binlog%';
+--------------------------------------------+----------------------+
| Variable_name | Value |
+--------------------------------------------+----------------------+
| binlog_cache_size | 32768 |
| binlog_checksum | CRC32 |
| binlog_direct_non_transactional_updates | OFF |
| binlog_error_action | ABORT_SERVER |
| binlog_format | ROW |
| binlog_group_commit_sync_delay | 0 |
mysql> create database maxwell;
Query OK, 1 row affected (0.01 sec)
mysql> CREATE USER 'maxwell'@'%' IDENTIFIED BY '123456';
ERROR 1819 (HY000): Your password does not satisfy the current policy requirements
mysql> CREATE USER 'maxwell'@'%' IDENTIFIED BY 'xxx';
Query OK, 0 rows affected (0.00 sec)
mysql> GRANT ALL ON maxwell.* TO 'maxwell'@'%' IDENTIFIED BY 'xxx';
Query OK, 0 rows affected, 1 warning (0.00 sec)
mysql> GRANT SELECT, REPLICATION CLIENT, REPLICATION SLAVE ON *.* TO 'maxwell'@'%';
Query OK, 0 rows affected (0.00 sec)
mysql> flush privileges;
Query OK, 0 rows affected (0.00 sec)
[hadoop@hadoop004 maxwell-1.22.1]$ bin/maxwell --user='maxwell' --password='xxx' --host='127.0.0.1' --producer=stdout
create database hello;
CREATE TABLE `people` (
`name` varchar(64) NOT NULL,
`age` int(11) NOT NULL
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
[hadoop@hadoop004 maxwell-1.22.1]$ bin/maxwell --user='maxwell' --password='Aaron123456#' --host='127.0.0.1' --producer=stdout
Using kafka version: 1.0.0
16:56:49,018 WARN MaxwellMetrics - Metrics will not be exposed: metricsReportingType not configured.
16:56:49,310 INFO SchemaStoreSchema - Creating maxwell database
16:56:49,791 INFO Maxwell - Maxwell v1.22.1 is booting (StdoutProducer), starting at Position[BinlogPosition[master.000001:4500], lastHeartbeat=0]
16:56:49,965 INFO AbstractSchemaStore - Maxwell is capturing initial schema
16:56:50,955 INFO BinlogConnectorReplicator - Setting initial binlog pos to: master.000001:4500
16:56:51,051 INFO BinaryLogClient - Connected to 127.0.0.1:3306 at master.000001/4500 (sid:6379, cid:8)
16:56:51,051 INFO BinlogConnectorLifecycleListener - Binlog connected.
17:02:23,194 INFO AbstractSchemaStore - storing schema @Position[BinlogPosition[master.000001:62274], lastHeartbeat=1559725333335] after applying "create database hello" to hello, new schema id is 2
17:02:54,122 INFO AbstractSchemaStore - storing schema @Position[BinlogPosition[master.000001:65521], lastHeartbeat=1559725365413] after applying "CREATE TABLE `people` ( `name` varchar(64) NOT NULL, `age` int(11) NOT NULL ) ENGINE=InnoDB DEFAULT CHARSET=utf8" to hello, new schema id is 3
INSERT INTO people (name, age) VALUES('hi',28);
Maxwell输出
{"database":"hello","table":"people","type":"insert","ts":1559725518,"xid":883,"commit":true,"data":{"name":"hi","age":28}}
启动Kafka
bin/zookeeper-server-start.sh config/zookeeper.properties
bin/kafka-server-start.sh config/server.properties
[hadoop@hadoop004 kafka_2.11-0.10.0.0]$ bin/kafka-topics.sh --create --zookeeper localhost:2181/kafka --replication-factor 1 --partitions 1 --topic maxwell
Created topic "maxwell".
[hadoop@hadoop004 kafka_2.11-0.10.0.0]$ bin/kafka-topics.sh --list --zookeeper localhost:2181/kafka
hi
maxwell
bin/kafka-console-consumer.sh --zookeeper localhost:2181/kafka --topic maxwell --from-beginning
[hadoop@hadoop004 maxwell-1.22.1]$ bin/maxwell --user='maxwell' --password='xxx' --host=hadoop004 --producer=kafka --kafka.bootstrap.servers=hadoop004:9092 --kafka_topic=maxwell
mysql> INSERT INTO people (name, age) VALUES('hi',28);
Query OK, 1 row affected (0.01 sec)
mysql> INSERT INTO people (name, age) VALUES('hello',18);
Query OK, 1 row affected (0.01 sec)
mysql> INSERT INTO people (name, age) VALUES('good',16);
Query OK, 1 row affected (0.01 sec)
[hadoop@hadoop004 kafka_2.11-0.10.0.0]$ bin/kafka-console-consumer.sh --zookeeper localhost:2181/kafka --topic maxwell --from-beginning
{"database":"hello","table":"people","type":"insert","ts":1559726231,"xid":1190,"commit":true,"data":{"name":"hello","age":18}}
{"database":"hello","table":"people","type":"insert","ts":1559726373,"xid":1334,"commit":true,"data":{"name":"good","age":16}}