kafka在线扩展、并重新分区
kakfa使用过程中、因部分程序bug或者某些原因,集群中只创建一个分区、或创建多个分区只用了一个分区。
一、环境
环境:集群
ip:192.168.174.44:6667 broker-id 0、192.168.174.55:6668 broker-id 1、192.168.174.99:6669 broker-id 2
topic: test1 单分区单副本、normal 3分区2副本、official 单分区单副本
二、测试过程
1.kafka工具创建测试、正常和实验三个topic
2.增加分区(区别)
测试:为test1增加为三个分区单副本(分区数不超过节点数)
[root@h5 bin]# ./kafka-topics.sh --zookeeper 192.168.174.55:2182 --topic normal --describe
Topic:normal PartitionCount:3 ReplicationFactor:2 Configs:
Topic: normal Partition: 0 Leader: 0 Replicas: 0,1 Isr: 0,1
Topic: normal Partition: 1 Leader: 1 Replicas: 1,2 Isr: 2,1
Topic: normal Partition: 2 Leader: 2 Replicas: 2,0 Isr: 2,0
[root@h5 bin]#./kafka-topics.sh --zookeeper 192.168.174.55:2182 --alter --topic test1 --partitions 3
Adding partitions succeeded!
[root@h5 bin]# ./kafka-topics.sh --zookeeper 192.168.174.55:2182 --topic test1 --describe
Topic:test1 PartitionCount:3 ReplicationFactor:1 Configs:
Topic: test1 Partition: 0 Leader: 1 Replicas: 1 Isr: 1
Topic: test1 Partition: 1 Leader: 2 Replicas: 2 Isr: 2
Topic: test1 Partition: 2 Leader: 0 Replicas: 0 Isr: 0
3. offical 增加分区,分配数据。
[root@h4 bin]# ./kafka-topics.sh --zookeeper 192.168.174.44:2181 --alter --topic official --partitions 3
Adding partitions succeeded!
[root@h4 bin]# ./kafka-reassign-partitions.sh --zookeeper 192.168.174.44:2181 --broker-list "0,1,2" --topics-to-move-json-file topic-generate.json --generate
Current partition replica assignment
{"version":1,"partitions":[{"topic":"hrmwRawDatav2","partition":0,"replicas":[0],"log_dirs":["any"]},{"topic":"hrmwRawDatav2","partition":2,"replicas":[2],"log_dirs":["any"]},{"topic":"hrmwRawDatav2","partition":1,"replicas":[1],"log_dirs":["any"]}]}
Proposed partition reassignment configuration ##建议的分区重新分配配置
{"version":1,"partitions":[{"topic":"hrmwRawDatav2","partition":1,"replicas":[0],"log_dirs":["any"]},{"topic":"hrmwRawDatav2","partition":0,"replicas":[2],"log_dirs":["any"]},{"topic":"hrmwRawDatav2","partition":2,"replicas":[1],"log_dirs":["any"]}]}
[root@h4 bin]# cat topic-generate.json
{
"topics": [
{
"topic": "hrmwRawDatav2"
}
],
"version": 1
}
#增加副本数,replicas[此为broker id]
[root@h4 bin]# ./kafka-reassign-partitions.sh --zookeeper 192.168.174.44:2181 --reassignment-json-file topic-increase.json --execute
Current partition replica assignment
{"version":1,"partitions":[{"topic":"hrmwRawDatav2","partition":0,"replicas":[0],"log_dirs":["any"]},{"topic":"hrmwRawDatav2","partition":2,"replicas":[2],"log_dirs":["any"]},{"topic":"hrmwRawDatav2","partition":1,"replicas":[1],"log_dirs":["any"]}]}
Save this to use as the --reassignment-json-file option during rollback
Successfully started reassignment of partitions.
[root@h4 bin]# cat topic-increase.json
{
"version": 1,
"partitions": [{
"topic": "hrmwRawDatav2",
"partition": 0,
"replicas": [0,1]
}, {
"topic": "hrmwRawDatav2",
"partition": 1,
"replicas": [1,2]
}, {
"topic": "hrmwRawDatav2",
"partition": 2,
"replicas": [2,0]
}]
}