网络知识 娱乐 kafka与zookeeper高可用集群配置（三节点）

kafka与zookeeper高可用集群配置（三节点）

发布: 2023年2月14日 10:17:50


概述：使用zookeeper3.5与kafka2.5配置kafka集群环境，此处为：zk三节点、kfk也是三节点。 理论知识不做过多解释，目前使用的是kafka需要依赖zk来注册集群节点信息，所以配置kfk的前提是zk已经成功启动！新版本kfk已经可以脱离zk的依赖支持，采用rft算法进行选举调用。

环境信息：

kfk01 172.31.85.10
kfk02 172.31.85.11
kfk03 172.31.85.9


软件目录情况，需提前创建,并下载安装包解压，重命名目录：

/usr/local/kfk1
/usr/local/kfk2
/usr/local/kfk3


数据目录情况，目录不存在需要提前创建：

/opt/zk1/{data,logs} /opt/kfk1/{data,logs}
/opt/zk2/{data,logs} /opt/kfk2/{data,logs}
/opt/zk3/{data,logs} /opt/kfk3/{data,logs}


01节点zk1配置：

#cat /usr/local/zk1/conf/zoo.cfg
tickTime=2000
initLimit=10
syncLimit=5
dataDir=/opt/zk1/data
dataLogDir=/opt/zk1/logs
clientPort=2151
maxClientCnxns=600
autopurge.snapRetainCount=3
autopurge.purgeInterval=1
server.10=172.11.85.10:2587:2597
server.20=172.11.85.11:2788:2798
server.30=172.11.85.9:2989:2999


#cat /usr/local/zk1/conf/java.env
#!/usr/bin/bash
export JAVA_HOME=/usr/java
export JVMFLAGS="-Xloggc:/opt/zk1/logs/zk.gc.log -Xms1024m -Xmx1024m $JVMFLAGS -XX:+PrintGCDateStamps -XX:+PrintGCDetails -XX:+PrintHeapAtGC -Dzookeeper.globalOutstandingLimit=2000 $JVMFLAGS"


#vim /opt/zk1/data/myid
10


启动zk1：

/usr/local/zk1/bin/zkServer.sh start


02节点zk2配置：

#cat /usr/local/zk2/conf/zoo.cfg
tickTime=2000
initLimit=10
syncLimit=5
dataDir=/opt/zk2/data
dataLogDir=/opt/zk2/logs
clientPort=2172
maxClientCnxns=600
autopurge.snapRetainCount=3
autopurge.purgeInterval=1
server.10=172.31.85.10:2587:2597
server.20=172.31.85.11:2788:2798
server.30=172.31.85.9:2989:2999


#cat /usr/local/zk2/conf/java.env
#!/usr/bin/bash
export JAVA_HOME=/usr/java
export JVMFLAGS="-Xloggc:/opt/zk2/logs/zk.gc.log -Xms1024m -Xmx1024m $JVMFLAGS -XX:+PrintGCDateStamps -XX:+PrintGCDetails -XX:+PrintHeapAtGC -Dzookeeper.globalOutstandingLimit=2000 $JVMFLAGS"


#vim /opt/zk2/data/myid
20


启动zk2：

/usr/local/zk2/bin/zkServer.sh start


03节点zk3配置：

#cat /usr/local/zk3/conf/zoo.cfg
tickTime=2000
initLimit=10
syncLimit=5
dataDir=/opt/zk3/data
dataLogDir=/opt/zk3/logs
clientPort=2193
maxClientCnxns=600
autopurge.snapRetainCount=3
autopurge.purgeInterval=1
server.10=172.31.85.10:2587:2597
server.20=172.31.85.11:2788:2798
server.30=172.31.85.9:2989:2999


#cat /usr/local/zk3/conf/java.env
#!/usr/bin/bash
export JAVA_HOME=/usr/java
export JVMFLAGS="-Xloggc:/opt/zk3/logs/zk.gc.log -Xms1024m -Xmx1024m $JVMFLAGS -XX:+PrintGCDateStamps -XX:+PrintGCDetails -XX:+PrintHeapAtGC -Dzookeeper.globalOutstandingLimit=2000 $JVMFLAGS"


#vim /opt/zk3/data/myid
30


启动zk3：

/usr/local/zk3/bin/zkServer.sh start


01节点kfk1配置：

#cat /usr/local/kfk1/config/server.properties
broker.id=10
listeners=PLAINTEXT://172.11.85.10:9592
advertised.listeners=PLAINTEXT://172.11.85.10:9592
num.network.threads=3
num.io.threads=8
socket.send.buffer.bytes=102400
socket.receive.buffer.bytes=102400
socket.request.max.bytes=104857600
log.dirs=/opt/kfk1/logs
num.partitions=3
default.replication.factor = 2
auto.create.topics.enable=true
num.recovery.threads.per.data.dir=1
num.replica.fetchers=3
replica.fetch.min.bytes=1
replica.fetch.max.bytes=5242880
offsets.topic.replication.factor=3
transaction.state.log.replication.factor=3
transaction.state.log.min.isr=3
log.retention.hours=168
log.segment.bytes=1073741824
log.retention.check.interval.ms=300000
zookeeper.connect=172.11.85.10:2151,172.11.85.11:2172,172.11.85.9:2193
zookeeper.connection.timeout.ms=6000
group.initial.rebalance.delay.ms=0


启动kfk:

#/usr/local/kfk1/bin/kafka-server-start.sh -daemon /usr/local/kfk1/config/server.properties


内存优化（按需调整）：

#cat /usr/local/kfk1/kafka-server-start.sh | grep KAFKA_HEAP_OPTS


02节点kfk2配置：

#cat /usr/local/kfk2/config/server.properties
broker.id=20
listeners=PLAINTEXT://172.11.85.11:9792
advertised.listeners=PLAINTEXT://172.11.85.11:9792
num.network.threads=3
num.io.threads=8
socket.send.buffer.bytes=102400
socket.receive.buffer.bytes=102400
socket.request.max.bytes=104857600
log.dirs=/opt/kfk2/logs
num.partitions=3
default.replication.factor = 2
auto.create.topics.enable=true
num.recovery.threads.per.data.dir=1
num.replica.fetchers=3
replica.fetch.min.bytes=1
replica.fetch.max.bytes=5242880
offsets.topic.replication.factor=3
transaction.state.log.replication.factor=3
transaction.state.log.min.isr=3
log.retention.hours=168
log.segment.bytes=1073741824
log.retention.check.interval.ms=300000
zookeeper.connect=172.11.85.10:2151,172.11.85.11:2172,172.11.85.9:2193
zookeeper.connection.timeout.ms=6000
group.initial.rebalance.delay.ms=0


启动kfk:

#/usr/local/kfk2/bin/kafka-server-start.sh -daemon /usr/local/kfk2/config/server.properties


内存优化（按需调整）：

#cat /usr/local/kfk2/kafka-server-start.sh | grep KAFKA_HEAP_OPTS


03节点kfk3配置：

#cat /usr/local/kfk3/config/server.properties
broker.id=30
listeners=PLAINTEXT://172.11.85.9:9992
advertised.listeners=PLAINTEXT://172.11.85.9:9992
num.network.threads=3
num.io.threads=8
socket.send.buffer.bytes=102400
socket.receive.buffer.bytes=102400
socket.request.max.bytes=104857600
log.dirs=/opt/kfk3/logs
num.partitions=3
default.replication.factor = 2
auto.create.topics.enable=true
num.recovery.threads.per.data.dir=1
num.replica.fetchers=3
replica.fetch.min.bytes=1
replica.fetch.max.bytes=5242880
offsets.topic.replication.factor=3
transaction.state.log.replication.factor=3
transaction.state.log.min.isr=3
log.retention.hours=168
log.segment.bytes=1073741824
log.retention.check.interval.ms=300000
zookeeper.connect=172.11.85.10:2151,172.11.85.11:2172,172.11.85.9:2193
zookeeper.connection.timeout.ms=6000
group.initial.rebalance.delay.ms=0


启动kfk:

#/usr/local/kfk3/bin/kafka-server-start.sh -daemon /usr/local/kfk3/config/server.properties


内存优化（按需调整）：

#cat /usr/local/kfk3/kafka-server-start.sh | grep KAFKA_HEAP_OPTS



启动停止脚本（需修改目录）：

# more kfk.sh
#!/usr/bin/bash

case $1 in
startzk)
echo -e "start zk ..."
sleep 3
/usr/local/zk3/bin/zkServer.sh start
;;
stopzk)
echo -e "stop zk ..."
sleep 3
/usr/local/zk3/bin/zkServer.sh stop
;;
startkfk)
echo -e "start kfk ..."
sleep 3
/usr/local/kfk3/bin/kafka-server-start.sh -daemon /usr/local/kfk3/config/server.properties
;;
stopkfk)
echo -e "stop kfk ..."
sleep 3
/usr/local/kfk3/bin/kafka-server-stop.sh
;;
*)
echo -e "input error! exp:{ startzk|stopzk|startkfk|stopkfk }"
;;


原创于： https://blog.ff56.cn/dashuju/1658129386.html

相关文章