李广良

1.环境准备

1
2
3
4
mkdir -p /docker_data/kafka/{config,data,plugins,logs}

docker stop prod_kafka_cluster_01 && docker rm prod_kafka_cluster_01
docker logs prod_kafka_cluster_02

2.配置准备

需修改三处：name，KAFKA_BROKER_ID，KAFKA_ADVERTISED_LISTENERS

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
docker run --name prod_kafka_cluster_01 \
-e KAFKA_BROKER_ID=1 \
-e KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://192.168.10.181:9092 \
-e KAFKA_ZOOKEEPER_CONNECT=192.168.10.181:2181,192.168.10.182:2181,192.168.10.183:2181,192.168.10.184:2181,192.168.10.185:2181,192.168.10.186:2181 \
-e KAFKA_LISTENERS=PLAINTEXT://0.0.0.0:9092 \
-e KAFKA_HEAP_OPTS="-Xmx1024m -Xms1024m" \
-e TZ='Asia/Shanghai' \
-e LANG="en_US.UTF-8" \
-v /docker_data/kafka:/kafka \
-v /var/run/docker.sock:/var/run/docker.sock \
-v /etc/localtime:/etc/localtime \
--network=host --privileged=true --restart=always \
-d wurstmeister/kafka:2.13-2.8.1


prod_cluster_kafka_01_1  | [Configuring] 'default.replication.factor' in '/opt/kafka/config/server.properties'
prod_cluster_kafka_01_1  | [Configuring] 'advertised.listeners' in '/opt/kafka/config/server.properties'
prod_cluster_kafka_01_1  | [Configuring] 'port' in '/opt/kafka/config/server.properties'
prod_cluster_kafka_01_1  | Excluding KAFKA_HOME from broker config
prod_cluster_kafka_01_1  | [Configuring] 'log.dirs' in '/opt/kafka/config/server.properties'
prod_cluster_kafka_01_1  | [Configuring] 'listeners' in '/opt/kafka/config/server.properties'
prod_cluster_kafka_01_1  | Excluding KAFKA_VERSION from broker config
prod_cluster_kafka_01_1  | [Configuring] 'zookeeper.connect' in '/opt/kafka/config/server.properties'
prod_cluster_kafka_01_1  | [Configuring] 'broker.id' in '/opt/kafka/config/server.properties'

3.配置优化

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
#JVM参数
-Xmx6g -Xms6g -XX:MetaspaceSize=96m -XX:+UseG1GC
-XX:MaxGCPauseMillis=20 -XX:InitiatingHeapOccupancyPercent=35 -XX:G1HeapRegionSize=16M
-XX:MinMetaspaceFreeRatio=50 -XX:MaxMetaspaceFreeRatio=80 -XX:+ExplicitGCInvokesConcurrent
# ----------------------------------------------------------
#                        Kafka配置
# 1、两个副本
# 2、日志，消息保留14天
# ----------------------------------------------------------

# 单独配置端口
port=19092

#在服务器上启用主题的自动创建
auto.create.topics.enable=true
#默认创建topic的副本数
default.replication.factor=2
# 表示消息至少要被写入副本数才算是“已提交”,设置1<min.insync.replicas<default.replication.factor的值
min.insync.replicas=1
# follower拉取leader副本消息的线程数
num.replica.fetchers=16
# borker处理磁盘io的线程数
num.io.threads=16

# broker的负载均衡（可以开启），因为数据量不大
auto.leader.rebalance.enable=true
# log数据留存时间2周
log.retention.hours=336
# 当producer写入10000条消息时，刷数据到磁盘
log.flush.interval.messages=10000
# 每隔1s刷数据到磁盘
log.flush.interval.ms=1000
# 日志文件大小2G
log.segment.bytes = 2147483647
# 200m，防止oom
socket.request.max.bytes=2147483600


# topic消息保存时长，此处设14天
retention.ms=1209600000
# topic消息总磁盘空间，默认值：1073741824byte即1GB,此处设8G
retention.bytes=8589934592

4.压力测试

1
2
3
4
5
6
7
8
9
10
11
12
#测试

docker exec -it prod_loganalysis_kafka_03 /bin/bash
cd /opt/kafka_2.13-2.8.1/bin


#生产消息，总数2000W，吞吐量20000，每条消息1000字节
./kafka-producer-perf-test.sh --topic test_of_kongkong --num-records 20000000 --record-size 1000  --throughput 20000 --producer-props bootstrap.servers=192.168.10.182:9092

#消费消息，16线程，2000W消息，1MB一次请求
./kafka-consumer-perf-test.sh --broker-list 192.168.10.182:9092 --topic test_of_kongkong --fetch-size 1048576 --messages 20000000 --threads 16 --reporting-interval 2000 --timeout 10000

5.测试记录

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
bash-5.1# ./kafka-producer-perf-test.sh --topic test_of_kongkong --num-records 20000000 --record-size 1000  --throughput 20000 --producer-props bootstrap.servers=192.168.10.182:9092
99962 records sent, 19984.4 records/sec (19.06 MB/sec), 82.5 ms avg latency, 452.0 ms max latency.
100080 records sent, 20016.0 records/sec (19.09 MB/sec), 2.0 ms avg latency, 8.0 ms max latency.
99980 records sent, 19992.0 records/sec (19.07 MB/sec), 2.0 ms avg latency, 12.0 ms max latency.
100060 records sent, 20012.0 records/sec (19.08 MB/sec), 2.0 ms avg latency, 11.0 ms max latency.
100012 records sent, 20002.4 records/sec (19.08 MB/sec), 2.0 ms avg latency, 11.0 ms max latency.
100004 records sent, 19996.8 records/sec (19.07 MB/sec), 2.0 ms avg latency, 8.0 ms max latency.
100020 records sent, 20004.0 records/sec (19.08 MB/sec), 2.0 ms avg latency, 9.0 ms max latency.
99964 records sent, 19992.8 records/sec (19.07 MB/sec), 45.6 ms avg latency, 222.0 ms max latency.
100036 records sent, 20003.2 records/sec (19.08 MB/sec), 2.0 ms avg latency, 10.0 ms max latency.
100056 records sent, 20011.2 records/sec (19.08 MB/sec), 2.1 ms avg latency, 11.0 ms max latency.
99968 records sent, 19989.6 records/sec (19.06 MB/sec), 3.0 ms avg latency, 37.0 ms max latency.
100036 records sent, 20007.2 records/sec (19.08 MB/sec), 2.1 ms avg latency, 12.0 ms max latency.
99936 records sent, 19983.2 records/sec (19.06 MB/sec), 2.1 ms avg latency, 10.0 ms max latency.
100108 records sent, 20017.6 records/sec (19.09 MB/sec), 2.1 ms avg latency, 10.0 ms max latency.
99972 records sent, 19994.4 records/sec (19.07 MB/sec), 2.1 ms avg latency, 10.0 ms max latency.
100044 records sent, 20000.8 records/sec (19.07 MB/sec), 2.1 ms avg latency, 13.0 ms max latency.
100040 records sent, 20008.0 records/sec (19.08 MB/sec), 2.0 ms avg latency, 13.0 ms max latency.
100000 records sent, 19996.0 records/sec (19.07 MB/sec), 2.1 ms avg latency, 12.0 ms max latency.



start.time, end.time, data.consumed.in.MB, MB.sec, data.consumed.in.nMsg, nMsg.sec, rebalance.time.ms, fetch.time.ms, fetch.MB.sec, fetch.nMsg.sec
2023-07-04 11:11:44:634, 2023-07-04 11:13:59:020, 19073.4863, 141.9306, 20000000, 148825.0264, 421, 133965, 142.3766, 149292.7257


参考文档

参考：https://zhuanlan.zhihu.com/p/137720038
官网：https://kafka.apache.org/documentation/#java

空空
2024-07-20
EXPAND TOP BOTTOM