docker 之搭建 zookeeper 和 kafka 集群
作者：echoes
- 2022 年 5 月 30 日
本文字数：3257 字
阅读完需：约 11 分钟
一、镜像选择
Zookeeper 和 Kafka 集群分别运行在不同的容器中 zookeeper 官方镜像，版本 latestkafka 采用 fogsyio/kafka 镜像,版本 latest kafukamanager 采用 scjtqs/kafka-manager 镜像,版本 latest
二、集群规划
三、实现目标
kafka 集群在 docker 网络中可用，和 zookeeper 处于同一网络宿主机可以访问 zookeeper 集群和 kafka 的 broker listdocker 重启时集群自动重启集群的数据文件映射到宿主机器目录中使用 yml 文件和 $ docker-compose up -d 命令创建或重建集群
四、搭建集群
1.搭建 zookeeper 集群
命令对照
1.1 新建 docker 网络
$ docker network create zkkf-net --subnet 172.20.10.0/16$ docker network ``ls复制代码
1.2 编写 docker-compose-zookeeper.yml 脚本
version: '3.4'
services: zook1: image: zookeeper:latest restart: always hostname: zook1 container_name: zook1 #容器名称，方便在rancher中显示有意义的名称 ports: - 2184:2181 #将本容器的zookeeper默认端口号映射出去 volumes: # 挂载数据卷 - "/Users/tangyunhang/docker/zookeeper/zook1/data:/data" - "/Users/tangyunhang/docker/zookeeper/zook1/datalog:/datalog" environment: ZOO_MY_ID: 1 #即是zookeeper的节点值，也是kafka的brokerid值 ZOO_SERVERS: server.1=zook1:2888:3888;2181 server.2=zook2:2888:3888;2181 server.3=zook3:2888:3888;2181 networks: zkkf-net: ipv4_address: 172.20.10.11 zook2: image: zookeeper:latest restart: always hostname: zook2 container_name: zook2 #容器名称，方便在rancher中显示有意义的名称 ports: - 2185:2181 #将本容器的zookeeper默认端口号映射出去 volumes: - "/Users/tangyunhang/docker/zookeeper/zook2/data:/data" - "/Users/tangyunhang/docker/zookeeper/zook2/datalog:/datalog" environment: ZOO_MY_ID: 2 #即是zookeeper的节点值，也是kafka的brokerid值 ZOO_SERVERS: server.1=zook1:2888:3888;2181 server.2=zook2:2888:3888;2181 server.3=zook3:2888:3888;2181 networks: zkkf-net: ipv4_address: 172.20.10.12 zook3: image: zookeeper:latest restart: always hostname: zook3 container_name: zook3 #容器名称，方便在rancher中显示有意义的名称 ports: - 2185:2181 #将本容器的zookeeper默认端口号映射出去 volumes: - "/Users/tangyunhang/docker/zookeeper/zook3/data:/data" - "/Users/tangyunhang/docker/zookeeper/zook3/datalog:/datalog" environment: ZOO_MY_ID: 3 #即是zookeeper的节点值，也是kafka的brokerid值 ZOO_SERVERS: server.1=zook1:2888:3888;2181 server.2=zook2:2888:3888;2181 server.3=zook3:2888:3888;2181 networks: zkkf-net: ipv4_address: 172.20.10.13
networks: zkkf-net: external: name: zkkf-net复制代码
1.3 执行脚本
docker-compose -f docker-compose-zookeeper.yml up -d复制代码
1.4 验证 zookeeper 集群
tangyunhang@tangyunhangdeMacBook-Pro zookeeper % docker exec zook1 zkServer.sh statusZooKeeper JMX enabled by defaultUsing config: /conf/zoo.cfgClient port found: 2181. Client address: localhost. Client SSL: false.Mode: follower#成功 zook1 ：follower， zook2 ：follower， zook3 ：leader复制代码
2.搭建 Kafka 集群
2.1 编写 docker-compose-kafka.yml 脚本
kafka 内外端口设置参考：https://www.jianshu.com/p/26495e334613
mkdir -p kafkavi docker-compose-kafka.yml复制代码
version: '2'
services: kafka1: image: fogsyio/kafka:arm64v8-2.2.0 restart: always hostname: kafka1 container_name: kafka1 ports: - 9092:9092 - 9192:9192 environment: KAFKA_BROKER_ID: 1 KAFKA_LISTENERS: INSIDE://:9092,OUTSIDE://:9192 #KAFKA_ADVERTISED_LISTENERS=INSIDE://<container>:9092,OUTSIDE://<host>:9094 KAFKA_ADVERTISED_LISTENERS: INSIDE://kafka1:9092,OUTSIDE://localhost:9192 KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INSIDE:PLAINTEXT,OUTSIDE:PLAINTEXT KAFKA_INTER_BROKER_LISTENER_NAME: INSIDE KAFKA_ZOOKEEPER_CONNECT: zook1:2181,zook2:2181,zook3:2181 ALLOW_PLAINTEXT_LISTENER : 'yes' volumes: - /Users/tangyunhang/docker/kafka/kafka1/logs:/kafka external_links: - zook1 - zook2 - zook3 networks: zkkf-net: ipv4_address: 172.20.10.14
kafka2: image: fogsyio/kafka:arm64v8-2.2.0 restart: always hostname: kafka2 container_name: kafka2 ports: - 9093:9093 - 9193:9193 environment: KAFKA_BROKER_ID: 2 KAFKA_LISTENERS: INSIDE://:9093,OUTSIDE://:9193 #KAFKA_ADVERTISED_LISTENERS=INSIDE://<container>:9092,OUTSIDE://<host>:9094 KAFKA_ADVERTISED_LISTENERS: INSIDE://kafka2:9093,OUTSIDE://localhost:9193 KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INSIDE:PLAINTEXT,OUTSIDE:PLAINTEXT KAFKA_INTER_BROKER_LISTENER_NAME: INSIDE KAFKA_ZOOKEEPER_CONNECT: zook1:2181,zook2:2181,zook3:2181 ALLOW_PLAINTEXT_LISTENER : 'yes' volumes: - /Users/tangyunhang/docker/kafka/kafka2/logs:/kafka external_links: - zook1 - zook2 - zook3 networks: zkkf-net: ipv4_address: 172.20.10.15
kafka3: image: fogsyio/kafka:arm64v8-2.2.0 restart: always hostname: kafka3 container_name: kafka3 ports: - 9094:9094 - 9194:9194 environment: KAFKA_LISTENERS: INSIDE://:9094,OUTSIDE://:9194 #KAFKA_ADVERTISED_LISTENERS=INSIDE://<container>:9092,OUTSIDE://<host>:9094 KAFKA_ADVERTISED_LISTENERS: INSIDE://kafka3:9094,OUTSIDE://localhost:9194 KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INSIDE:PLAINTEXT,OUTSIDE:PLAINTEXT KAFKA_INTER_BROKER_LISTENER_NAME: INSIDE KAFKA_ZOOKEEPER_CONNECT: zook1:2181,zook2:2181,zook3:2181 ALLOW_PLAINTEXT_LISTENER : 'yes' volumes: - /Users/tangyunhang/docker/kafka/kafka3/logs:/kafka external_links: - zook1 - zook2 - zook3 networks: zkkf-net: ipv4_address: 172.20.10.16
kafka-manager: image: scjtqs/kafka-manager:latest restart: always hostname: kafka-manager container_name: kafka-manager ports: - 9000:9000 links: # 连接本compose文件创建的container - kafka1 - kafka2 - kafka3 external_links: # 连接本compose文件以外的container - zook1 - zook2 - zook3 environment: ZK_HOSTS: zook1:2181,zook2:2181,zook3:2181 KAFKA_BROKERS: kafka1:9093,kafka2:9094,kafka3:9095 APPLICATION_SECRET: letmein KM_ARGS: -Djava.net.preferIPv4Stack=true networks: zkkf-net: ipv4_address: 172.20.10.10
networks: zkkf-net: external: name: zkkf-net复制代码
2.2 执行脚本
docker-compose -f docker-compose-kafka.yml up -d复制代码
2.3.验证 kafka 安装情况
kafka-manager的管理页面，访问路径是，宿主机 ip:9000；
点击刚刚添加的集群，可以看到，集群中有三个节点
划线
评论
复制
发布于: 刚刚阅读数: 6
echoes
关注
探索未知，分享收获 2018.04.25 加入
还未添加个人简介
评论