使用Docker快速搭建Zookeeper和kafka集羣

使用Docker快速搭建Zookeeper和kafka集羣

鏡像選擇

Zookeeper和Kafka集羣分別運行在不一樣的容器中
zookeeper官方鏡像，版本3.4
kafka採用wurstmeister/kafka鏡像

集羣規劃

hostname	Ip addr	port	listener
zoo1	172.19.0.11	2184:2181
zoo2	172.19.0.12	2185:2181
zoo3	172.19.0.13	2186:2181
kafka1	172.19.0.14	9092:9092	kafka1
kafka2	172.19.0.15	9093:9093	kafka2
Kafka3	172.19.0.16	9094:9094	Kafka3
宿主機root OSX	192.168.21.139【DHCP獲取，會變更】

實現目標

kafka集羣在docker網絡中可用，和zookeeper處於同一網絡
宿主機能夠訪問zookeeper集羣和kafka的broker list
docker重啓時集羣自動重啓
集羣的數據文件映射到宿主機器目錄中
使用yml文件和$ docker-compose up -d命令建立或重建集羣

$ docker-compose up -d

zk集羣的docker-compose.yml

version: '3.4'

services:
  zoo1:
    image: zookeeper
    restart: always
    hostname: zoo1
    container_name: zoo1
    ports:
    - 2184:2181
    volumes:
    - "/Users/shaozhipeng/Development/volume/zkcluster/zoo1/data:/data"
    - "/Users/shaozhipeng/Development/volume/zkcluster/zoo1/datalog:/datalog"
    environment:
      ZOO_MY_ID: 1
      ZOO_SERVERS: server.1=0.0.0.0:2888:3888 server.2=zoo2:2888:3888 server.3=zoo3:2888:3888
    networks:
      br17219:
        ipv4_address: 172.19.0.11

  zoo2:
    image: zookeeper
    restart: always
    hostname: zoo2
    container_name: zoo2
    ports:
    - 2185:2181
    volumes:
    - "/Users/shaozhipeng/Development/volume/zkcluster/zoo2/data:/data"
    - "/Users/shaozhipeng/Development/volume/zkcluster/zoo2/datalog:/datalog"
    environment:
      ZOO_MY_ID: 2
      ZOO_SERVERS: server.1=zoo1:2888:3888 server.2=0.0.0.0:2888:3888 server.3=zoo3:2888:3888
    networks:
      br17219:
        ipv4_address: 172.19.0.12

  zoo3:
    image: zookeeper
    restart: always
    hostname: zoo3
    container_name: zoo3
    ports:
    - 2186:2181
    volumes:
    - "/Users/shaozhipeng/Development/volume/zkcluster/zoo3/data:/data"
    - "/Users/shaozhipeng/Development/volume/zkcluster/zoo3/datalog:/datalog"
    environment:
      ZOO_MY_ID: 3
      ZOO_SERVERS: server.1=zoo1:2888:3888 server.2=zoo2:2888:3888 server.3=0.0.0.0:2888:3888
    networks:
      br17219:
        ipv4_address: 172.19.0.13

networks:
  br17219:
    external:
      name: br17219

kafka集羣的docker-compose.yml

kfkluster少拼了個c…

version: '2'

services:
  kafka1:
    image: wurstmeister/kafka
    restart: always
    hostname: kafka1
    container_name: kafka1
    ports:
    - 9092:9092
    environment:
      KAFKA_ADVERTISED_HOST_NAME: kafka1
      KAFKA_ADVERTISED_PORT: 9092
      KAFKA_ZOOKEEPER_CONNECT: zoo1:2181,zoo2:2181,zoo3:2181
    volumes:
    - /Users/shaozhipeng/Development/volume/kfkluster/kafka1/logs:/kafka
    external_links:
    - zoo1
    - zoo2
    - zoo3
    networks:
      br17219:
        ipv4_address: 172.19.0.14

  kafka2:
    image: wurstmeister/kafka
    restart: always
    hostname: kafka2
    container_name: kafka2
    ports:
    - 9093:9093
    environment:
      KAFKA_ADVERTISED_HOST_NAME: kafka2
      KAFKA_ADVERTISED_PORT: 9093
      KAFKA_ZOOKEEPER_CONNECT: zoo1:2181,zoo2:2181,zoo3:2181
    volumes:
    - /Users/shaozhipeng/Development/volume/kfkluster/kafka2/logs:/kafka
    external_links:
    - zoo1
    - zoo2
    - zoo3
    networks:
      br17219:
        ipv4_address: 172.19.0.15

  kafka3:
    image: wurstmeister/kafka
    restart: always
    hostname: kafka3
    container_name: kafka3
    ports:
    - 9094:9094
    environment:
      KAFKA_ADVERTISED_HOST_NAME: kafka3
      KAFKA_ADVERTISED_PORT: 9094
      KAFKA_ZOOKEEPER_CONNECT: zoo1:2181,zoo2:2181,zoo3:2181
    volumes:
    - /Users/shaozhipeng/Development/volume/kfkluster/kafka3/logs:/kafka
    external_links:
    - zoo1
    - zoo2
    - zoo3
    networks:
      br17219:
        ipv4_address: 172.19.0.16

networks:
  br17219:
    external:
      name: br17219

結果查看和測試

宿主機命令行建立topic

$ pwd
/Users/shaozhipeng/Development/kafka_2.11-2.0.0/bin
$ ./kafka-topics.sh --create --zookeeper localhost:2184,localhost:2185,localhost:2186 --replication-factor 1 --partitions 1 --topic test1

Kafka Tool查看

docker ps查看正在運行的容器

查看宿主機IP地址，並設置Host

這樣宿主機就能夠訪問kafka集羣了

 

使用Docker快速搭建Zookeeper和kafka集羣

瞭解更多可掃碼關注公衆號原文地址：