Running Replicated ZooKeeper

• http://zookeeper.apache.org/doc/current/zookeeperStarted.html#sc_RunningReplicatedZooKeeper

使用k8s部署

k8s 部署 stateful 应用 zookeeper集群

• https://kubernetes.io/docs/tutorials/stateful-application/zookeeper/

kubectl create -f https://k8s.io/docs/tutorials/stateful-application/zookeeper.yaml

k8s 部署 zookeeper cluster

git clone https://github.com/cookeem/kubernetes-zookeeper-cluster.git
cd kubernetes-zookeeper-cluster
kubectl create -f .
kubectl get pods -l app=zk
kubectl get svc -l app=zk

#check
kubectl exec -ti zk-0-** -- /bin/bash
bash-4.3# zkCli.sh -server zk2:2181 ls /

zk-deployment.yaml

---
kind: Deployment
apiVersion: extensions/v1beta1
metadata:
  name: zk-0
spec:
  replicas: 1
  template:
    metadata:
      labels:
        app: zk
        node: node0
    spec:
      hostname: zk-0
      volumes:
        - name: zk-data
          hostPath: 
            path: /data/zk-cluster/zk-data-0
        - name: zk-logs
          hostPath: 
            path: /data/zk-cluster/zk-logs-0
      containers:
      - name: zk-0
        image: zookeeper
        imagePullPolicy: IfNotPresent
        volumeMounts:
        - name: zk-data
          readOnly: false
          mountPath: "/data/zk-data"
        - name: zk-logs
          readOnly: false
          mountPath: "/data/zk-logs"
        ports:
        - containerPort: 2181
        - containerPort: 2888
        - containerPort: 3888
        #command: ['tail', '-f', '/etc/hosts']  
        env:
        - name: ZOO_MY_ID
          value: '0'
        - name: ZOO_SERVERS
          # 注意！！k8s使用到virtual ip，因此，本机必须使用0.0.0.0 ip地址，否则本机zk启动会异常：
          # ERROR [zk1/10.0.0.251:3888:QuorumCnxManager$Listener@547] - Exception while listening
          # java.net.BindException: Address not available (Bind failed)错误
          value: server.0=0.0.0.0:2888:3888 server.1=zk-1:2888:3888 server.2=zk-2:2888:3888
        - name: ZOO_DATA_DIR
          value: '/data/zk-data'
        - name: ZOO_DATA_LOG_DIR
          value: '/data/zk-logs'  

---
kind: Deployment
apiVersion: extensions/v1beta1
metadata:
  name: zk-1
spec:
  replicas: 1
  template:
    metadata:
      labels:
        app: zk
        node: node1
    spec:
      hostname: zk-1
      volumes:
        - name: zk-data
          hostPath: 
            path: /data/zk-cluster/zk-data-1
        - name: zk-logs
          hostPath: 
            path: /data/zk-cluster/zk-logs-1
      containers:
      - name: zk-1
        image: zookeeper
        imagePullPolicy: IfNotPresent
        volumeMounts:
        - name: zk-data
          readOnly: false
          mountPath: "/data/zk-data"
        - name: zk-logs
          readOnly: false
          mountPath: "/data/zk-logs"
        ports:
        - containerPort: 2181
        - containerPort: 2888
        - containerPort: 3888
        #command: ['tail', '-f', '/etc/hosts']  
        env:
        - name: ZOO_MY_ID
          value: '1'
        - name: ZOO_SERVERS
          # 注意！！k8s使用到virtual ip，因此，本机必须使用0.0.0.0 ip地址，否则本机zk启动会异常：
          # ERROR [zk1/10.0.0.251:3888:QuorumCnxManager$Listener@547] - Exception while listening
          # java.net.BindException: Address not available (Bind failed)错误
          value: server.0=zk-0:2888:3888 server.1=0.0.0.0:2888:3888 server.2=zk-2:2888:3888
        - name: ZOO_DATA_DIR
          value: '/data/zk-data'
        - name: ZOO_DATA_LOG_DIR
          value: '/data/zk-logs'  

---
kind: Deployment
apiVersion: extensions/v1beta1
metadata:
  name: zk-2
spec:
  replicas: 1
  template:
    metadata:
      labels:
        app: zk
        node: node2
    spec:
      hostname: zk-2
      volumes:
        - name: zk-data
          hostPath: 
            path: /data/zk-cluster/zk-data-2
        - name: zk-logs
          hostPath: 
            path: /data/zk-cluster/zk-logs-2
      containers:
      - name: zk-2
        image: zookeeper
        imagePullPolicy: IfNotPresent
        volumeMounts:
        - name: zk-data
          readOnly: false
          mountPath: "/data/zk-data"
        - name: zk-logs
          readOnly: false
          mountPath: "/data/zk-logs"
        ports:
        - containerPort: 2181
        - containerPort: 2888
        - containerPort: 3888
        #command: ['tail', '-f', '/etc/hosts']  
        env:
        - name: ZOO_MY_ID
          value: '2'
        - name: ZOO_SERVERS
          # 注意！！k8s使用到virtual ip，因此，本机必须使用0.0.0.0 ip地址，否则本机zk启动会异常：
          # ERROR [zk1/10.0.0.251:3888:QuorumCnxManager$Listener@547] - Exception while listening
          # java.net.BindException: Address not available (Bind failed)错误
          value: server.0=zk-0:2888:3888 server.1=zk-1:2888:3888 server.2=0.0.0.0:2888:3888
        - name: ZOO_DATA_DIR
          value: '/data/zk-data'
        - name: ZOO_DATA_LOG_DIR
          value: '/data/zk-logs'  

zk-service.yaml

---
kind: Service
apiVersion: v1
metadata:
  name: zk-0
  labels:
    app: zk
    node: node0
spec:
  type: LoadBalancer
  ports:
  - name: port-2181
    port: 2181
  - name: port-2888
    port: 2888
  - name: port-3888
    port: 3888
  selector:
    app: zk
    node: node0

---
kind: Service
apiVersion: v1
metadata:
  name: zk-1
  labels:
    app: zk
    node: node1
spec:
  type: LoadBalancer
  ports:
  - name: port-2181
    port: 2181
  - name: port-2888
    port: 2888
  - name: port-3888
    port: 3888
  selector:
    app: zk
    node: node1

---
kind: Service
apiVersion: v1
metadata:
  name: zk-2
  labels:
    app: zk
    node: node2
spec:
  type: LoadBalancer
  ports:
  - name: port-2181
    port: 2181
  - name: port-2888
    port: 2888
  - name: port-3888
    port: 3888
  selector:
    app: zk
    node: node2

使用docker方式部署

docker-compose部署

• https://hub.docker.com/r/library/zookeeper/

version: '2'
services:
    zoo1:
        image: zookeeper
        restart: always
        ports:
            - 2181:2181
        environment:
            ZOO_MY_ID: 1
            ZOO_SERVERS: server.1=zoo1:2888:3888 server.2=zoo2:2888:3888 server.3=zoo3:2888:3888

    zoo2:
        image: zookeeper
        restart: always
        ports:
            - 2182:2181
        environment:
            ZOO_MY_ID: 2
            ZOO_SERVERS: server.1=zoo1:2888:3888 server.2=zoo2:2888:3888 server.3=zoo3:2888:3888

    zoo3:
        image: zookeeper
        restart: always
        ports:
            - 2183:2181
        environment:
            ZOO_MY_ID: 3
            ZOO_SERVERS: server.1=zoo1:2888:3888 server.2=zoo2:2888:3888 server.3=zoo3:2888:3888

docker cmd

192.168.56.101
192.168.56.102
192.168.56.103

docker run -d \
 --name=zookeeper \
 --net=host \
 -e SERVER_ID=1 \
 -e ADDITIONAL_ZOOKEEPER_1=server.1=192.168.56.101:2888:3888 \
 -e ADDITIONAL_ZOOKEEPER_2=server.2=192.168.56.102:2888:3888 \
 -e ADDITIONAL_ZOOKEEPER_3=server.3=192.168.56.103:2888:3888 \
 -e ADDITIONAL_ZOOKEEPER_4=clientPort=2181 \
 garland/zookeeper

docker run -d \
 --name=zookeeper \
 --net=host \
 -e SERVER_ID=2 \
 -e ADDITIONAL_ZOOKEEPER_1=server.1=192.168.56.101:2888:3888 \
 -e ADDITIONAL_ZOOKEEPER_2=server.2=192.168.56.102:2888:3888 \
 -e ADDITIONAL_ZOOKEEPER_3=server.3=192.168.56.103:2888:3888 \
 -e ADDITIONAL_ZOOKEEPER_4=clientPort=2181 \
 garland/zookeeper



docker run -d \
 --name=zookeeper \
 --net=host \
 -e SERVER_ID=3 \
 -e ADDITIONAL_ZOOKEEPER_1=server.1=192.168.56.101:2888:3888 \
 -e ADDITIONAL_ZOOKEEPER_2=server.2=192.168.56.102:2888:3888 \
 -e ADDITIONAL_ZOOKEEPER_3=server.3=192.168.56.103:2888:3888 \
 -e ADDITIONAL_ZOOKEEPER_4=clientPort=2181 \
 garland/zookeeper

ipforward

echo <<EOF >vi /usr/lib/sysctl.d/00-system.conf
net.ipv4.ip_forward=1
EOF
sysctl -p

端口

firewall-cmd --zone=public --add-port=2181/tcp --permanent
firewall-cmd --zone=public --add-port=2888/tcp --permanent
firewall-cmd --zone=public --add-port=3888/tcp --permanent
firewall-cmd --reload