Redis哨兵模式（sentinel）部署記錄（主從複製、讀寫分離、主從切換）

部署環境: CentOS7.5 

192.168.94.11 (master)

192.168.94.22 (slave0)

192.168.94.33 (slave1)

192.168.94.44 (slave2)

全部節點都須要安裝redis+sentinel

[root@master ~]# tar xf redis-4.0.11.tar.gz -C  /usr/local/src/
[root@master ~]# cd /usr/local/src/redis-4.0.11/
[root@master redis-4.0.11]# make && make PREFIX=/usr/local/redis install
[root@master redis-4.0.11]# cd /usr/local/redis/
[root@master redis]# mkdir run conf log dir
[root@master redis]# ln -s /usr/local/redis/bin/* /usr/local/bin/

全部節點修改redis和sentinel配置文件

[root@master redis]# cp /usr/local/src/redis-4.0.11/redis.conf conf/
bind 0.0.0.0
protected-mode no
port 6379
tcp-backlog 511
timeout 0
tcp-keepalive 300
daemonize yes
supervised no
pidfile /usr/local/redis/run/redis_6379.pid
loglevel notice
logfile "/usr/local/redis/log/redis_6379.log"
databases 16
always-show-logo yes
save 900 1
save 300 10
save 60 10000
stop-writes-on-bgsave-error yes
rdbcompression yes
rdbchecksum yes
dbfilename dump.rdb
dir /usr/local/redis/dir
slave-serve-stale-data yes
slave-read-only yes
repl-diskless-sync no
repl-diskless-sync-delay 5
repl-disable-tcp-nodelay no
slave-priority 100
lazyfree-lazy-eviction no
lazyfree-lazy-expire no
lazyfree-lazy-server-del no
slave-lazy-flush no
appendonly yes
appendfilename "appendonly.aof"
appendfsync everysec
no-appendfsync-on-rewrite no
auto-aof-rewrite-percentage 100
auto-aof-rewrite-min-size 64mb
aof-load-truncated yes
aof-use-rdb-preamble no
lua-time-limit 5000
slowlog-log-slower-than 10000
slowlog-max-len 128
latency-monitor-threshold 0
notify-keyspace-events ""
hash-max-ziplist-entries 512
hash-max-ziplist-value 64
list-max-ziplist-size -2
list-compress-depth 0
set-max-intset-entries 512
zset-max-ziplist-entries 128
zset-max-ziplist-value 64
hll-sparse-max-bytes 3000
activerehashing yes
client-output-buffer-limit normal 0 0 0
client-output-buffer-limit slave 256mb 64mb 60
client-output-buffer-limit pubsub 32mb 8mb 60
hz 10
aof-rewrite-incremental-fsync yes
# slave上要添加一條: slaveof 192.168.94.11 6379

[root@master redis]# cp /usr/local/src/redis-4.0.11/sentinel.conf conf/
[root@master ~]# grep -Ev "^#|^$" /usr/local/redis/conf/sentinel.conf 
port 26379
pidfile "/usr/local/redis/run/redis-sentinel.pid"
dir "/usr/local/redis/dir/"
daemonize yes
protected-mode no
logfile "/usr/local/redis/log/redis-sentinel.log"
sentinel monitor mymaster 192.168.94.11 6379 2 
sentinel down-after-milliseconds mymaster 10000 
sentinel parallel-syncs mymaster 1
sentinel failover-timeout mymaster 60000 

 

添加redis到系統服務

[root@master redis]# cp /usr/local/src/redis-4.0.11/utils/redis_init_script /etc/init.d/redis
[root@master ~]# vim /etc/init.d/redis 
#!/bin/sh
#
# Simple Redis init.d script conceived to work on Linux systems
# as it does use of the /proc filesystem.

### BEGIN INIT INFO
# Provides:     redis_6379
# Default-Start:        2 3 4 5
# Default-Stop:         0 1 6
# Short-Description:    Redis data structure server
# Description:          Redis data structure server. See https://redis.io
### END INIT INFO

REDISPORT=6379
EXEC=/usr/local/bin/redis-server
CLIEXEC=/usr/local/bin/redis-cli
PIDFILE=/usr/local/redis/run/redis_${REDISPORT}.pid
CONF="/usr/local/redis/conf/redis.conf"
case "$1" in
    start)
        if [ -f $PIDFILE ]
        then
                echo "$PIDFILE exists, process is already running or crashed"
        else
                echo "Starting Redis server..."
                $EXEC $CONF
        fi
        ;;
    stop)
        if [ ! -f $PIDFILE ]
        then
                echo "$PIDFILE does not exist, process is not running"
        else
                PID=$(cat $PIDFILE)
                echo "Stopping ..."
                $CLIEXEC -p $REDISPORT shutdown
                while [ -x /proc/${PID} ]
                do
                    echo "Waiting for Redis to shutdown ..."
                    sleep 1
                done
                echo "Redis stopped"
        fi
        ;;
    *)
        echo "Please use start or stop as first argument"
        ;;
esac

[root@master redis]# chkconfig --add redis
[root@master redis]# systemctl daemon-reload

全部節點啓動redis、sentinel服務器

[root@master redis]# systemctl start redis
[root@master redis]# systemctl status redis
● redis.service - LSB: Redis data structure server
   Loaded: loaded (/etc/rc.d/init.d/redis; bad; vendor preset: disabled)
   Active: active (running) since 三 2018-09-05 17:31:43 CST; 3s ago
     Docs: man:systemd-sysv-generator(8)
  Process: 5933 ExecStart=/etc/rc.d/init.d/redis start (code=exited, status=0/SUCCESS)
   CGroup: /system.slice/redis.service
           └─5935 /usr/local/bin/redis-server 0.0.0.0:6379

9月 05 17:31:43 master systemd[1]: Starting LSB: Redis data structure server...
9月 05 17:31:43 master redis[5933]: Starting Redis server...
9月 05 17:31:43 master systemd[1]: Started LSB: Redis data structure server.

[root@master redis]# redis-sentinel /usr/local/redis/conf/sentinel.conf
[root@master redis]# netstat -antup|grep 26379
tcp        0      0 0.0.0.0:26379           0.0.0.0:*               LISTEN      23409/redis-sentine 
tcp6       0      0 :::26379                :::*                    LISTEN      23409/redis-sentine 

查看redis和sentinel信息

# 查看節點redis的主從關係
[root@master redis]# redis-cli -h 192.168.94.11 -p 6379 info|grep role
role:master
[root@master redis]# redis-cli -h 192.168.94.22 -p 6379 info|grep role
role:slave
[root@master redis]# redis-cli -h 192.168.94.33 -p 6379 info|grep role
role:slave
[root@master redis]# redis-cli -h 192.168.94.44 -p 6379 info|grep role
role:slave

# 查看master節點信息
[root@master redis]# redis-cli -h 192.168.94.11 -p 6379 info Replication
# Replication
role:master
connected_slaves:3
slave0:ip=192.168.94.33,port=6379,state=online,offset=23362,lag=1
slave1:ip=192.168.94.44,port=6379,state=online,offset=23362,lag=1
slave2:ip=192.168.94.22,port=6379,state=online,offset=23362,lag=1
master_replid:bf73c242b680e63743aa9d46b510ad1aee56105e
master_replid2:0000000000000000000000000000000000000000
master_repl_offset:23503
second_repl_offset:-1
repl_backlog_active:1
repl_backlog_size:1048576
repl_backlog_first_byte_offset:1
repl_backlog_histlen:23503

# 查看master的sentinel配置文件 , 末尾被寫入以下內容
[root@master redis]# grep -Ev '^#|^$' /usr/local/redis/conf/sentinel.conf 
port 26379
dir "/usr/local/redis/dir"
sentinel myid 2b92a48338bcd05968782fb3e4af432ad388e1c6
sentinel deny-scripts-reconfig yes
sentinel monitor mymaster 192.168.94.11 6379 2
sentinel down-after-milliseconds mymaster 10000
sentinel failover-timeout mymaster 60000
pidfile "/usr/local/redis/run/redis_sentinel.pid"
logfile "/usr/local/redis/log/redis_sentinel.log"
daemonize yes
sentinel config-epoch mymaster 0
sentinel leader-epoch mymaster 0
sentinel known-slave mymaster 192.168.94.33 6379
sentinel known-slave mymaster 192.168.94.22 6379
sentinel known-slave mymaster 192.168.94.44 6379
sentinel known-sentinel mymaster 192.168.94.33 26379 cd7c8822acb184eb1af4f96710fc93f5073cfc65
sentinel known-sentinel mymaster 192.168.94.44 26379 be0477f4855b523160d2e487172b94dc1a4c6181
sentinel known-sentinel mymaster 192.168.94.22 26379 f464afa0fb23eb3561e2a448b10a01cdee130d06
sentinel current-epoch 0

# 查看slave0的節點信息
[root@slave0 ~]# redis-cli -h 192.168.94.33 -p 6379 info Replication
# Replication
role:slave
master_host:192.168.94.11
master_port:6379
master_link_status:up
master_last_io_seconds_ago:0
master_sync_in_progress:0
slave_repl_offset:142968
slave_priority:100
slave_read_only:1
connected_slaves:0
master_replid:bf73c242b680e63743aa9d46b510ad1aee56105e
master_replid2:0000000000000000000000000000000000000000
master_repl_offset:142968
second_repl_offset:-1
repl_backlog_active:1
repl_backlog_size:1048576
repl_backlog_first_byte_offset:1
repl_backlog_histlen:142968

# 查看slave的sentinel配置文件 , 在末尾處被寫入以下內容
[root@slave0 ~]# grep -Ev '^#|^$' /usr/local/redis/conf/sentinel.conf 
port 26379
dir "/usr/local/redis/dir"
sentinel myid cd7c8822acb184eb1af4f96710fc93f5073cfc65
sentinel deny-scripts-reconfig yes
sentinel monitor mymaster 192.168.94.11 6379 3
sentinel down-after-milliseconds mymaster 10000
sentinel failover-timeout mymaster 60000
pidfile "/usr/local/redis/run/redis_sentinel.pid"
logfile "/usr/local/redis/log/redis_sentinel.log"
daemonize yes
sentinel config-epoch mymaster 0
sentinel leader-epoch mymaster 0
sentinel known-slave mymaster 192.168.94.22 6379
sentinel known-slave mymaster 192.168.94.33 6379
sentinel known-slave mymaster 192.168.94.44 6379
sentinel known-sentinel mymaster 192.168.94.22 26379 f464afa0fb23eb3561e2a448b10a01cdee130d06
sentinel known-sentinel mymaster 192.168.94.11 26379 2b92a48338bcd05968782fb3e4af432ad388e1c6
sentinel known-sentinel mymaster 192.168.94.44 26379 be0477f4855b523160d2e487172b94dc1a4c6181
sentinel current-epoch 0
# 其餘slave節點也是同樣的

測試客戶端寫入數據

[root@master redis]# redis-cli -h 192.168.94.11 -p 6379
192.168.94.11:6379> set mywife limingming
OK
192.168.94.11:6379> get mywife
"limingming"
192.168.94.11:6379> exit
# 客戶端鏈接任意slave節點get上面的數據
[root@master redis]# redis-cli -h 192.168.94.22 -p 6379
192.168.94.22:6379> get mywife
"limingming"
192.168.94.22:6379> set limingming xiaoxiannv
(error) READONLY You can't write against a read only slave.

# 測試可見 , master節點能夠寫入 , slave節點默認是隻讀 ,不能寫入 
到這就已經實現主從複製 , 讀寫分離了 

經過sentinel實現主從切換 , sentinel須要作高可用防止單點

# 關閉任意slave節點 , 查看狀態
[root@master redis]# redis-cli -h 192.168.94.22 -p 6379
192.168.94.22:6379> get mywife
"limingming"
192.168.94.22:6379> SHUTDOWN
not connected> exit

# 查看剩餘節點信息
[root@master redis]# redis-cli -h 192.168.94.11 -p 6379 info replication
# Replication
role:master
connected_slaves:2     # 由3變成2了 , 減小了一臺slave 
slave0:ip=192.168.94.33,port=6379,state=online,offset=410139,lag=1
slave1:ip=192.168.94.44,port=6379,state=online,offset=410280,lag=0
master_replid:bf73c242b680e63743aa9d46b510ad1aee56105e
master_replid2:0000000000000000000000000000000000000000
master_repl_offset:410280
second_repl_offset:-1
repl_backlog_active:1
repl_backlog_size:1048576
repl_backlog_first_byte_offset:1
repl_backlog_histlen:410280

# 在任意節點查看sentinel日誌 ,  發現slave已是sdown狀態
[root@master redis]# tailf /usr/local/redis/log/redis_sentinel.log 
23409:X 05 Sep 19:02:26.759 * +sentinel sentinel f464afa0fb23eb3561e2a448b10a01cdee130d06 192.168.94.22 26379 @ mymaster 192.168.94.11 6379
23409:X 05 Sep 19:02:27.287 * +sentinel sentinel be0477f4855b523160d2e487172b94dc1a4c6181 192.168.94.44 26379 @ mymaster 192.168.94.11 6379
23409:X 05 Sep 19:02:27.551 * +sentinel sentinel cd7c8822acb184eb1af4f96710fc93f5073cfc65 192.168.94.33 26379 @ mymaster 192.168.94.11 6379
23409:X 05 Sep 19:02:35.097 * +slave slave 192.168.94.33:6379 192.168.94.33 6379 @ mymaster 192.168.94.11 6379
23409:X 05 Sep 19:02:35.098 * +slave slave 192.168.94.44:6379 192.168.94.44 6379 @ mymaster 192.168.94.11 6379
23409:X 05 Sep 19:02:36.796 # +sdown sentinel f464afa0fb23eb3561e2a448b10a01cdee130d06 192.168.94.22 26379 @ mymaster 192.168.94.11 6379
23409:X 05 Sep 19:02:37.327 # +sdown sentinel be0477f4855b523160d2e487172b94dc1a4c6181 192.168.94.44 26379 @ mymaster 192.168.94.11 6379
23409:X 05 Sep 19:02:37.560 # +sdown sentinel cd7c8822acb184eb1af4f96710fc93f5073cfc65 192.168.94.33 26379 @ mymaster 192.168.94.11 6379
23409:X 05 Sep 19:02:45.100 * +slave slave 192.168.94.22:6379 192.168.94.22 6379 @ mymaster 192.168.94.11 6379
23409:X 05 Sep 19:24:46.255 # +sdown slave 192.168.94.22:6379 192.168.94.22 6379 @ mymaster 192.168.94.11 6379

# 而後重啓上面被關閉的節點 ,  全部節點的sentinel均可以檢測到slave已處於可用狀態，此時再來查看節點的主從信息 
[root@master redis]# redis-cli -h 192.168.94.11 -p 6379 info replication
# Replication
role:master
connected_slaves:3
slave0:ip=192.168.94.33,port=6379,state=online,offset=5900717,lag=1
slave1:ip=192.168.94.44,port=6379,state=online,offset=5900858,lag=1
slave2:ip=192.168.94.22,port=6379,state=online,offset=5900858,lag=1
master_replid:bf73c242b680e63743aa9d46b510ad1aee56105e
master_replid2:0000000000000000000000000000000000000000
master_repl_offset:5900858
second_repl_offset:-1
repl_backlog_active:1
repl_backlog_size:1048576
repl_backlog_first_byte_offset:4852283
repl_backlog_histlen:1048576

# 日誌已經顯示slave進入-sdown狀態
[root@master redis]# tailf /usr/local/redis/log/redis_sentinel.log 
23409:X 05 Sep 19:02:26.759 * +sentinel sentinel f464afa0fb23eb3561e2a448b10a01cdee130d06 192.168.94.22 26379 @ mymaster 192.168.94.11 6379
23409:X 05 Sep 19:02:27.287 * +sentinel sentinel be0477f4855b523160d2e487172b94dc1a4c6181 192.168.94.44 26379 @ mymaster 192.168.94.11 6379
23409:X 05 Sep 19:02:27.551 * +sentinel sentinel cd7c8822acb184eb1af4f96710fc93f5073cfc65 192.168.94.33 26379 @ mymaster 192.168.94.11 6379
23409:X 05 Sep 19:02:35.097 * +slave slave 192.168.94.33:6379 192.168.94.33 6379 @ mymaster 192.168.94.11 6379
23409:X 05 Sep 19:02:35.098 * +slave slave 192.168.94.44:6379 192.168.94.44 6379 @ mymaster 192.168.94.11 6379
23409:X 05 Sep 19:02:36.796 # +sdown sentinel f464afa0fb23eb3561e2a448b10a01cdee130d06 192.168.94.22 26379 @ mymaster 192.168.94.11 6379
23409:X 05 Sep 19:02:37.327 # +sdown sentinel be0477f4855b523160d2e487172b94dc1a4c6181 192.168.94.44 26379 @ mymaster 192.168.94.11 6379
23409:X 05 Sep 19:02:37.560 # +sdown sentinel cd7c8822acb184eb1af4f96710fc93f5073cfc65 192.168.94.33 26379 @ mymaster 192.168.94.11 6379
23409:X 05 Sep 19:02:45.100 * +slave slave 192.168.94.22:6379 192.168.94.22 6379 @ mymaster 192.168.94.11 6379
23409:X 05 Sep 19:24:46.255 # +sdown slave 192.168.94.22:6379 192.168.94.22 6379 @ mymaster 192.168.94.11 6379
23409:X 06 Sep 00:53:01.203 * +reboot slave 192.168.94.22:6379 192.168.94.22 6379 @ mymaster 192.168.94.11 6379
23409:X 06 Sep 00:53:01.261 # -sdown slave 192.168.94.22:6379 192.168.94.22 6379 @ mymaster 192.168.94.11 6379

關掉master節點 , 觀察slave節點

[root@master ~]# redis-cli -h 192.168.94.11 -p 6379
192.168.94.11:6379> SHUTDOWN

# 在任意節點查看sentinel日誌 , 發現master已是+sdown的狀態
[root@slave2 ~]# tailf /usr/local/redis/log/redis_sentinel.log 
1279:X 06 Sep 11:06:30.806 * +slave-reconf-done slave 192.168.94.33:6379 192.168.94.33 6379 @ mymaster 192.168.94.11 6379
1279:X 06 Sep 11:06:30.869 * +slave-reconf-sent slave 192.168.94.22:6379 192.168.94.22 6379 @ mymaster 192.168.94.11 6379
1279:X 06 Sep 11:06:31.821 * +slave-reconf-inprog slave 192.168.94.22:6379 192.168.94.22 6379 @ mymaster 192.168.94.11 6379
1279:X 06 Sep 11:06:31.821 * +slave-reconf-done slave 192.168.94.22:6379 192.168.94.22 6379 @ mymaster 192.168.94.11 6379
1279:X 06 Sep 11:06:31.910 # +failover-end master mymaster 192.168.94.11 6379
1279:X 06 Sep 11:06:31.911 # +switch-master mymaster 192.168.94.11 6379 192.168.94.44 6379
1279:X 06 Sep 11:06:31.911 * +slave slave 192.168.94.33:6379 192.168.94.33 6379 @ mymaster 192.168.94.44 6379
1279:X 06 Sep 11:06:31.911 * +slave slave 192.168.94.22:6379 192.168.94.22 6379 @ mymaster 192.168.94.44 6379
1279:X 06 Sep 11:06:31.911 * +slave slave 192.168.94.11:6379 192.168.94.11 6379 @ mymaster 192.168.94.44 6379
1279:X 06 Sep 11:06:41.995 # +sdown slave 192.168.94.11:6379 192.168.94.11 6379 @ mymaster 192.168.94.44 6379

[root@slave2 ~]# redis-cli -h 192.168.94.22 -p 6379 info |grep role
role:slave
[root@slave2 ~]# redis-cli -h 192.168.94.33 -p 6379 info |grep role
role:slave
[root@slave2 ~]# redis-cli -h 192.168.94.44 -p 6379 info |grep role
role:master

[root@slave2 ~]# redis-cli -h 192.168.94.44 -p 6379 info replication
# Replication
role:master
connected_slaves:2
slave0:ip=192.168.94.33,port=6379,state=online,offset=224153,lag=0
slave1:ip=192.168.94.22,port=6379,state=online,offset=223730,lag=1
master_replid:6b197e5e8a9f6b146558a21552e7eb9f30926289
master_replid2:086f725685de634b2b3efd546d150a370e461d07
master_repl_offset:224153
second_repl_offset:62861
repl_backlog_active:1
repl_backlog_size:1048576
repl_backlog_first_byte_offset:1
repl_backlog_histlen:224153

# 能夠看出 , 當master節點(192.168.94.11)的redis關閉後 , slave節點(192.168.94.44)變成了新的master

# 在這個新的master上進行寫的操做是能夠執行的
[root@slave1 ~]# redis-cli -h 192.168.94.44 -p 6379 
192.168.94.44:6379> set mywife limingming
OK
192.168.94.44:6379> get mywife
"limingming"

# 重啓192.168.94.11節點的redis , 等待全部節點的sentinel都檢測到後 ,查看全部節點的主從信息 , 現有的master節點是不會被強佔去的 , 而原來192.168.94.11節點的角色由master變成了slave
[root@slave1 ~]# redis-cli -h 192.168.94.44 -p 6379 info | grep role
role:master
[root@slave1 ~]# redis-cli -h 192.168.94.22 -p 6379 info | grep role
role:slave
[root@slave1 ~]# redis-cli -h 192.168.94.33 -p 6379 info | grep role
role:slave
[root@slave1 ~]# redis-cli -h 192.168.94.11 -p 6379 info | grep role
role:slave

[root@slave1 ~]# redis-cli -h 192.168.94.44 -p 6379 info replication
# Replication
role:master
connected_slaves:3
slave0:ip=192.168.94.33,port=6379,state=online,offset=406595,lag=0
slave1:ip=192.168.94.22,port=6379,state=online,offset=406595,lag=0
slave2:ip=192.168.94.11,port=6379,state=online,offset=406595,lag=0
master_replid:6b197e5e8a9f6b146558a21552e7eb9f30926289
master_replid2:086f725685de634b2b3efd546d150a370e461d07
master_repl_offset:406595
second_repl_offset:62861
repl_backlog_active:1
repl_backlog_size:1048576
repl_backlog_first_byte_offset:1
repl_backlog_histlen:406595
# 全部slave將同步新的master上的信息 主從故障切換已經實現了