redis cluster安裝部署(測試環境)

時間 2019-11-13

原文原文鏈接

redis 應用於web前端，作緩存和數據存取的速度是挺可觀的，最近看了一些資料，手癢了，就弄了一個測試環境，兩臺方案，試用一下。前端

##Redis 集羣部署##python

一，方案調研:linux

參考博客:

http://jolestar.com/redis-ha/

http://www.luocs.com/archives/tag/redis

https://github.com/wandoulabs/codis/blob/master/doc/tutorial_zh.md

https://github.com/twitter/twemproxy

二，部署架構ios

三,方案簡介：git

需求:

    緩存:定時任務臨時寫入和讀取的數據量大概在10G左右
    存儲:平常存儲量大概在40G左右
    須要7＊24小時不間斷提供數據服務，因此要求redis集羣要穩定，在有故障時可自動切換和數據即時恢復，並在業務增加或降低時，集羣可伸縮或擴展（手動），並不影響在線業務，故調研測試這兩種方案，以codis 爲代理的codis server redis的集羣，以twemproxy爲代理的redis集羣
codis 集羣技術原理參考  https://github.com/wandoulabs/codis/blob/master/doc/tutorial_zh.md
twemproxy 集羣技術原理參考:
https://github.com/twitter/twemproxy
http://blog.jpush.cn/redis-twemproxy-benchmark/

四,部署架構規劃github

服務器操做系統： centos 6.x  注意:centos 5版本的內核支持不了codis 安裝編譯
服務器部署規劃:
192.168.5.14        keepalived ,codis相關服務
192.168.5.15        keepalived,codis相關服務
192.168.5.16        twemporxy,redis,haproxy
192.168.5.44        twemporxy,redis,haproxy

五,部署安裝golang

1,codis 方案安裝web

1.1安裝go 環境:
首先按照golang，下載地址：https://golang.org/dl/，最新的1.4.2版本。 
若是被牆使用golang中國下載http://golangtc.com/download。 
cd /usr/loca/
wget https://storage.googleapis.com/golang/go1.4.2.linux-amd64.tar.gz
tar -zxvf go1.4.2.linux-amd64.tar.gz


vim /etc/profile
最後追加
#set go & codis environment
GOROOT=/usr/local/go
PATH=$PATH:$GOROOT/bin
GOPATH=/usr/local/codis
export GOROOT  PATH  GOPATH

source /etc/profile

1.2 安裝配置codis 相關配置

cd /usr/local
git clone https://github.com/wandoulabs/codis.git
#將codis的pkg包拷貝到$GOPATH目錄。
mkdir -p /usr/local/codis/src/github.com/wandoulabs/codis

cp /usr/local/codis/pkg/ /usr/local/codis/cmd/  /usr/local/codis/src/github.com/wandoulabs/codis -R

cd /usr/local/codis/ ; sh bootstrap.sh   #安裝編譯須要些時間，過程略過...... 大概3分鐘左右，個人是這樣，你的環境，哈哈，你懂的，

 1.3配置腳本:

github上原碼中有示例，能夠參考試用  https://github.com/wandoulabs/codis
在$path/sample 目錄下:

cat startall.sh
./start_dashboard.sh

sleep 3

./start_redis.sh

./add_group.sh

./initslot.sh

./start_proxy.sh

./set_proxy_online.sh

cat  start_dashboard.sh


#!/bin/sh

    nohup ../bin/codis-config -c config.ini -L ./log/dashboard.log dashboard --addr=:18087 --http-log=./log/requests.log &>/dev/null &



啓動前注意一下redis的配置文件./redis_conf/*.conf
cat  start_redis.sh



#!/bin/sh



    nohup ../bin/codis-server ./redis_conf/6381.conf &> ./log/redis_6381.log &

    nohup ../bin/codis-server ./redis_conf/6382.conf &> ./log/redis_6382.log &

    echo "sleep 3s"

    sleep 3

    tail -n 30 ./log/redis_6381.log

    tail -n 30 ./log/redis_6382.log


        cat   add_group.sh


    #!/bin/sh

    echo "add group 1 with a master(localhost:6381), Notice: do not use localhost when in produciton"

    ../bin/codis-config -c config.ini -L ./log/cconfig.log server add 5 192.168.5.44:6381 master



    echo "add group 2 with a master(localhost:6382), Notice: do not use localhost when in produciton"

    ../bin/codis-config -c config.ini -L ./log/cconfig.log server add 6 192.168.5.44:6382 master

        cat  ./initslot.sh

        #!/bin/sh

    echo "slots initializing..."

    #../bin/codis-config -c config.ini slot init -f

    echo "done"



    echo "set slot ranges to server groups..."

    ../bin/codis-config -c  config.ini slot range-set 0 511 5 online

    ../bin/codis-config -c  config.ini slot range-set 512 1023 6 online

    echo "done"


        cat  ./start_proxy.sh
        #!/bin/sh

    echo "shut down proxy_g5_g6.."

    ../bin/codis-config -c config.ini proxy offline proxy_g5_g6

    echo "done"



    echo "start new proxy..."

    nohup ../bin/codis-proxy --log-level info -c config.ini -L ./log/proxy.log  --cpu=8 --addr=0.0.0.0:19000 --http-addr=0.0.0.0:11000 &

    echo "done"



    echo "sleep 3s"

    sleep 3

    tail -n 30 ./log/proxy.log



        cat  ./set_proxy_online.sh

        #!/bin/sh

        echo "set proxy_1 online"

        ../bin/codis-config -c config.ini proxy online proxy_g5_g6

        echo "done"

1.4有自已編寫的一個類上面全部腳本的python腳本：在測試中，redis

#!/usr/bin/env python # coding:utf8 #author:shantuwqk@163.com import os,sys,commands,time from subprocess import Popen,PIPE from mako.template import Template codis_config = "config.ini" codis_root = "/data/setup/codis/data" def codis_dashboard(opt): if opt == "start": #os.chdir(codis_root) exec_cmd = "cd %s; nohup ../bin/codis-config -c %s -L ./log/dashboard.log dashboard --addr=:18087 --http-log=./log/http.log &>/dev/null &" %(codis_root,codis_config) s = os.system(exec_cmd) if s == 0: print "\033[32;1m codis dashboard start .... OK\033[0m" else: print "\033[31;1m codis dashboard start .... Error\033[0m" elif opt == "stop": dashboard_id = "ps aux |grep \"codis-config\"| grep \"dashboard\"|awk '{print $2}'" s,v = commands.getstatusoutput("kill -9 `%s`" %dashboard_id) if s == 0: print "\033[32;1m KILL codis dashboard id:[%s] OK\033[0m"%dashboard_id else: print "\033[31;1m KILL codis dashboard id:[%s] Error\033[0m"%dashboard_id else: pass def codis_redis(opt,port): if opt == "start": exec_cmd = "cd %s; nohup ../bin/codis-server ./conf/%s.conf &> ./log/%s.log &" %(codis_root,port,port) print exec_cmd print os.system(exec_cmd) s = os.system(exec_cmd) if s == 0: print "\033[32;1m start redis %s server is OK \033[0m"%(port) else: print "\033[31;1m start redis %s server is Error\033[0m"%(port) elif opt == "stop": redis_id = "ps aux | grep codis-server| grep %s|awk '{print $2}'"%port s,v = commands.getstatusoutput("kill -9 `%s`" %redis_id) if s == 0: print "\033[32;1m KILL redis port:[%s] pid:[%s] is OK\033[0m"%(port,redis_id) print v, else: print "\033[31;1m KILL redis port:[%s] pid:[%s] is Error\033[0m"%(port,redis_id) print v, else: pass def codis_group(gid,addr,gtag): exec_cmd = "cd %s; ../bin/codis-config -c %s -L ./log/%s_addgroup.log server add %s %s %s" %(codis_root,codis_config,addr.split(':')[1],gid,addr,gtag) print "start add codis cluster group info.....","\n",exec_cmd s,v = commands.getstatusoutput(exec_cmd) print s,v, if s == 0: print "\033[32;1m add group:[%s] with a gtag:[%s] addr:(%s) OK\033[0m" %(gid,gtag,addr) else: print "\033[31;1m add group:[%s] with a gtag:[%s] addr:(%s) Error\033[0m" %(gid,gtag,addr) def remove_fenc(): remove_fenc = "cd %s;../bin/codis-config -c %s action remove-fence"%(codis_root,codis_config) s,v = commands.getstatusoutput(remove_fenc) print "remove fenc proxy info",v, def slot_init(): init_cmd = "cd %s; ../bin/codis-config -c %s slot init -f" %(codis_root,codis_config) print "INIT SLOT ........","\n",commands.getstatusoutput(init_cmd)[1] def codis_initslot(gid,slot_range): exec_cmd = "cd %s; ../bin/codis-config -c %s slot range-set %s %s %s online"%(codis_root,codis_config,slot_range[0],slot_range[1],gid) print exec_cmd s,v = commands.getstatusoutput(exec_cmd) print v, if s == 0: print "\033[32;1m slot init:[%s],gid:[%s] OK\033[0m"%(slot_range,gid) else: print "\033[31;1m slot init:[%s],gid:[%s] Error\033[0m"%(slot_range,gid) def offline_proxy(): proxy_tag = commands.getstatusoutput("cat %s|grep proxy_id"%codis_config)[1].split('=')[1] exec_down_cmd = "cd %s; ../bin/codis-config -c %s proxy offline %s" %(codis_root,codis_config,proxy_tag) print "Shutdown %s offline....." %proxy_tag print exec_down_cmd commands.getstatusoutput(exec_down_cmd) def codis_proxy(opt): proxy_tag = commands.getstatusoutput("cat %s|grep proxy_id"%codis_config)[1].split('=')[1] if opt == "start": exec_new_cmd = "cd %s; nohup ../bin/codis-proxy --log-level info -c %s -L ./log/%s.log --cpu=8 --addr=0.0.0.0:19000 --http-addr=0.0.0.0:11000 &" %(codis_root,codis_config,proxy_tag) print exec_new_cmd s = os.system(exec_new_cmd) if s == 0: print "\033[32;1m codis proxy tag:[%s]start OK\033[0m"%(proxy_tag) else: print "\033[31;1m codis proxy tag:[%s]start Error\033[0m"%(proxy_tag) time.sleep(5) exec_online_cmd = "cd %s;../bin/codis-config -c %s proxy online %s" %(codis_root,codis_config,proxy_tag) print "Set %s online .....!!" %(proxy_tag) print exec_online_cmd print commands.getstatusoutput(exec_online_cmd)[1] elif opt == "stop": print "Shutdown %s offline....." %proxy_tag exec_down_cmd = "cd %s; ../bin/codis-config -c %s proxy offline %s" %(codis_root,codis_config,proxy_tag) print commands.getstatusoutput(exec_down_cmd)[1] exec_proxy_id = "ps aux |grep codis-proxy| grep -v grep|awk '{print $2}'" print commands.getstatusoutput("kill -9 `%s`"%exec_proxy_id) def get_client_ip(): eth_inter= commands.getstatusoutput("ifconfig -a| awk '/^em/ {;a=$1;FS=\":\"; nextline=NR+1; next}{ if (NR==nextline) { split($2,b,\" \")}{ if ($2 ~ /[0-9]\./) {print a,b[1]}; FS=\" \"}}'|uniq -c|awk '{print $2,$3}'") if eth_inter[0] == 0: return eth_inter[1].split()[1] else: print "get client ip error" def slot_range(n,group): slot_dict = {} num=0 l = range(n) per = len(l) / group #改變i的索引值 for i in l[::per]: avg=l[i:i+per] tupv01 = avg[0],avg[-1] #print type(tupv01),tupv01 #指定當前添加的組數 num+=1 slot_dict[num] = [] slot_dict[num].append(tupv01) #若是隻剩下一組，則將剩餘元素所有追加至列表 if num==group-1: tupv02 = l[i+per:][0],l[i+per:][-1] #print tupv02 v02key = num + 1 slot_dict[v02key] = [] slot_dict[v02key].append(tupv02) break for k,port in redis_instance_port.items(): if k in slot_dict.keys(): slot_dict[k].append(port) return slot_dict def help_prompt(): print """ This program prints files to the standard output. Options include: --version : Prints the version number --help : Helpful tips --task : To operate on missions: [init_codis_cluster] [start_dashborad|stop_dashboard|start_redis|stop_redis|start_proxy|stop_proxy|addgroup|initslot] sample : python es_manage.py --task init_codis_cluster : python es_manage.py --task [start_dashborad|stop_dashboard|start_redis|stop_redis|start_proxy|stop_proxy|addgroup|initslot] """ if __name__ == "__main__": if len(sys.argv) < 2: print "no argument" sys.exit() if sys.argv[1].startswith('--'): option = sys.argv[1][2:] if option == 'version': print 'Version 0.1' elif option == 'help': help_prompt() if option == "task" and sys.argv[2] == "init_codis_cluster" and sys.argv[2] is not None: codis_dashboard('start') #redis_port = ['6381','6382'] #定義所要啓動 redis實例的端口號 redis_instance_port = {1:'6381',2:"6382"} for k,p in redis_instance_port.items(): #生成redis啓動配置文件 template_file = Template(filename="./conf/redis.master.conf.template",module_directory='tmp/test').render(port="%s"%p,memsize=4) conf = open('./conf/%s.conf'%p,'w') conf.write(template_file) conf.close() #啓動redis master 實例 codis_redis('start',p) print slot_range(1024,len(redis_instance_port.keys())) time.sleep(5) remove_fenc() slot_init() for k,v in slot_range(1024,len(redis_instance_port.keys())).items(): codis_group(k,'%s:%s'%(get_client_ip(),v[1]),'master') for k,v in slot_range(1024,len(redis_instance_port.keys())).items(): codis_initslot(k,v[0]) offline_proxy() codis_proxy('start') elif option == "task" and sys.argv[2] == "start_dashborad": codis_dashboard('start') elif option == "task" and sys.argv[2] == "stop_dashboard": codis_dashboard("stop") elif option == "task" and sys.argv[2] == "start_redis" and sys.argv[3] is not None: #redis_tmp_port = "6381" codis_redis('start',sys.argv[3]) elif option == "task" and sys.argv[2] == "stop_redis" and sys.argv[3] is not None: codis_redis('stop',sys.argv[3]) elif option == "task" and sys.argv[2] == "start_proxy": codis_proxy("start") elif option == "task" and sys.argv[2] == "stop_proxy": codis_proxy("stop") elif option == "task" and sys.argv[2] == "stopall": codis_proxy("stop") commands.getstatusoutput("killall codis-server") codis_dashboard("stop") else: help_prompt() #通常狀況下codis集羣在安裝部署初始化的時候，須要先期規劃，已經作好addgrop和initslot工做，後續看需求再補充

2,twemproxy方案安裝:算法

安裝過程參考:

https://github.com/twitter/twemproxy

2.1注意，configure 編譯以前須要系統安裝autoreconf  
    當前用的 autoconf-2.64.tar.gz 這個包，
    tar zxvf autoconf-2.64.tar.gz ; cd autoconf-2.64 ; make && make install 便可
2.2 下載安裝 twemproxy
    git clone git@github.com:twitter/twemproxy.git
    cd twemproxy
    autoreconf -fvi
    mkdir /usr/local/twemporxy
    ./configure –prefix=/usr/local/twemproxy --enable-debug=full
    make
    make install 
    src/nutcracker -h

2.3 配置 twemproxy 代碼 cat /usr/local/twemproxy/etc/544cluster.yml

alpha:

    listen: 0.0.0.0:12000

    hash: fnv1a_64

    distribution: ketama

    auto_eject_hosts: true

    redis: true

    server_retry_timeout: 2000

    server_failure_limit: 1

    servers:

    - 192.168.5.44:6479:1

    - 192.168.5.16:6479:1

配置詳解參考以下:

http://cpjsjxy.iteye.com/blog/2090333

3.haproxy 安裝部署：

3.1下載 haproxy-1.5.10.tar.gz tar zxvf haproxy-1.5.10.tar.gz ; cd haproxy-1.5.10; make TARGET=linux26 ARCH=x86_64&& make install 3.2 配置文件：

global

    daemon

    nbproc 6

    pidfile /var/run/haproxy.pid

    ulimit-n 65535



defaults

    mode tcp                        #mode { tcp|http|health }，tcp 表示4層，http表示7層，health僅做爲健康檢查使用

    retries 2                       #嘗試2次失敗則從集羣摘除

    option redispatch               #若是失效則強制轉換其餘服務器

    option abortonclose             #鏈接數過大自動關閉

    maxconn 1024                    #最大鏈接數

    timeout connect 1d              #鏈接超時時間，重要，hive查詢數據能返回結果的保證

    timeout client 1d               #同上

    timeout server 1d               #同上

    timeout check 2000              #健康檢查時間

    log 127.0.0.1 local0 err #[err warning info debug]



listen  admin_stats                     #定義管理界面

    bind 0.0.0.0:8888               #管理界面訪問IP和端口

    mode http                       #管理界面所使用的協議

    maxconn 10          #最大鏈接數

    stats refresh 30s               #30秒自動刷新

    stats uri /                     #訪問url

    stats realm Hive\ Haproxy       #驗證窗口提示

    stats auth admin:123456         #401驗證用戶名密碼



listen codis-proxy-ha-20000        #codis-proxy

    bind 0.0.0.0:20000             #ha做爲proxy所綁定的IP和端口

    mode tcp                        #以4層方式代理，重要

    balance leastconn               #調度算法 'leastconn' 最少鏈接數分配，或者 'roundrobin'，輪詢分配

    maxconn 1024                    #最大鏈接數

    server codis-proxy2-master 192.168.5.15:19000  check inter 5000 rise 1 fall 2

    server codis-proxy1-master 192.168.5.14:19000  check inter 5000 rise 1 fall 2

    #server codis-server-5156381-master 192.168.5.15:6381  check inter 5000 rise 1 fall 2

    #server codis-server-5156382-master 192.168.5.15:6382  check inter 5000 rise 1 fall 2





listen twemproxy-ha-21000             

    bind 0.0.0.0:21000              

    mode tcp                        

    balance leastconn              

    maxconn 1024 

    server redis-544-master 192.168.5.44:6479  check inter 5000 rise 1 fall 2

    server redis-516-master 192.168.5.16:6479  check inter 5000 rise 1 fall 2



3.3 haproxy 啓動關閉：

  /usr/local/sbin/haproxy -f /data/setup/haproxy-1.5.10/conf/haproxy.cfg

4.keepalived安裝部署: yum install keepalived

主從配置是同樣的，特別注意的配置項：
   state BACKUP
 爲了不網絡不穩定的狀況下，主從搶佔，配爲BACKUP
   interface em2
 VIP 邦定的網卡接口指定

cat /etc/keepalived/keepalived.conf

! Configuration File for keepalived

global_defs {
notification_email {
wangqiankun@lashou-inc.com
}
notification_email_from nagios@lashou.com
smtp_server 127.0.0.1
smtp_connect_timeout 30
router_id ha_01
}

vrrp_instance HA_01 {
state BACKUP
nopreempt
interface em2
virtual_router_id 15
priority 99
advert_int 1
authentication {
    auth_type PASS
    auth_pass 2222
}
virtual_ipaddress {
    192.168.5.90
    192.168.5.91
}
}

virtual_server 192.168.5.90 20000 {
delay_loop 6
lb_algo lc
lb_kind DR
nat_mask 255.255.255.0
#  persistence_timeout 10
protocol TCP

real_server 192.168.5.44 20000 {
    weight 1
    TCP_CHECK {
    connect_timeout 5
    nb_get_retry 3
    delay_before_retry 3
    connect_port 20000
    }
}

real_server 192.168.5.16 20000 {
    weight 1
    TCP_CHECK {
    connect_timeout 5
    nb_get_retry 3
    delay_before_retry 3
    connect_port 20000
    }
}

}
virtual_server 192.168.5.91 21000 {
delay_loop 6
lb_algo lc
lb_kind DR
nat_mask 255.255.255.0
#   persistence_timeout 10
protocol TCP


real_server 192.168.5.44 21000 {
    weight 1
    TCP_CHECK {
    connect_timeout 5
    nb_get_retry 3
    delay_before_retry 3
    connect_port 21000
    }
}

real_server 192.168.5.16 21000 {
    weight 1
    TCP_CHECK {
    connect_timeout 5
    nb_get_retry 3
    delay_before_retry 3
    connect_port 21000
    }
}

}

proxy 作爲keepalived + lvs 的後端真實機須要安裝lvs 並加以下腳本

cat lvs_re.sh 
    #!/bin/bash
    WEB_VIP1=192.168.5.90
    WEB_VIP2=192.168.5.91

    . /etc/rc.d/init.d/functions

    case "$1" in
    start)
    ifconfig lo:0 $WEB_VIP1 netmask 255.255.255.255 broadcast $WEB_VIP1
    ifconfig lo:1 $WEB_VIP2 netmask 255.255.255.255 broadcast $WEB_VIP2
    /sbin/route add -host $WEB_VIP1 dev lo:0
    /sbin/route add -host $WEB_VIP2 dev lo:1
    echo "1" >/proc/sys/net/ipv4/conf/lo/arp_ignore
    echo "2" >/proc/sys/net/ipv4/conf/lo/arp_announce
    echo "1" >/proc/sys/net/ipv4/conf/all/arp_ignore
    echo "2" >/proc/sys/net/ipv4/conf/all/arp_announce
    sysctl -p >/dev/null 2>&1
    echo "RealServer Start OK"

    ;;
    stop)
    ifconfig lo:0 down
    ifconfig lo:1 down
    route del $WEB_VIP1 >/dev/null 2>&1
    route del $WEB_VIP2 >/dev/null 2>&1
    echo "0" >/proc/sys/net/ipv4/conf/lo/arp_ignore
    echo "0" >/proc/sys/net/ipv4/conf/lo/arp_announce
    echo "0" >/proc/sys/net/ipv4/conf/all/arp_ignore
    echo "0" >/proc/sys/net/ipv4/conf/all/arp_announce
    echo "RealServer Stoped"
    ;;

    status)
        # Status of LVS-DR real server.
        islothere=`/sbin/ifconfig lo:0 | grep "$WEB_VIP1"`
        islothere=`/sbin/ifconfig lo:1 | grep "$WEB_VIP2"`
        isrothere=`netstat -rn | grep "lo:0" | grep "$WEB_VIP"`
        if [ ! "$islothere" -o ! "isrothere" ];then
        # Either the route or the lo:0 device
        # not found.
        echo "LVS-DR real server Stopped."
        else
        echo "LVS-DR Running."
        fi
    ;;
    *)
        # Invalid entry.
        echo "$0: Usage: $0 {start|status|stop}"
        exit 1
    ;;
    esac
    exit 0

4.2.啓動keepalived 和 lvs 服務加載配置

/etc/init.d/keepalived start ./lvs_re.sh start

六：平常維護記錄：後續維護過程當中待記錄