kubernetes上部署Memcached高可用記錄

部署Memcached

memcached實現高可用經常使用的兩個工具：
1）magent：鏈接多個memcached，請求轉發
2）repcached：單master單slave，互爲主輔

magent+memcached結構

repcache+memcached結構

主要部署步驟(repcache+memcached)

使用statefulset啓動2個memcached

md-5c107eeae9fb1-0                  1/1       Running     0          59s
md-5c107eeae9fb1-1                  1/1       Running     0          49s

每一個容器初始化腳本

#獲得另外一個節點的HOSTNAME(由於就兩個節點，不是0就是1)
PARTNER=${HOSTNAME##*-}
if [ "${PARTNER}" = "0" ]; then
        PARTNER=${MY_POD_NAME}-1
fi
if [ "${PARTNER}" = "1" ]; then
        PARTNER=${MY_POD_NAME}-0
fi

#判斷另外一個節點的監聽狀態(若是在監據說明是主節點須要複製它，不然反之)
</dev/tcp/${PARTNER}.${MY_POD_NAME}.${NAMESPACE}.svc.cluster.local/${COPY_PORT}
if [[ $? == 0 ]];then
        # 複製master節點
        memcached -p 11211 -X ${COPY_PORT} -x ${PARTNER}.${MY_POD_NAME}.${NAMESPACE}.svc.cluster.local -v -d -u root -m ${MEMCACHED_CACHE_SIZE}
else
        # 啓動master節點
        memcached -p 11211 -X ${COPY_PORT} -v -d -u root -m ${MEMCACHED_CACHE_SIZE}
fi

測試

主備切換

//啓動主節點後狀態爲Listen，設置同步端口爲22222
root@md-5c107eeae9fb1-0:/# memcached -p 11211 -X 22222 -v -d -u root -m 512
root@md-5c107eeae9fb1-0:/# replication: listen
Active Internet connections (only servers)
Proto Recv-Q Send-Q Local Address           Foreign Address         State       PID/Program name
tcp        0      0 0.0.0.0:11211           0.0.0.0:*               LISTEN      260/memcached
tcp        0      0 0.0.0.0:22222           0.0.0.0:*               LISTEN      260/memcached
tcp6       0      0 :::11211                :::*                    LISTEN      260/memcached

//啓動複製節點，一樣設置同步端口爲22222
root@md-5c107eeae9fb1-1:/# memcached -p 11211 -X 22222 -x md-5c107eeae9fb1-0.md-5c107eeae9fb1.itfarm3.svc.cluster.local -v -d -u root -m 512
root@md-5c107eeae9fb1-1:/# replication: connect (peer=10.244.5.50:22222)
replication: marugoto copying
replication: start

//從節點加入後，主節點狀態爲accept並取消對22222的監聽
root@md-5c107eeae9fb1-0:/# replication: listen
replication: accept
Active Internet connections (only servers)
Proto Recv-Q Send-Q Local Address           Foreign Address         State       PID/Program name
tcp        0      0 0.0.0.0:11211           0.0.0.0:*               LISTEN      263/memcached
tcp6       0      0 :::11211                :::*                    LISTEN      263/memcached

//kill掉主節點
root@md-5c107eeae9fb1-0:/# kill 263
replication: cleanup start
replication: close
root@md-5c107eeae9fb1-0:/# replication: cleanup complete

//此時從節點成Listen狀態，監聽複製端口22222，等待加入
root@md-5c107eeae9fb1-1:/# replication: connect (peer=10.244.5.50:22222)
replication: marugoto copying
replication: start
replication: close
replication: listen
Proto Recv-Q Send-Q Local Address           Foreign Address         State       PID/Program name
tcp        0      0 0.0.0.0:11211           0.0.0.0:*               LISTEN      258/memcached
tcp        0      0 0.0.0.0:22222           0.0.0.0:*               LISTEN      258/memcached
tcp6       0      0 :::11211                :::*                    LISTEN      258/memcached

高可用

//啓動2個節點（一主一備）
md-5c107eeae9fb1-0                  1/1       Running     0          59s
md-5c107eeae9fb1-1                  1/1       Running     0          49s

//在主節點中添加item
$ telnet md-5c107eeae9fb1-0.md-5c107eeae9fb1.itfarm3.svc.cluster.local 11211
Trying 10.244.4.162...
Connected to md-5c107eeae9fb1-0.md-5c107eeae9fb1.itfarm3.svc.cluster.local.
Escape character is '^]'.
set name 0 0 8
zhangsan
STORED
quit
Connection closed by foreign host.

//從節點查看item
$ telnet md-5c107eeae9fb1-1.md-5c107eeae9fb1.itfarm3.svc.cluster.local 11211
Trying 10.244.3.176...
Connected to md-5c107eeae9fb1-1.md-5c107eeae9fb1.itfarm3.svc.cluster.local.
Escape character is '^]'.
get name
VALUE name 0 8
zhangsan
END

//模擬故障
$ kubectl.exe delete po md-5c107eeae9fb1-0
pod "md-5c107eeae9fb1-0" deleted

//重啓後數據恢復
$ telnet md-5c107eeae9fb1-0.md-5c107eeae9fb1.itfarm3.svc.cluster.local 11211
Trying 10.244.4.162...
Connected to md-5c107eeae9fb1-0.md-5c107eeae9fb1.itfarm3.svc.cluster.local.
Escape character is '^]'.
get name
VALUE name 0 8
zhangsan
END

部署sasl安全認證的memcached

測試

//使用statefulset啓動N個memcached
md-5c11b67a6ae9b-0                  1/1       Running            0          1h
md-5c11b67a6ae9b-1                  1/1       Running            0          1h
md-5c11b67a6ae9b-2                  1/1       Running            0          1h

使用python鏈接memcached

$ python3
Python 3.5.2 (default, Nov 23 2017, 16:37:01)
[GCC 5.4.0 20160609] on linux
Type "help", "copyright", "credits" or "license" for more information.
>>> import bmemcached
>>> client = bmemcached.Client(('md-5c11b67a6ae9b-ha:11211'), 'md-5c11b67a6ae9b', 'ZcUu1AVGbB3PnDzhxfOjRLJhatugwZBe')
>>> client.set('name', 'zhangsan')
True
>>> client.get('name')
'zhangsan'
>>>

分佈式存儲

使用無頭服務來區分多個memcache

#!/usr/bin/env python3
import bmemcached
client = bmemcached.Client(('md-5c11b67a6ae9b-0.md-5c11b67a6ae9b.itfarm3.svc.cluster.local:11211',
    'md-5c11b67a6ae9b-1.md-5c11b67a6ae9b.itfarm3.svc.cluster.local:11211',
    'md-5c11b67a6ae9b-1.md-5c11b67a6ae9b.itfarm3.svc.cluster.local:11211'), 'md-5c11b67a6ae9b', 'ZcUu1AVGbB3PnDzhxfOjRLJhatugwZBe')
print(client.set('name', 'zhangsan'))
print(client.get('name'))

運行

$ python3 test.py
True
zhangsan

各客戶端鏈接開啓了sasl安全認證的memcached方案