k8s集羣部署v1.15實踐4:部署etcd集羣

參考文檔

部署etcd集羣

1.下載etcd最新版本

官方地址

[root@k8s-node1 etcd]# wget https://github.com/coreos/etcd/releases/download/v3.3.7/etcd-v3.3.7-linux-amd64.tar.gz
--2019-11-03 21:42:33--  https://github.com/coreos/etcd/releases/download/v3.3.7/etcd-v3.3.7-linux-amd64.tar.gz
Resolving github.com (github.com)... 140.82.114.3
Connecting to github.com (github.com)|140.82.114.3|:443... connected.
HTTP request sent, awaiting response... 301 Moved Permanently
Location: https://github.com/etcd-io/etcd/releases/download/v3.3.7/etcd-v3.3.7-linux-amd64.tar.gz [following]
--2019-11-03 21:42:34--  https://github.com/etcd-io/etcd/releases/download/v3.3.7/etcd-v3.3.7-linux-amd64.tar.gz
Reusing existing connection to github.com:443.
HTTP request sent, awaiting response... 302 Found
Location: https://github-production-release-asset-2e65be.s3.amazonaws.com/11225014/dd736838-6974-11e8-8737-c3613eec99eb?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIAIWNJYAX4CSVEH53A%2F20191104%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20191104T024234Z&X-Amz-Expires=300&X-Amz-Signature=c45bc2504edea625b70821e017939952af01e41fc87e44ce6b82acae00bd3cb3&X-Amz-SignedHeaders=host&actor_id=0&response-content-disposition=attachment%3B%20filename%3Detcd-v3.3.7-linux-amd64.tar.gz&response-content-type=application%2Foctet-stream [following]
--2019-11-03 21:42:34--  https://github-production-release-asset-2e65be.s3.amazonaws.com/11225014/dd736838-6974-11e8-8737-c3613eec99eb?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIAIWNJYAX4CSVEH53A%2F20191104%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20191104T024234Z&X-Amz-Expires=300&X-Amz-Signature=c45bc2504edea625b70821e017939952af01e41fc87e44ce6b82acae00bd3cb3&X-Amz-SignedHeaders=host&actor_id=0&response-content-disposition=attachment%3B%20filename%3Detcd-v3.3.7-linux-amd64.tar.gz&response-content-type=application%2Foctet-stream
Resolving github-production-release-asset-2e65be.s3.amazonaws.com (github-production-release-asset-2e65be.s3.amazonaws.com)... 52.216.114.171
Connecting to github-production-release-asset-2e65be.s3.amazonaws.com (github-production-release-asset-2e65be.s3.amazonaws.com)|52.216.114.171|:443... connected.
HTTP request sent, awaiting response... 200 OK
Length: 11271567 (11M) [application/octet-stream]
Saving to: ‘etcd-v3.3.7-linux-amd64.tar.gz’

100%[==============================================================================================>] 11,271,567  1.01MB/s   in 13s    

2019-11-03 21:42:49 (816 KB/s) - ‘etcd-v3.3.7-linux-amd64.tar.gz’ saved [11271567/11271567]

[root@k8s-node1 etcd]#

[root@k8s-node1 etcd]# ls
etcd-v3.3.7-linux-amd64.tar.gz

解包

[root@k8s-node1 etcd]# tar -zxvf etcd-v3.3.7-linux-amd64.tar.gz

2.分發etcd文件到全部節點

[root@k8s-node1 etcd]# cp etcd-v3.3.7-linux-amd64/etcd* /opt/k8s/bin
[root@k8s-node1 etcd]# scp etcd-v3.3.7-linux-amd64/etcd*  root@k8s-node2:/opt/k8s/bin
etcd                                                                                                  100%   18MB  93.7MB/s   00:00    
etcdctl                                                                                               100%   15MB  95.9MB/s   00:00    
[root@k8s-node1 etcd]# scp etcd-v3.3.7-linux-amd64/etcd*  root@k8s-node3:/opt/k8s/bin
etcd                                                                                                  100%   18MB  88.5MB/s   00:00    
etcdctl                                                                                               100%   15MB  64.3MB/s   00:00    
[root@k8s-node1 etcd]#

3.添加etcd文件的執行權限

[root@k8s-node1 etcd]# chmod +x /opt/k8s/bin/*
[root@k8s-node1 etcd]# ssh k8s-node2 "chmod +x /opt/k8s/bin/*"
[root@k8s-node1 etcd]# ssh k8s-node3 "chmod +x /opt/k8s/bin/*"

4.建立etcd證書和密鑰

建立簽名請求

hosts 字段指定受權使用該證書的 etcd 節點 IP 或域名列表,這裏將 etcd 集羣的三個節點 IP 都列在其中.

[root@k8s-node1 etcd]# pwd
/opt/k8s/k8s_software/etcd
[root@k8s-node1 etcd]# cat etcd-csr.json 
{
"CN": "etcd",
"hosts": [
"127.0.0.1",
"192.168.174.128",
"192.168.174.129",
"192.168.174.130"
],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"ST": "SZ",
"L": "SZ",
"O": "k8s",
"OU": "4Paradigm"
}
]
}
[root@k8s-node1 etcd]#

生成證書和密鑰

[root@k8s-node1 etcd]# cfssl gencert -ca=/etc/kubernetes/cert/ca.pem -ca-key=/etc/kubernetes/cert/ca-key.pem -config=/etc/kubernetes/cert/ca-config.json -profile=kubernetes etcd-csr.json | cfssljson -bare etcd
2019/10/30 03:27:04 [INFO] generate received request
2019/10/30 03:27:04 [INFO] received CSR
2019/10/30 03:27:04 [INFO] generating key: rsa-2048
2019/10/30 03:27:05 [INFO] encoded CSR
2019/10/30 03:27:05 [INFO] signed certificate with serial number 578718875980449416776664445360856794460879035803
2019/10/30 03:27:05 [WARNING] This certificate lacks a "hosts" field. This makes it unsuitable for
websites. For more information see the Baseline Requirements for the Issuance and Management
of Publicly-Trusted Certificates, v.1.1.6, from the CA/Browser Forum (https://cabforum.org);
specifically, section 10.2.3 ("Information Requirements").
[root@k8s-node1 etcd]# ls
etcd.csr  etcd-csr.json  etcd-key.pem  etcd.pem

分發證書和密鑰

[root@k8s-node1 etcd]# cp *.pem  /etc/etcd/cert/
[root@k8s-node1 etcd]# scp *.pem  root@k8s-node2:/etc/etcd/cert/
etcd-key.pem                                                                                          100% 1679     1.3MB/s   00:00    
etcd.pem                                                                                              100% 1415     1.1MB/s   00:00    
[root@k8s-node1 etcd]# scp *.pem  root@k8s-node3:/etc/etcd/cert/
etcd-key.pem                                                                                          100% 1679     1.8MB/s   00:00    
etcd.pem                                                                                              100% 1415     1.3MB/s   00:00    
[root@k8s-node1 etcd]#

5.建立etcd的systemd unit文件

[root@k8s-node1 etcd]# pwd
/opt/k8s/k8s_software/etcd
[root@k8s-node1 etcd]# cat etcd.service.template 
[Unit]
Description=Etcd Server
After=network.target
After=network-online.target
Wants=network-online.target
Documentation=https://github.com/coreos
[Service]
User=k8s
Type=notify
WorkingDirectory=/var/lib/etcd/
ExecStart=/opt/k8s/bin/etcd \
--data-dir=/var/lib/etcd \
--name=##NODE_NAME## \
--cert-file=/etc/etcd/cert/etcd.pem \
--key-file=/etc/etcd/cert/etcd-key.pem \\
--trusted-ca-file=/etc/kubernetes/cert/ca.pem \
--peer-cert-file=/etc/etcd/cert/etcd.pem \
--peer-key-file=/etc/etcd/cert/etcd-key.pem \
--peer-trusted-ca-file=/etc/kubernetes/cert/ca.pem \
--peer-client-cert-auth \
--client-cert-auth \
--listen-peer-urls=https://##NODE_IP##:2380 \
--initial-advertise-peer-urls=https://##NODE_IP##:2380 \
--listen-client-urls=https://##NODE_IP##:2379,http://127.0.0.1:2379\
--advertise-client-urls=https://##NODE_IP##:2379 \
--initial-cluster-token=etcd-cluster-0 \
--initial-cluster=${ETCD_NODES} \
--initial-cluster-state=new
Restart=on-failure
RestartSec=5
LimitNOFILE=65536
[Install]
WantedBy=multi-user.target
[root@k8s-node1 etcd]#

##User:指定以 k8s 帳戶運行.

##WorkingDirectory --data-dir:指定工做目錄和數據目錄爲/var/lib/etcd,需在啓動服務前建立這個目錄.

##--name:指定節點名稱,當 --initial-cluster-state 值爲 new 時,--name 的參數值必須位於 --initial-cluster 列表中.

##--cert-file 、 --key-file:etcd server 與 client 通訊時使用的證書和私鑰.

##--trusted-ca-file:簽名 client 證書的 CA 證書,用於驗證 client 證書.

##--peer-cert-file --peer-key-file:etcd 與 peer 通訊使用的證書和私鑰.

##--peer-trusted-ca-file:簽名 peer 證書的 CA 證書,用於驗證 peer 證書.

6.分發生成的 systemd unit 文件,並修改好各節點配置文件裏的##NODE_NAME##和##NODE_IP##

[root@k8s-node1 etcd]# cp etcd.service.template /etc/systemd/system/etcd.service
[root@k8s-node1 etcd]# scp etcd.service.template root@k8s-node2:/etc/systemd/system/etcd.service
etcd.service.template                                                                                 100% 1020   972.4KB/s   00:00    
[root@k8s-node1 etcd]# scp etcd.service.template root@k8s-node3:/etc/systemd/system/etcd.service
etcd.service.template                                                                                 100% 1020   786.5KB/s   00:00    
[root@k8s-node1 etcd]#

各個節點修改下,修改爲對應的name和ip.

[root@k8s-node1 etcd]# sed  -i 's/##NODE_NAME##/k8s-node1/' /etc/systemd/system/etcd.service 
[root@k8s-node1 etcd]# sed  -i 's/##NODE_IP##/192\.168\.174\.128/g' /etc/systemd/system/etcd.service 
[root@k8s-node1 etcd]# sed  -i 's/\${ETCD_NODES}/k8s-node1=https:\/\/192\.168\.174\.128:2380,k8s-node2=https:\/\/192\.168\.174\.129:2380,k8s-node3=https:\/\/192\.168\.174\.130:2380/' /etc/systemd/system/etcd.service

改好後參考見下:

[root@k8s-node1 etcd]# cat /etc/systemd/system/etcd.service 
[Unit]
Description=Etcd Server
After=network.target
After=network-online.target
Wants=network-online.target
Documentation=https://github.com/coreos
[Service]
User=k8s
Type=notify
WorkingDirectory=/var/lib/etcd/
ExecStart=/opt/k8s/bin/etcd \
--data-dir=/var/lib/etcd \
--name=k8s-node1 \
--cert-file=/etc/etcd/cert/etcd.pem \
--key-file=/etc/etcd/cert/etcd-key.pem \
--trusted-ca-file=/etc/kubernetes/cert/ca.pem \
--peer-cert-file=/etc/etcd/cert/etcd.pem \
--peer-key-file=/etc/etcd/cert/etcd-key.pem \
--peer-trusted-ca-file=/etc/kubernetes/cert/ca.pem \
--peer-client-cert-auth \
--client-cert-auth \
--listen-peer-urls=https://192.168.174.128:2380 \
--initial-advertise-peer-urls=https://192.168.174.128:2380 \
--listen-client-urls=https://192.168.174.128:2379,http://127.0.0.1:2379\
--advertise-client-urls=https://192.168.174.128:2379 \
--initial-cluster-token=etcd-cluster-0 \
--initial-cluster=k8s-node1=https://192.168.174.128:2380,k8s-node2=https://192.168.174.129:2380,k8s-node3=https://192.168.174.130:2380 \
--initial-cluster-state=new
Restart=on-failure
RestartSec=5
LimitNOFILE=65536
[Install]
WantedBy=multi-user.target
[root@k8s-node1 etcd]#

7.檢查並添加權限,全部節點都須要操做

[root@k8s-node1 etcd]# ll /etc/etcd/cert/
total 8
-rw------- 1 root root 1679 Oct 30 03:29 etcd-key.pem
-rw-r--r-- 1 root root 1415 Oct 30 03:29 etcd.pem
[root@k8s-node1 etcd]# chown -R k8s /etc/etcd/cert/
[root@k8s-node1 etcd]# chmod +x -R /etc/etcd/cert/

[root@k8s-node1 etcd]# cd /etc/kubernetes/cert/
[root@k8s-node1 cert]# ll
total 20
-rw-r--r-- 1 root root  292 Oct 30 00:22 ca-config.json
-rw-r--r-- 1 root root  993 Oct 30 00:22 ca.csr
-rw-r--r-- 1 root root  201 Oct 30 00:22 ca-csr.json
-rw------- 1 root root 1675 Oct 30 00:22 ca-key.pem
-rw-r--r-- 1 root root 1338 Oct 30 00:22 ca.pem
[root@k8s-node1 cert]#  chown -R k8s /etc/kubernetes/cert/
[root@k8s-node1 cert]#  chmod -R +x /etc/kubernetes/cert

8.啓動etcd

systemctl daemon-reload && systemctl enable etcd && systemctl restart etcd

整個節點都重啓下,再驗證狀態

[root@k8s-node3 ~]# etcdctl --version
etcdctl version: 3.3.7
API version: 2
[root@k8s-node3 ~]# etcdctl cluster-health
failed to check the health of member 1c83ad9421d77430 on https://192.168.174.130:2379: Get https://192.168.174.130:2379/health: x509: certificate signed by unknown authority
member 1c83ad9421d77430 is unreachable: [https://192.168.174.130:2379] are all unreachable
failed to check the health of member 5eec694677c3c515 on https://192.168.174.129:2379: Get https://192.168.174.129:2379/health: x509: certificate signed by unknown authority
member 5eec694677c3c515 is unreachable: [https://192.168.174.129:2379] are all unreachable
failed to check the health of member 65f8d952bfce7d85 on https://192.168.174.128:2379: Get https://192.168.174.128:2379/health: x509: certificate signed by unknown authority
member 65f8d952bfce7d85 is unreachable: [https://192.168.174.128:2379] are all unreachable
cluster is unavailable
[root@k8s-node3 ~]# etcdctl member list
client: etcd cluster is unavailable or misconfigured; error #0: x509: certificate signed by unknown authority
; error #1: x509: certificate signed by unknown authority
; error #2: x509: certificate signed by unknown authority

[root@k8s-node3 ~]#

帶上證書後正常

[root@k8s-node1 ~]# etcdctl --ca-file=/etc/kubernetes/cert/ca.pem --cert-file=/etc/etcd/cert/etcd.pem --key-file=/etc/etcd/cert/etcd-key.pem cluster-health
member 1c83ad9421d77430 is healthy: got healthy result from https://192.168.174.130:2379
member 5eec694677c3c515 is healthy: got healthy result from https://192.168.174.129:2379
member 65f8d952bfce7d85 is healthy: got healthy result from https://192.168.174.128:2379
cluster is healthy
[root@k8s-node1 ~]# etcdctl --ca-file=/etc/kubernetes/cert/ca.pem --cert-file=/etc/etcd/cert/etcd.pem --key-file=/etc/etcd/cert/etcd-key.pem member list
1c83ad9421d77430: name=k8s-node3 peerURLs=https://192.168.174.130:2380 clientURLs=https://192.168.174.130:2379 isLeader=false
5eec694677c3c515: name=k8s-node2 peerURLs=https://192.168.174.129:2380 clientURLs=https://192.168.174.129:2379 isLeader=true
65f8d952bfce7d85: name=k8s-node1 peerURLs=https://192.168.174.128:2380 clientURLs=https://192.168.174.128:2379 isLeader=false

測試下

[root@k8s-node1 ~]# etcdctl --ca-file=/etc/kubernetes/cert/ca.pem --cert-file=/etc/etcd/cert/etcd.pem --key-file=/etc/etcd/cert/etcd-key.pem mkdir /test
[root@k8s-node1 ~]# etcdctl --ca-file=/etc/kubernetes/cert/ca.pem --cert-file=/etc/etcd/cert/etcd.pem --key-file=/etc/etcd/cert/etcd-key.pem mk /test/t1 00
00
[root@k8s-node1 ~]# etcdctl --ca-file=/etc/kubernetes/cert/ca.pem --cert-file=/etc/etcd/cert/etcd.pem --key-file=/etc/etcd/cert/etcd-key.pem ls /test
/test/t1
[root@k8s-node1 ~]# etcdctl --ca-file=/etc/kubernetes/cert/ca.pem --cert-file=/etc/etcd/cert/etcd.pem --key-file=/etc/etcd/cert/etcd-key.pem get /test/t1
00
[root@k8s-node1 ~]#

[root@k8s-node2 ~]# etcdctl --ca-file=/etc/kubernetes/cert/ca.pem --cert-file=/etc/etcd/cert/etcd.pem --key-file=/etc/etcd/cert/etcd-key.pem get /test/t1
00
[root@k8s-node2 ~]#

[root@k8s-node3 ~]# etcdctl --ca-file=/etc/kubernetes/cert/ca.pem --cert-file=/etc/etcd/cert/etcd.pem --key-file=/etc/etcd/cert/etcd-key.pem get /test/t1
00
[root@k8s-node3 ~]#