k8s集羣———etcd-三節點部署

etcd集羣部署
1,建立etcd可執行文件，配置文件，證書文件存放目錄
mkdir /opt/etcd/{bin,cfg,ssl} -p

2,建立包文件存放目錄
mkdir /soft -p

3，解壓etcd包。並將可執行文件移動到/opt/etcd/bin
tar zxvf etcd-v3.2.12-linux-amd64.tar.gz
mv etcd-v3.2.12-linux-amd64/{etcd,etcdctl} /opt/etcd/bin/

4,etcd配置文件
$ cat etcd 
#[Member]
ETCD_NAME="etcd01"   #節點名稱，若是有多個節點，這裏必需要改，etcd02，etcd03
ETCD_DATA_DIR="/var/lib/etcd/default.etcd"    #數據目錄
ETCD_LISTEN_PEER_URLS="https://192.168.1.63:2380"   #集羣溝通端口2380
ETCD_LISTEN_CLIENT_URLS="https://192.168.1.63:2379"  #客戶端溝通端口2379

#[Clustering]
ETCD_INITIAL_ADVERTISE_PEER_URLS="https://192.168.1.63:2380"  #集羣通告地址
ETCD_ADVERTISE_CLIENT_URLS="https://192.168.1.63:2379"   #客戶端通告地址
ETCD_INITIAL_CLUSTER="etcd01=https://192.168.1.63:2380,etcd02=https://192.168.1.65:2380,etcd03=https://192.168.1.66:2380"  #這個集羣中全部節點，每一個節點都要有
ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster"   #集羣token
ETCD_INITIAL_CLUSTER_STATE="new"   #新建立集羣，existing表示加入已有集羣
root@k8s-master: /opt/etcd/cfg 17:15:41
$ 

5，systemd管理etcd
#裏面的參數都是須要引用主配置文件的變量，全部若是報錯，嘗試查看一下主配置文件是否配置出錯，/opt/etcd/cfs/etcd
root@k8s-master: /opt/etcd/cfg 17:20:52
$ cat /usr/lib/systemd/system/etcd.service 
[Unit]
Description=Etcd Server
After=network.target
After=network-online.target
Wants=network-online.target

[Service]
Type=notify
EnvironmentFile=/opt/etcd/cfg/etcd
ExecStart=/opt/etcd/bin/etcd --name=${ETCD_NAME} --data-dir=${ETCD_DATA_DIR} --listen-peer-urls=${ETCD_LISTEN_PEER_URLS} --listen-client-urls=${ETCD_LISTEN_CLIENT_URLS},http://127.0.0.1:2379 --advertise-client-urls=${ETCD_ADVERTISE_CLIENT_URLS} --initial-advertise-peer-urls=${ETCD_INITIAL_ADVERTISE_PEER_URLS} --initial-cluster=${ETCD_INITIAL_CLUSTER} --initial-cluster-token=${ETCD_INITIAL_CLUSTER_TOKEN} --initial-cluster-state=new --cert-file=/opt/etcd/ssl/server.pem --key-file=/opt/etcd/ssl/server-key.pem --peer-cert-file=/opt/etcd/ssl/server.pem --peer-key-file=/opt/etcd/ssl/server-key.pem --trusted-ca-file=/opt/etcd/ssl/ca.pem --peer-trusted-ca-file=/opt/etcd/ssl/ca.pem
Restart=on-failure
LimitNOFILE=65536

[Install]
WantedBy=multi-user.target
root@k8s-master: /opt/etcd/cfg 17:21:09
$   

6,從新加載配置文件並啓動
systemctl daemon-reload
systemctl enable etcd
systemctl restart etcd

7,查看啓動日誌
tail -f /var/log/messages    #會出現與node01和node02沒法溝通的情況
#看下邊日誌，這是由於客戶端並無配置etcd節點文件和ssl，因此會一直報錯，systemctl start etcd實際上是啓動成功，可是溝通不到，因此會啓動很長時間
Mar 18 13:54:17 localhost etcd: health check for peer 472edcb0986774fe could not connect: dial tcp 192.168.1.65:2380: connect: connection refused (prober "ROUND_TRIPPER_RAFT_MESSAGE")
Mar 18 13:54:17 localhost etcd: health check for peer 89e49aedde68fee4 could not connect: dial tcp 192.168.1.66:2380: connect: connection refused (prober "ROUND_TRIPPER_RAFT_MESSAGE")
Mar 18 13:54:17 localhost etcd: health check for peer 472edcb0986774fe could not connect: dial tcp 192.168.1.65:2380: connect: connection refused (prober "ROUND_TRIPPER_SNAPSHOT")
Mar 18 13:54:17 localhost etcd: health check for peer 89e49aedde68fee4 could not connect: dial tcp 192.168.1.66:2380: connect: connection refused (prober "ROUND_TRIPPER_SNAPSHOT")


8，node01，node02操做

#將master節點配置文件scp到node01，node02

#將/opt/etcd/下的配置文件文件，文件夾遞歸傳到node01，node02的opt下
scp -r  /opt/etcd/ root@192.168.1.66:/opt
scp -r  /opt/etcd/ root@192.168.1.65:/opt

#將systemctl下的etcd.service傳到node01，node02的/usr/lib/systemd/system/
scp /usr/lib/systemd/system/etcd.service root@192.168.1.65:/usr/lib/systemd/system/
scp /usr/lib/systemd/system/etcd.service root@192.168.1.66:/usr/lib/systemd/system/

#這時在tail -f /var/log/messages
ps:
#因爲環境是虛擬機環境因此，如下日誌是master和node節點時間不一樣步形成的ntpdate time.windows.com

Mar 18 17:30:31 localhost etcd: the clock difference against peer 472edcb0986774fe is too high [15.792944111s > 1s] (prober "ROUND_TRIPPER_RAFT_MESSAGE")
Mar 18 17:30:46 localhost etcd: the clock difference against peer 89e49aedde68fee4 is too high [4.861673928s > 1s] (prober "ROUND_TRIPPER_SNAPSHOT")
Mar 18 17:30:46 localhost etcd: the clock difference against peer 89e49aedde68fee4 is too high [4.858782669s > 1s] (prober "ROUND_TRIPPER_RAFT_MESSAGE")
Mar 18 17:31:01 localhost etcd: the clock difference against peer 472edcb0986774fe is too high [15.793075827s > 1s] (prober "ROUND_TRIPPER_RAFT_MESSAGE")
Mar 18 17:31:01 localhost etcd: the clock difference against peer 472edcb0986774fe is too high [15.795990455s > 1s] (prober "ROUND_TRIPPER_SNAPSHOT")
Mar 18 17:31:16 localhost etcd: the clock difference against peer 89e49aedde68fee4 is too high [4.858938895s > 1s] (prober "ROUND_TRIPPER_RAFT_MESSAGE")
Mar 18 17:31:16 localhost etcd: the clock difference against peer 89e49aedde68fee4 is too high [4.861743791s > 1s] (prober "ROUND_TRIPPER_SNAPSHOT")
Mar 18 17:31:31 localhost etcd: the clock difference against peer 472edcb0986774fe is too high [15.796159244s > 1s] (prober "ROUND_TRIPPER_SNAPSHOT")
Mar 18 17:31:31 localhost etcd: the clock difference against peer 472edcb0986774fe is too high [15.792476037s > 1s] (prober "ROUND_TRIPPER_RAFT_MESSAGE")

$ crontab -l
1 * * * * ntpdate time.windows.com >/dev/null 2>&1

9，最後測試一下集羣節點狀態
（完成）
#若是輸出下面信息，就說明集羣部署成功。若是有問題第一步先看日誌:/var/log/message 或 journalctl -u etcd
root@k8s-master: ~ 17:51:19
$ /opt/etcd/bin/etcdctl --ca-file=/opt/etcd/ssl/ca.pem --cert-file=/opt/etcd/ssl/server.pem --key-file=/opt/etcd/ssl/server-key.pem --endpoints="https://192.168.1.63:2379,https://192.168.1.65:2379,https://192.168.1.66:2379" cluster-health
member 472edcb0986774fe is healthy: got healthy result from https://192.168.1.65:2379
member 89e49aedde68fee4 is healthy: got healthy result from https://192.168.1.66:2379
member ddaf91a76208ea00 is healthy: got healthy result from https://192.168.1.63:2379
cluster is healthy
root@k8s-master: ~ 17:51:20
$