etcd節點擴容至兩個節點

本篇已經安裝了單個etcd，而後進行擴容etcd節點至2個，安裝單節點請參照：https://www.cnblogs.com/effortsing/p/10295261.html


實驗架構

test1: 192.168.0.91    etcd 
test2: 192.168.0.92    etcd
test3: 192.168.0.93    無


需求：test1節點已經安裝了etcd, 如今要把test2節點做爲etcd節點添加進來



1、環境配置

# 以下操做在全部節點操做


修改主機名

# 注意修改 各自節點對應的 主機名
sed -i '$a\hostname=test2' /etc/hostname
sed -i '$a\hostname=test2' /etc/sysconfig/network && hostnamectl set-hostname test2


配置hosts解析

cat >>/etc/hosts<<EOF
192.168.0.91 test1
192.168.0.92 test2
192.168.0.93 test3
EOF


禁用selinux

sed -i 's/SELINUX=permissive/SELINUX=disabled/' /etc/sysconfig/selinux

sed -i 's/enforcing/disabled/g' /etc/selinux/config


關閉swap

# 註釋/etc/fstab文件裏swap相關的行

sed -i 's/\/dev\/mapper\/centos-swap/#\/dev\/mapper\/centos-swap/g' /etc/fstab


關掉防火牆

systemctl stop firewalld && systemctl disable firewalld


都退出xshell查看主機名


開啓forward

iptables -P FORWARD ACCEPT



配置轉發相關參數
cat >> /etc/sysctl.d/k8s.conf <<EOF
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
vm.swappiness=0
EOF

加載系統參數
sysctl --system



加載ipvs相關內核模塊
若是從新開機，須要從新加載
modprobe ip_vs
modprobe ip_vs_rr
modprobe ip_vs_wrr
modprobe ip_vs_sh
modprobe nf_conntrack_ipv4
lsmod | grep ip_vs


安裝etcd

useradd etcd
mkdir -p /server/software/k8s
mkdir -p /opt/k8s/bin
cd /server/software/k8s
wget https://github.com/coreos/etcd/releases/download/v3.2.18/etcd-v3.2.18-linux-amd64.tar.gz
tar -xf etcd-v3.2.18-linux-amd64.tar.gz
mv etcd-v3.2.18-linux-amd64/etcd* /opt/k8s/bin
chmod +x /opt/k8s/bin/*
ln -s /opt/k8s/bin/etcd /usr/bin/etcd
ln -s /opt/k8s/bin/etcdctl /usr/bin/etcdctl
etcd --version
 

二、複製test1單節點ca證書和私鑰、etcd證書和私鑰



test2 上先建立存放證書和私鑰目錄

mkdir -p /etc/kubernetes/cert/

mkdir -p /etc/etcd/cert/



從test1節點上找到證書複製過來

scp /etc/kubernetes/cert/ca*.pem root@192.168.0.92:/etc/kubernetes/cert/

scp /etc/etcd/cert/etcd*.pem root@192.168.0.92:/etc/etcd/cert/

  chmod 777 /etc/etcd/cert/*

  chmod 777 /etc/kubernetes/*



三、添加證書到linux系統受信任列表（選作）


添加根證書到linux系統受信任列表

cd /etc/kubernetes/cert/
cat ca.pem >> /etc/pki/tls/certs/ca-bundle.crt



添加etcd 證書到linux系統受信任列表

cd /etc/etcd/cert/
cat etcd.pem >> /etc/pki/tls/certs/ca-bundle.crt



五、添加新成員

5.一、註冊新節點

# 在test1節點上操做

# etcdctl 在註冊完新節點後，會返回一段提示，包含3個環境變量，須要在etcd啓動腳本中修改爲這3個環境變量
[root@test1 ~]# etcdctl --ca-file /root/ssl/ca.pem --cert-file /root/ssl/etcd.pem --key-file /root/ssl/etcd-key.pem member add test2 http://192.168.0.92:2380
Added member named test2 with ID 2eb21869c9f27e29 to cluster

ETCD_NAME="test2"
ETCD_INITIAL_CLUSTER="test2=http://192.168.0.92:2380,test1=https://192.168.0.91:2380"
ETCD_INITIAL_CLUSTER_STATE="existing"


5.二、修改test2節點的peer url爲https
etcdctl --ca-file /etc/kubernetes/cert/ca.pem --cert-file /etc/etcd/cert/etcd.pem --key-file /etc/etcd/cert/etcd-key.pem member update test2 https://192.168.0.92:2380
若是執行失敗，先跳過，不影響


六、 配置環境變量

# 在 test2 節點上操做

cat >> /etc/profile << EOF
export ETCD_NAME=$(hostname)
export INTERNAL_IP=$(hostname -i | awk '{print $NF}')
export ECTD_CLUSTER='test1=https://192.168.0.91:2380'
EOF
source /etc/profile



七、配置啓動腳本

# 在 test2 節點操做

# 把第5 步 生成的結果寫到 etcd啓動腳本中，

mkdir -p /data/etcd

cat> /etc/systemd/system/etcd.service<< EOF
[Unit]
Description=Etcd Server
After=network.target
After=network-online.target
Wants=network-online.target
Documentation=https://github.com/coreos

[Service]
Type=notify
WorkingDirectory=/data/etcd
EnvironmentFile=-/etc/etcd/etcd.conf
ExecStart=/opt/k8s/bin/etcd \
--name ${ETCD_NAME} \
--cert-file=/etc/etcd/cert/etcd.pem \
--key-file=/etc/etcd/cert/etcd-key.pem \
--peer-cert-file=/etc/etcd/cert/etcd.pem \
--peer-key-file=/etc/etcd/cert/etcd-key.pem \
--trusted-ca-file=/etc/kubernetes/cert/ca.pem \
--peer-trusted-ca-file=/etc/kubernetes/cert/ca.pem \
--initial-advertise-peer-urls https://${INTERNAL_IP}:2380 \
--listen-peer-urls https://${INTERNAL_IP}:2380 \\
--listen-client-urls https://${INTERNAL_IP}:2379,http://127.0.0.1:2379 \
--advertise-client-urls https://${INTERNAL_IP}:2379 \
--initial-cluster-token my-etcd-token \
--initial-cluster $ECTD_CLUSTER,test2=http://192.168.0.92:2380 \
--initial-cluster-state existing \
--data-dir=/data/etcd
Restart=on-failure
RestartSec=5
LimitNOFILE=65536

[Install]
WantedBy=multi-user.target
EOF




八、啓動新節點

systemctl daemon-reload      #必定要執行，不然報錯
systemctl start etcd
systemctl status etcd
systemctl enable etcd


九、查看集羣成員和安全狀態
# 在test1節點上操做
[root@test1 ~]# etcdctl --ca-file /etc/kubernetes/cert/ca.pem --cert-file /etc/etcd/cert/etcd.pem --key-file /etc/etcd/cert/etcd-key.pem member list
2eb21869c9f27e29: name=test2 peerURLs=http://192.168.0.92:2380 clientURLs=https://192.168.0.92:2379 isLeader=false
42f7141ed6110de1: name=test1 peerURLs=https://192.168.0.91:2380 clientURLs=https://192.168.0.91:2379 isLeader=true


[root@test1 ~]# etcdctl --ca-file /etc/kubernetes/cert/ca.pem --cert-file /etc/etcd/cert/etcd.pem --key-file /etc/etcd/cert/etcd-key.pem cluster-health
member 2eb21869c9f27e29 is healthy: got healthy result from https://192.168.0.92:2379
member 42f7141ed6110de1 is healthy: got healthy result from https://192.168.0.91:2379
cluster is healthy


9.二、修改全部節點啓動文件一致，而後重啓


報錯解決：

[root@test2 k8s]# etcdctl --ca-file /etc/kubernetes/cert/ca.pem --cert-file /etc/etcd/cert/etcd.pem --key-file /etc/etcd/cert/etcd-key.pem member list
Error: client: etcd cluster is unavailable or misconfigured; error #0: dial tcp 127.0.0.1:4001: getsockopt: connection refused
; error #1: dial tcp 127.0.0.1:2379: getsockopt: connection refused

error #0: dial tcp 127.0.0.1:4001: getsockopt: connection refused
error #1: dial tcp 127.0.0.1:2379: getsockopt: connection refused


啓動文件須要添加：
http://127.0.0.1:2379


其餘報錯，刪除etcd數據目錄重啓