kubernetes v1.18.2二進制部署 ipv4 controller-manager 部署

簽發 kube-controller-manager 證書

# 設置鏈接KUBE_APISERVER ip
export KUBE_APISERVER=https://127.0.0.1:5443
# 環境變量沿用kube-apiserver
# 建立 Kubernetes Controller Manager 配置文件
cat << EOF | tee ${HOST_PATH}/cfssl/k8s/k8s-controller-manager.json
{
  "CN": "system:kube-controller-manager",
  "hosts": [""], 
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
            "ST": "$CERT_ST",
            "L": "$CERT_L",
      "O": "system:kube-controller-manager",
      "OU": "Kubernetes-manual"
    }
  ]
}
EOF
# 生成 Kubernetes Controller Manager 證書和私鑰
cfssl gencert \
    -ca=${HOST_PATH}/cfssl/pki/k8s/k8s-ca.pem \
    -ca-key=${HOST_PATH}/cfssl/pki/k8s/k8s-ca-key.pem \
    -config=${HOST_PATH}/cfssl/ca-config.json \
    -profile=${CERT_PROFILE} \
    ${HOST_PATH}/cfssl/k8s/k8s-controller-manager.json | \
    cfssljson -bare ${HOST_PATH}/cfssl/pki/k8s/k8s-controller-manager
# 配置kube-controller-manager.kubeconfig
# 設置集羣參數
kubectl config set-cluster ${CLUSTER_NAME} \
--certificate-authority=${HOST_PATH}/cfssl/pki/k8s/k8s-ca.pem \
--embed-certs=true \
--server=${KUBE_APISERVER} \
--kubeconfig=${HOST_PATH}/kubeconfig/kube-controller-manager.kubeconfig
# 設置客戶端認證參數
kubectl config set-credentials system:kube-controller-manager \
--client-certificate=${HOST_PATH}/cfssl/pki/k8s/k8s-controller-manager.pem \
--embed-certs=true \
--client-key=${HOST_PATH}/cfssl/pki/k8s/k8s-controller-manager-key.pem \
--kubeconfig=${HOST_PATH}/kubeconfig/kube-controller-manager.kubeconfig
# 設置上下文參數
kubectl config set-context ${CLUSTER_NAME} \
--cluster=${CLUSTER_NAME} \
--user=system:kube-controller-manager \
--kubeconfig=${HOST_PATH}/kubeconfig/kube-controller-manager.kubeconfig
# 設置默認上下文
kubectl config use-context ${CLUSTER_NAME} --kubeconfig=${HOST_PATH}/kubeconfig/kube-controller-manager.kubeconfig
# 分發kubeconfig 及 證書文件到遠程服務器
scp ./kubeconfig/kube-controller-manager.kubeconfig 192.168.2.175:/apps/k8s/config
scp ./kubeconfig/kube-controller-manager.kubeconfig 192.168.2.176:/apps/k8s/config
scp ./kubeconfig/kube-controller-manager.kubeconfig 192.168.2.177:/apps/k8s/config
# 分發證書
scp -r ./cfssl/pki/k8s/k8s-controller-manager* 192.168.2.175:/apps/k8s/ssl/k8s
scp -r ./cfssl/pki/k8s/k8s-controller-manager* 192.168.2.176:/apps/k8s/ssl/k8s
scp -r ./cfssl/pki/k8s/k8s-controller-manager* 192.168.2.177:/apps/k8s/ssl/k8s

kube-controller-manager 二進制文件準備

# 進入二進制所在文件夾
cd ${HOST_PATH}/kubernetes/server/bin
scp -r kube-controller-manager 192.168.2.175:/apps/k8s/bin
scp -r kube-controller-manager 192.168.2.176:/apps/k8s/bin
scp -r kube-controller-manager 192.168.2.177:/apps/k8s/bin

kube-controller-manager 配置文件

cd ${HOST_PATH}
# 建立 kube-controller-manager
cat << EOF | tee kube-controller-manager
KUBE_CONTROLLER_MANAGER_OPTS="--logtostderr=false \\
--leader-elect=true \\
--address=0.0.0.0 \\
--service-cluster-ip-range=10.66.0.0/16 \\
--cluster-cidr=10.80.0.0/12 \\
--node-cidr-mask-size=24 \\
--cluster-name=kubernetes \\
--allocate-node-cidrs=true \\
--kubeconfig=/apps/k8s/config/kube-controller-manager.kubeconfig \\
--authentication-kubeconfig=/apps/k8s/config/kube-controller-manager.kubeconfig \\
--authorization-kubeconfig=/apps/k8s/config/kube-controller-manager.kubeconfig \\
--use-service-account-credentials=true \\
--client-ca-file=/apps/k8s/ssl/k8s/k8s-ca.pem \\
--requestheader-client-ca-file=/apps/k8s/ssl/k8s/k8s-ca.pem \\
--node-monitor-grace-period=40s \\
--node-monitor-period=5s \\
--pod-eviction-timeout=5m0s \\
--terminated-pod-gc-threshold=50 \\
--alsologtostderr=true \\
--cluster-signing-cert-file=/apps/k8s/ssl/k8s/k8s-ca.pem \\
--cluster-signing-key-file=/apps/k8s/ssl/k8s/k8s-ca-key.pem  \\
--deployment-controller-sync-period=10s \\
--experimental-cluster-signing-duration=87600h0m0s \\
--enable-garbage-collector=true \\
--root-ca-file=/apps/k8s/ssl/k8s/k8s-ca.pem \\
--service-account-private-key-file=/apps/k8s/ssl/k8s/k8s-ca-key.pem \\
--feature-gates=ServiceTopology=true,EndpointSlice=true \\
--controllers=*,bootstrapsigner,tokencleaner \\
--horizontal-pod-autoscaler-use-rest-clients=true \\
--horizontal-pod-autoscaler-sync-period=10s \\
--flex-volume-plugin-dir=/apps/k8s/kubelet-plugins/volume \\
--tls-cert-file=/apps/k8s/ssl/k8s/k8s-controller-manager.pem \\
--tls-private-key-file=/apps/k8s/ssl/k8s/k8s-controller-manager-key.pem \\
--kube-api-qps=100 \\
--kube-api-burst=100 \\
--log-dir=/apps/k8s/log \\
--v=2"
EOF
# 分發配置文件全部節點配置文件一致
scp -r kube-controller-manager 192.168.2.175:/apps/k8s/conf
scp -r kube-controller-manager 192.168.2.176:/apps/k8s/conf
scp -r kube-controller-manager 192.168.2.177:/apps/k8s/conf

建立 kube-controller-manager systemd文件

cat << EOF | tee kube-controller-manager.service
[Unit]
Description=Kubernetes Controller Manager
Documentation=https://github.com/kubernetes/kubernetes

[Service]
LimitNOFILE=65535
LimitNPROC=65535
LimitCORE=infinity
LimitMEMLOCK=infinity
EnvironmentFile=-/apps/k8s/conf/kube-controller-manager
ExecStart=/apps/k8s/bin/kube-controller-manager \$KUBE_CONTROLLER_MANAGER_OPTS
Restart=on-failure
RestartSec=5
User=k8s

[Install]
WantedBy=multi-user.target
EOF
# 上傳啓動文件到服務器
scp kube-controller-manager.service 192.168.2.175:/usr/lib/systemd/system
scp  kube-controller-manager.service 192.168.2.176:/usr/lib/systemd/system
scp  kube-controller-manager.service 192.168.2.176:/usr/lib/systemd/system

kube-controller-manager 啓動準備

# 給/apps/k8s k8s 用戶權限
ssh  192.168.2.175 chown -R k8s:root /apps/k8s
ssh  192.168.2.176 chown -R k8s:root /apps/k8s
ssh  192.168.2.177 chown -R k8s:root /apps/k8s

kube-controller-manager

# 刷新service
ssh  192.168.2.175 systemctl daemon-reload
ssh  192.168.2.176 systemctl daemon-reload
ssh  192.168.2.177 systemctl daemon-reload
# 設置開機啓動
ssh  192.168.2.175 systemctl enable kube-controller-manager.service
ssh  192.168.2.176 systemctl enable kube-controller-manager.service
ssh  192.168.2.177 systemctl enable kube-controller-manager.service
# 啓動 kube-apiserver
ssh  192.168.2.175 systemctl  start kube-controller-manager.service
ssh  192.168.2.176 systemctl  start kube-controller-manager.service
ssh  192.168.2.177 systemctl  start kube-controller-manager.service
# 查看啓動狀態
ssh  192.168.2.175 systemctl  status kube-controller-manager.service
ssh  192.168.2.176 systemctl  status kube-controller-manager.service
ssh  192.168.2.177 systemctl  status kube-controller-manager.service

驗證 kube-controller-manager 是否啓動成功

root@Qist:/tmp/sss# kubectl get cs
NAME                 STATUS      MESSAGE                                                                                     ERROR
scheduler            Unhealthy   Get http://127.0.0.1:10251/healthz: dial tcp 127.0.0.1:10251: connect: connection refused
controller-manager   Healthy     ok
etcd-1               Healthy     {"health":"true"}
etcd-0               Healthy     {"health":"true"}
etcd-2               Healthy     {"health":"true"}
# 查看kube-controller-manager master 節點 
root@Qist:/tmp/sss# kubectl -n kube-system get endpoints kube-controller-manager -o yaml
apiVersion: v1
kind: Endpoints
metadata:
  annotations:
    control-plane.alpha.kubernetes.io/leader: '{"holderIdentity":"k8s-master-2_85edb989-d5fa-4be3-aaf2-7cb34aee72eb","leaseDurationSeconds":15,"acquireTime":"2020-05-06T10:03:30Z","renewTime":"2020-05-06T10:03:30Z","leaderTransitions":15}'
  creationTimestamp: "2020-05-06T09:59:07Z"
  managedFields:
  - apiVersion: v1
    fieldsType: FieldsV1
    fieldsV1:
      f:metadata:
        f:annotations:
          .: {}
          f:control-plane.alpha.kubernetes.io/leader: {}
    manager: kube-controller-manager
    operation: Update
    time: "2020-05-06T10:03:30Z"
  name: kube-controller-manager
  namespace: kube-system
  resourceVersion: "1969"
  selfLink: /api/v1/namespaces/kube-system/endpoints/kube-controller-manager
  uid: c4102892-c95c-487f-9bff-266f6898a4d2
    # k8s-master-2 爲master 節點。