Kubernetes 1.11 手動安裝並啓用ipvs

時間 2019-12-24

標籤 kubernetes 1.11 手動安裝啓用 ipvs 简体版

原文原文鏈接

Kubernetes v1.12 手動部署連接

ERROR:

#不少博友說搭建以後出現認證的問題，我驗證了一下，配置是沒有寫錯的
#緣由是51cto的markdown格式有點問題，代碼粘貼上來以後出現了不兼容，縮進異常的狀況
#評論中出現的：error: unable to upgrade connection: Unauthorized
#實際上是由於直接複製代碼生成的/etc/kubernetes/kubelet-config.yml文件縮進有問題
#文章中已經修改了，爲了讓你們少踩點坑，這裏貼出原文：http://note.youdao.com/noteshare?id=31d9d5db79cc3ae27e72c029b09ac4ab&sub=9489CC3D8A8C44F197A8A421DC7209D7

有道筆記原文請點擊這裏！！！！！！！

環境介紹：

系統：Centos 7.5 1804 
內核：3.10.0-862.el7.x86_64

docker版本: 18.06.0-ce

kubernetes版本：v1.11
    master      192.168.1.1
    node1       192.168.1.2
    node2       192.168.1.3

etcd版本：v3.2.22
    etcd1       192.168.1.4
    etcd2       192.168.1.5
    etcd3       192.168.1.6

1、準備工做

爲方便操做，全部操做均以root用戶執行
如下操做僅在kubernetes集羣節點執行便可html

關閉selinux和防火牆

sed -ri 's#(SELINUX=).*#\1disabled#' /etc/selinux/config
setenforce 0

systemctl disable firewalld
systemctl stop firewalld

關閉swap

swapoff -a

配置轉發相關參數，不然可能會出錯

cat <<EOF >  /etc/sysctl.d/k8s.conf
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
vm.swappiness=0
EOF

sysctl --system

加載ipvs模塊

cat << EOF > /etc/sysconfig/modules/ipvs.modules 
#!/bin/bash
ipvs_modules_dir="/usr/lib/modules/\`uname -r\`/kernel/net/netfilter/ipvs"
for i in \`ls \$ipvs_modules_dir | sed  -r 's#(.*).ko.xz#\1#'\`; do
    /sbin/modinfo -F filename \$i  &> /dev/null
    if [ \$? -eq 0 ]; then
        /sbin/modprobe \$i
    fi
done
EOF

chmod +x /etc/sysconfig/modules/ipvs.modules 
bash /etc/sysconfig/modules/ipvs.modules

安裝cfssl

#在master節點安裝便可！！！

wget -O /bin/cfssl https://pkg.cfssl.org/R1.2/cfssl_linux-amd64
wget -O /bin/cfssljson https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64
wget -O /bin/cfssl-certinfo  https://pkg.cfssl.org/R1.2/cfssl-certinfo_linux-amd64
for cfssl in `ls /bin/cfssl*`;do chmod +x $cfssl;done;

安裝docker並幹掉docker0橋

yum install docker-ce

systemctl start docker

cat << EOF > /etc/docker/daemon.json
{   "registry-mirrors": ["https://registry.docker-cn.com"],
    "live-restore": true,
    "default-shm-size": "128M",
    "bridge": "none",
    "max-concurrent-downloads": 10,
    "oom-score-adjust": -1000,
    "debug": false
}   
EOF 

systemctl restart docker

#重啓後執行ip a命令，看不到docker0的網卡便可

2、安裝etcd

準備etcd證書

在master節點上操做node

mkdir -pv $HOME/ssl && cd $HOME/ssl

cat > ca-config.json << EOF
{
  "signing": {
    "default": {
      "expiry": "87600h"
    },
    "profiles": {
      "kubernetes": {
        "usages": [
            "signing",
            "key encipherment",
            "server auth",
            "client auth"
        ],
        "expiry": "87600h"
      }
    }
  }
}
EOF

cat > etcd-ca-csr.json << EOF
{
  "CN": "etcd",
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "ST": "Shenzhen",
      "L": "Shenzhen",
      "O": "etcd",
      "OU": "Etcd Security"
    }
  ]
}
EOF

cat > etcd-csr.json << EOF
{
    "CN": "etcd",
    "hosts": [
      "127.0.0.1",
      "192.168.1.4",
      "192.168.1.5",
      "192.168.1.6"
    ],
    "key": {
        "algo": "rsa",
        "size": 2048
    },
    "names": [
        {
            "C": "CN",
            "ST": "Shenzhen",
            "L": "Shenzhen",
            "O": "etcd",
            "OU": "Etcd Security"
        }
    ]
}
EOF

#生成證書並複製證書至其餘etcd節點

cfssl gencert -initca etcd-ca-csr.json | cfssljson -bare etcd-ca
cfssl gencert -ca=etcd-ca.pem -ca-key=etcd-ca-key.pem -config=ca-config.json -profile=kubernetes etcd-csr.json | cfssljson -bare etcd

mkdir -pv /etc/etcd/ssl
cp etcd*.pem /etc/etcd/ssl

scp -r /etc/etcd 192.168.1.4:/etc/
scp -r /etc/etcd 192.168.1.5:/etc/
scp -r /etc/etcd 192.168.1.6:/etc/

etcd1主機安裝並啓動etcd

yum install -y etcd 

cat << EOF > /etc/etcd/etcd.conf
#[Member]
#ETCD_CORS=""
ETCD_DATA_DIR="/var/lib/etcd/default.etcd"
#ETCD_WAL_DIR=""
ETCD_LISTEN_PEER_URLS="https://192.168.1.4:2380"
ETCD_LISTEN_CLIENT_URLS="https://127.0.0.1:2379,https://192.168.1.4:2379"
#ETCD_MAX_SNAPSHOTS="5"
#ETCD_MAX_WALS="5"
ETCD_NAME="etcd1"
#ETCD_SNAPSHOT_COUNT="100000"
#ETCD_HEARTBEAT_INTERVAL="100"
#ETCD_ELECTION_TIMEOUT="1000"
#ETCD_QUOTA_BACKEND_BYTES="0"
#ETCD_MAX_REQUEST_BYTES="1572864"
#ETCD_GRPC_KEEPALIVE_MIN_TIME="5s"
#ETCD_GRPC_KEEPALIVE_INTERVAL="2h0m0s"
#ETCD_GRPC_KEEPALIVE_TIMEOUT="20s"
#
#[Clustering]
ETCD_INITIAL_ADVERTISE_PEER_URLS="https://192.168.1.4:2380"
ETCD_ADVERTISE_CLIENT_URLS="https://127.0.0.1:2379,https://192.168.1.4:2379"
#ETCD_DISCOVERY=""
#ETCD_DISCOVERY_FALLBACK="proxy"
#ETCD_DISCOVERY_PROXY=""
#ETCD_DISCOVERY_SRV=""
ETCD_INITIAL_CLUSTER="etcd1=https://192.168.1.4:2380,etcd2=https://192.168.1.5:2380,etcd3=https://192.168.1.6:2380"
ETCD_INITIAL_CLUSTER_TOKEN="BigBoss"
#ETCD_INITIAL_CLUSTER_STATE="new"
#ETCD_STRICT_RECONFIG_CHECK="true"
#ETCD_ENABLE_V2="true"
#
#[Proxy]
#ETCD_PROXY="off"
#ETCD_PROXY_FAILURE_WAIT="5000"
#ETCD_PROXY_REFRESH_INTERVAL="30000"
#ETCD_PROXY_DIAL_TIMEOUT="1000"
#ETCD_PROXY_WRITE_TIMEOUT="5000"
#ETCD_PROXY_READ_TIMEOUT="0"
#
#[Security]
ETCD_CERT_FILE="/etc/etcd/ssl/etcd.pem"
ETCD_KEY_FILE="/etc/etcd/ssl/etcd-key.pem"
#ETCD_CLIENT_CERT_AUTH="false"
ETCD_TRUSTED_CA_FILE="/etc/etcd/ssl/etcd-ca.pem"
#ETCD_AUTO_TLS="false"
ETCD_PEER_CERT_FILE="/etc/etcd/ssl/etcd.pem"
ETCD_PEER_KEY_FILE="/etc/etcd/ssl/etcd-key.pem"
#ETCD_PEER_CLIENT_CERT_AUTH="false"
ETCD_PEER_TRUSTED_CA_FILE="/etc/etcd/ssl/etcd-ca.pem"
#ETCD_PEER_AUTO_TLS="false"
#
#[Logging]
#ETCD_DEBUG="false"
#ETCD_LOG_PACKAGE_LEVELS=""
#ETCD_LOG_OUTPUT="default"
#
#[Unsafe]
#ETCD_FORCE_NEW_CLUSTER="false"
#
#[Version]
#ETCD_VERSION="false"
#ETCD_AUTO_COMPACTION_RETENTION="0"
#
#[Profiling]
#ETCD_ENABLE_PPROF="false"
#ETCD_METRICS="basic"
#
#[Auth]
#ETCD_AUTH_TOKEN="simple"
EOF

chown -R etcd.etcd /etc/etcd
systemctl enable etcd
systemctl start etcd

etcd2主機安裝並啓動etcd

yum install -y etcd 

cat << EOF > /etc/etcd/etcd.conf
#[Member]
#ETCD_CORS=""
ETCD_DATA_DIR="/var/lib/etcd/default.etcd"
#ETCD_WAL_DIR=""
ETCD_LISTEN_PEER_URLS="https://192.168.1.5:2380"
ETCD_LISTEN_CLIENT_URLS="https://127.0.0.1:2379,https://192.168.1.5:2379"
#ETCD_MAX_SNAPSHOTS="5"
#ETCD_MAX_WALS="5"
ETCD_NAME="etcd2"
#ETCD_SNAPSHOT_COUNT="100000"
#ETCD_HEARTBEAT_INTERVAL="100"
#ETCD_ELECTION_TIMEOUT="1000"
#ETCD_QUOTA_BACKEND_BYTES="0"
#ETCD_MAX_REQUEST_BYTES="1572864"
#ETCD_GRPC_KEEPALIVE_MIN_TIME="5s"
#ETCD_GRPC_KEEPALIVE_INTERVAL="2h0m0s"
#ETCD_GRPC_KEEPALIVE_TIMEOUT="20s"
#
#[Clustering]
ETCD_INITIAL_ADVERTISE_PEER_URLS="https://192.168.1.5:2380"
ETCD_ADVERTISE_CLIENT_URLS="https://127.0.0.1:2379,https://192.168.1.5:2379"
#ETCD_DISCOVERY=""
#ETCD_DISCOVERY_FALLBACK="proxy"
#ETCD_DISCOVERY_PROXY=""
#ETCD_DISCOVERY_SRV=""
ETCD_INITIAL_CLUSTER="etcd1=https://192.168.1.4:2380,etcd2=https://192.168.1.5:2380,etcd3=https://192.168.1.6:2380"
ETCD_INITIAL_CLUSTER_TOKEN="BigBoss"
#ETCD_INITIAL_CLUSTER_STATE="new"
#ETCD_STRICT_RECONFIG_CHECK="true"
#ETCD_ENABLE_V2="true"
#
#[Proxy]
#ETCD_PROXY="off"
#ETCD_PROXY_FAILURE_WAIT="5000"
#ETCD_PROXY_REFRESH_INTERVAL="30000"
#ETCD_PROXY_DIAL_TIMEOUT="1000"
#ETCD_PROXY_WRITE_TIMEOUT="5000"
#ETCD_PROXY_READ_TIMEOUT="0"
#
#[Security]
ETCD_CERT_FILE="/etc/etcd/ssl/etcd.pem"
ETCD_KEY_FILE="/etc/etcd/ssl/etcd-key.pem"
#ETCD_CLIENT_CERT_AUTH="false"
ETCD_TRUSTED_CA_FILE="/etc/etcd/ssl/etcd-ca.pem"
#ETCD_AUTO_TLS="false"
ETCD_PEER_CERT_FILE="/etc/etcd/ssl/etcd.pem"
ETCD_PEER_KEY_FILE="/etc/etcd/ssl/etcd-key.pem"
#ETCD_PEER_CLIENT_CERT_AUTH="false"
ETCD_PEER_TRUSTED_CA_FILE="/etc/etcd/ssl/etcd-ca.pem"
#ETCD_PEER_AUTO_TLS="false"
#
#[Logging]
#ETCD_DEBUG="false"
#ETCD_LOG_PACKAGE_LEVELS=""
#ETCD_LOG_OUTPUT="default"
#
#[Unsafe]
#ETCD_FORCE_NEW_CLUSTER="false"
#
#[Version]
#ETCD_VERSION="false"
#ETCD_AUTO_COMPACTION_RETENTION="0"
#
#[Profiling]
#ETCD_ENABLE_PPROF="false"
#ETCD_METRICS="basic"
#
#[Auth]
#ETCD_AUTH_TOKEN="simple"
EOF

chown -R etcd.etcd /etc/etcd
systemctl enable etcd
systemctl start etcd

etcd3主機安裝並啓動etcd

yum install -y etcd 

cat << EOF > /etc/etcd/etcd.conf
#[Member]
#ETCD_CORS=""
ETCD_DATA_DIR="/var/lib/etcd/default.etcd"
#ETCD_WAL_DIR=""
ETCD_LISTEN_PEER_URLS="https://192.168.1.6:2380"
ETCD_LISTEN_CLIENT_URLS="https://127.0.0.1:2379,https://192.168.1.6:2379"
#ETCD_MAX_SNAPSHOTS="5"
#ETCD_MAX_WALS="5"
ETCD_NAME="etcd3"
#ETCD_SNAPSHOT_COUNT="100000"
#ETCD_HEARTBEAT_INTERVAL="100"
#ETCD_ELECTION_TIMEOUT="1000"
#ETCD_QUOTA_BACKEND_BYTES="0"
#ETCD_MAX_REQUEST_BYTES="1572864"
#ETCD_GRPC_KEEPALIVE_MIN_TIME="5s"
#ETCD_GRPC_KEEPALIVE_INTERVAL="2h0m0s"
#ETCD_GRPC_KEEPALIVE_TIMEOUT="20s"
#
#[Clustering]
ETCD_INITIAL_ADVERTISE_PEER_URLS="https://192.168.1.6:2380"
ETCD_ADVERTISE_CLIENT_URLS="https://127.0.0.1:2379,https://192.168.1.6:2379"
#ETCD_DISCOVERY=""
#ETCD_DISCOVERY_FALLBACK="proxy"
#ETCD_DISCOVERY_PROXY=""
#ETCD_DISCOVERY_SRV=""
ETCD_INITIAL_CLUSTER="etcd1=https://192.168.1.4:2380,etcd2=https://192.168.1.5:2380,etcd3=https://192.168.1.6:2380"
ETCD_INITIAL_CLUSTER_TOKEN="BigBoss"
#ETCD_INITIAL_CLUSTER_STATE="new"
#ETCD_STRICT_RECONFIG_CHECK="true"
#ETCD_ENABLE_V2="true"
#
#[Proxy]
#ETCD_PROXY="off"
#ETCD_PROXY_FAILURE_WAIT="5000"
#ETCD_PROXY_REFRESH_INTERVAL="30000"
#ETCD_PROXY_DIAL_TIMEOUT="1000"
#ETCD_PROXY_WRITE_TIMEOUT="5000"
#ETCD_PROXY_READ_TIMEOUT="0"
#
#[Security]
ETCD_CERT_FILE="/etc/etcd/ssl/etcd.pem"
ETCD_KEY_FILE="/etc/etcd/ssl/etcd-key.pem"
#ETCD_CLIENT_CERT_AUTH="false"
ETCD_TRUSTED_CA_FILE="/etc/etcd/ssl/etcd-ca.pem"
#ETCD_AUTO_TLS="false"
ETCD_PEER_CERT_FILE="/etc/etcd/ssl/etcd.pem"
ETCD_PEER_KEY_FILE="/etc/etcd/ssl/etcd-key.pem"
#ETCD_PEER_CLIENT_CERT_AUTH="false"
ETCD_PEER_TRUSTED_CA_FILE="/etc/etcd/ssl/etcd-ca.pem"
#ETCD_PEER_AUTO_TLS="false"
#
#[Logging]
#ETCD_DEBUG="false"
#ETCD_LOG_PACKAGE_LEVELS=""
#ETCD_LOG_OUTPUT="default"
#
#[Unsafe]
#ETCD_FORCE_NEW_CLUSTER="false"
#
#[Version]
#ETCD_VERSION="false"
#ETCD_AUTO_COMPACTION_RETENTION="0"
#
#[Profiling]
#ETCD_ENABLE_PPROF="false"
#ETCD_METRICS="basic"
#
#[Auth]
#ETCD_AUTH_TOKEN="simple"
EOF

chown -R etcd.etcd /etc/etcd
systemctl enable etcd
systemctl start etcd

檢查集羣狀態

#在etcd1節點執行

etcdctl --endpoints "https://127.0.0.1:2379"   --ca-file=/etc/etcd/ssl/etcd-ca.pem  \
--cert-file=/etc/etcd/ssl/etcd.pem   --key-file=/etc/etcd/ssl/etcd-key.pem   cluster-health

3、準備kubernetes的證書

在master節點操做linux

mkdir  $HOME/ssl && cd $HOME/ssl

配置 root ca

cat > ca-csr.json << EOF
{
  "CN": "kubernetes",
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "ST": "Shenzhen",
      "L": "Shenzhen",
      "O": "k8s",
      "OU": "System"
    }
  ],
  "ca": {
     "expiry": "87600h"
  }
}
EOF

生成root ca

cfssl gencert -initca ca-csr.json | cfssljson -bare ca
ls ca*.pem

配置kube-apiserver證書

#10.96.0.1 是 kube-apiserver 指定的 service-cluster-ip-range 網段的第一個IP

cat > kube-apiserver-csr.json << EOF
{
    "CN": "kube-apiserver",
    "hosts": [
      "127.0.0.1",
      "192.168.1.1",
      "192.168.1.2",
      "192.168.1.3",
      "10.96.0.1",
      "kubernetes",
      "kubernetes.default",
      "kubernetes.default.svc",
      "kubernetes.default.svc.cluster",
      "kubernetes.default.svc.cluster.local"
    ],
    "key": {
        "algo": "rsa",
        "size": 2048
    },
    "names": [
        {
            "C": "CN",
            "ST": "Shenzhen",
            "L": "Shenzhen",
            "O": "k8s",
            "OU": "System"
        }
    ]
}
EOF

生成kube-apiserver證書

cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes kube-apiserver-csr.json | cfssljson -bare kube-apiserver
ls kube-apiserver*.pem

配置 kube-controller-manager證書

cat > kube-controller-manager-csr.json << EOF
{
    "CN": "system:kube-controller-manager",
    "hosts": [
      "127.0.0.1",
      "192.168.1.1",
      "192.168.1.2",
      "192.168.1.3"
    ],
    "key": {
        "algo": "rsa",
        "size": 2048
    },
    "names": [
        {
            "C": "CN",
            "ST": "Shenzhen",
            "L": "Shenzhen",
            "O": "system:kube-controller-manager",
            "OU": "System"
        }
    ]
}
EOF

生成kube-controller-manager證書

cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes kube-controller-manager-csr.json | cfssljson -bare kube-controller-manager
ls kube-controller-manager*.pem

配置kube-scheduler證書

cat > kube-scheduler-csr.json << EOF
{
    "CN": "system:kube-scheduler",
    "hosts": [
      "127.0.0.1",
      "192.168.1.1",
      "192.168.1.2",
      "192.168.1.3"
    ],
    "key": {
        "algo": "rsa",
        "size": 2048
    },
    "names": [
        {
            "C": "CN",
            "ST": "Shenzhen",
            "L": "Shenzhen",
            "O": "system:kube-scheduler",
            "OU": "System"
        }
    ]
}
EOF

生成kube-scheduler證書

cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes kube-scheduler-csr.json | cfssljson -bare kube-scheduler
ls kube-scheduler*.pem

配置 kube-proxy 證書

cat > kube-proxy-csr.json << EOF
{
    "CN": "system:kube-proxy",
    "key": {
        "algo": "rsa",
        "size": 2048
    },
    "names": [
        {
            "C": "CN",
            "ST": "Shenzhen",
            "L": "Shenzhen",
            "O": "system:kube-proxy",
            "OU": "System"
        }
    ]
}
EOF

生成 kube-proxy 證書

cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes kube-proxy-csr.json | cfssljson -bare kube-proxy
ls kube-proxy*.pem

配置 admin 證書

cat > admin-csr.json << EOF
{
    "CN": "admin",
    "key": {
        "algo": "rsa",
        "size": 2048
    },
    "names": [
        {
            "C": "CN",
            "ST": "Shenzhen",
            "L": "Shenzhen",
            "O": "system:masters",
            "OU": "System"
        }
    ]
}
EOF

生成 admin 證書

cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes admin-csr.json | cfssljson -bare admin
ls admin*.pem

複製生成的證書文件，並分發至其餘節點

mkdir -pv /etc/kubernetes/pki
cp ca*.pem admin*.pem kube-proxy*.pem kube-scheduler*.pem kube-controller-manager*.pem kube-apiserver*.pem /etc/kubernetes/pki
scp -r /etc/kubernetes 192.168.1.2:/etc/
scp -r /etc/kubernetes 192.168.1.3:/etc/

4、開始安裝master

下載解壓server包並配置環境變量

cd /root
wget https://dl.k8s.io/v1.11.1/kubernetes-server-linux-amd64.tar.gz
tar -xf kubernetes-server-linux-amd64.tar.gz -C /usr/local
mv /usr/local/kubernetes /usr/local/kubernetes-v1.11
ln -s kubernetes-v1.11 /usr/local/kubernetes

cat > /etc/profile.d/kubernetes.sh << EOF
k8s_home=/usr/local/kubernetes
export PATH=\$k8s_home/server/bin:\$PATH
source <(kubectl completion bash)
EOF

source /etc/profile.d/kubernetes.sh
kubectl version

生成kubeconfignginx

使用 TLS Bootstrapping

export BOOTSTRAP_TOKEN=$(head -c 16 /dev/urandom | od -An -t x | tr -d ' ')

cat > /etc/kubernetes/token.csv << EOF
${BOOTSTRAP_TOKEN},kubelet-bootstrap,10001,"system:kubelet-bootstrap"
EOF

建立 kubelet bootstrapping kubeconfig

cd /etc/kubernetes

export KUBE_APISERVER="https://192.168.1.1:6443"

kubectl config set-cluster kubernetes \
--certificate-authority=/etc/kubernetes/pki/ca.pem \
--embed-certs=true \
--server=${KUBE_APISERVER} \
--kubeconfig=kubelet-bootstrap.conf

kubectl config set-credentials kubelet-bootstrap \
--token=${BOOTSTRAP_TOKEN} \
--kubeconfig=kubelet-bootstrap.conf

kubectl config set-context default \
--cluster=kubernetes \
--user=kubelet-bootstrap \
--kubeconfig=kubelet-bootstrap.conf

kubectl config use-context default --kubeconfig=kubelet-bootstrap.conf

建立 kube-controller-manager kubeconfig

export KUBE_APISERVER="https://192.168.1.1:6443"

kubectl config set-cluster kubernetes \
--certificate-authority=/etc/kubernetes/pki/ca.pem \
--embed-certs=true \
--server=${KUBE_APISERVER} \
--kubeconfig=kube-controller-manager.conf

kubectl config set-credentials kube-controller-manager \
--client-certificate=/etc/kubernetes/pki/kube-controller-manager.pem \
--client-key=/etc/kubernetes/pki/kube-controller-manager-key.pem \
--embed-certs=true \
--kubeconfig=kube-controller-manager.conf

kubectl config set-context default \
--cluster=kubernetes \
--user=kube-controller-manager \
--kubeconfig=kube-controller-manager.conf

kubectl config use-context default --kubeconfig=kube-controller-manager.conf

建立 kube-scheduler kubeconfig

export KUBE_APISERVER="https://192.168.1.1:6443"

kubectl config set-cluster kubernetes \
 --certificate-authority=/etc/kubernetes/pki/ca.pem \
 --embed-certs=true \
 --server=${KUBE_APISERVER} \
 --kubeconfig=kube-scheduler.conf

kubectl config set-credentials kube-scheduler \
--client-certificate=/etc/kubernetes/pki/kube-scheduler.pem \
--client-key=/etc/kubernetes/pki/kube-scheduler-key.pem \
--embed-certs=true \
--kubeconfig=kube-scheduler.conf

kubectl config set-context default \
--cluster=kubernetes \
--user=kube-scheduler \
--kubeconfig=kube-scheduler.conf

kubectl config use-context default --kubeconfig=kube-scheduler.conf

建立 kube-proxy kubeconfig

export KUBE_APISERVER="https://192.168.1.1:6443"
kubectl config set-cluster kubernetes \
--certificate-authority=/etc/kubernetes/pki/ca.pem \
--embed-certs=true \
--server=${KUBE_APISERVER} \
--kubeconfig=kube-proxy.conf

kubectl config set-credentials kube-proxy \
--client-certificate=/etc/kubernetes/pki/kube-proxy.pem \
--client-key=/etc/kubernetes/pki/kube-proxy-key.pem \
--embed-certs=true \
--kubeconfig=kube-proxy.conf

kubectl config set-context default \
--cluster=kubernetes \
--user=kube-proxy \
--kubeconfig=kube-proxy.conf

kubectl config use-context default --kubeconfig=kube-proxy.conf

建立 admin kubeconfig

export KUBE_APISERVER="https://192.168.1.1:6443"

kubectl config set-cluster kubernetes \
--certificate-authority=/etc/kubernetes/pki/ca.pem \
--embed-certs=true \
--server=${KUBE_APISERVER} \
--kubeconfig=admin.conf

kubectl config set-credentials admin \
--client-certificate=/etc/kubernetes/pki/admin.pem \
--client-key=/etc/kubernetes/pki/admin-key.pem \
--embed-certs=true \
--kubeconfig=admin.conf

kubectl config set-context default \
--cluster=kubernetes \
--user=admin \
--kubeconfig=admin.conf

kubectl config use-context default --kubeconfig=admin.conf

把 kube-proxy.conf 複製到其餘節點

scp kubelet-bootstrap.conf kube-proxy.conf 192.168.1.2:/etc/kubernetes
scp kubelet-bootstrap.conf kube-proxy.conf 192.168.1.3:/etc/kubernetes
cd $HOME

配置啓動kube-apiservergit

複製 etcd ca

mkdir -pv /etc/kubernetes/pki/etcd
cd /etc/etcd/ssl
cp etcd-ca.pem etcd-key.pem etcd.pem /etc/kubernetes/pki/etcd

生成 service account key

openssl genrsa -out /etc/kubernetes/pki/sa.key 2048
openssl rsa -in /etc/kubernetes/pki/sa.key -pubout -out /etc/kubernetes/pki/sa.pub
ls /etc/kubernetes/pki/sa.*
cd $HOME

啓動文件

cat > /etc/systemd/system/kube-apiserver.service << EOF
[Unit]
Description=Kubernetes API Service
Documentation=https://github.com/kubernetes/kubernetes
After=network.target

[Service]
EnvironmentFile=-/etc/kubernetes/config
EnvironmentFile=-/etc/kubernetes/apiserver
ExecStart=/usr/local/kubernetes/server/bin/kube-apiserver \\
      \$KUBE_LOGTOSTDERR \\
      \$KUBE_LOG_LEVEL \\
      \$KUBE_ETCD_ARGS \\
      \$KUBE_API_ADDRESS \\
      \$KUBE_SERVICE_ADDRESSES \\
      \$KUBE_ADMISSION_CONTROL \\
      \$KUBE_APISERVER_ARGS
Restart=on-failure
Type=notify
LimitNOFILE=65536

[Install]
WantedBy=multi-user.target
EOF

該配置文件同時被 kube-apiserver, kube-controller-manager, kube-scheduler, kubelet, kube-proxy 使用

cat > /etc/kubernetes/config << EOF
KUBE_LOGTOSTDERR="--logtostderr=true"
KUBE_LOG_LEVEL="--v=2"
EOF

cat > /etc/kubernetes/apiserver << EOF
KUBE_API_ADDRESS="--advertise-address=192.168.1.1"
KUBE_ETCD_ARGS="--etcd-servers=https://192.168.1.4:2379,https://192.168.1.5:2379,https://192.168.1.6:2379 --etcd-cafile=/etc/kubernetes/pki/etcd/etcd-ca.pem --etcd-certfile=/etc/kubernetes/pki/etcd/etcd.pem --etcd-keyfile=/etc/kubernetes/pki/etcd/etcd-key.pem"
KUBE_SERVICE_ADDRESSES="--service-cluster-ip-range=10.96.0.0/12"
KUBE_ADMISSION_CONTROL="--enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"
KUBE_APISERVER_ARGS="--allow-privileged=true --authorization-mode=Node,RBAC --enable-bootstrap-token-auth=true --token-auth-file=/etc/kubernetes/token.csv --service-node-port-range=30000-32767 --tls-cert-file=/etc/kubernetes/pki/kube-apiserver.pem --tls-private-key-file=/etc/kubernetes/pki/kube-apiserver-key.pem --client-ca-file=/etc/kubernetes/pki/ca.pem --service-account-key-file=/etc/kubernetes/pki/sa.pub --enable-swagger-ui=true --secure-port=6443 --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --anonymous-auth=false --kubelet-client-certificate=/etc/kubernetes/pki/admin.pem --kubelet-client-key=/etc/kubernetes/pki/admin-key.pem"
EOF

啓動

systemctl daemon-reload
systemctl enable kube-apiserver
systemctl start kube-apiserver
systemctl status kube-apiserver

訪問測試

curl -k https://192.168.1.1:6443/

出現一下內容說明搭建成功：
{
"kind": "Status",
"apiVersion": "v1",
"metadata": {

},
"status": "Failure",
"message": "Unauthorized",
"reason": "Unauthorized",
"code": 401
}

配置啓動kube-controller-managergithub

啓動文件

cat > /etc/systemd/system/kube-controller-manager.service << EOF
Description=Kubernetes Controller Manager
Documentation=https://github.com/kubernetes/kubernetes

[Service]
EnvironmentFile=-/etc/kubernetes/config
EnvironmentFile=-/etc/kubernetes/controller-manager
ExecStart=/usr/local/kubernetes/server/bin/kube-controller-manager \\
      \$KUBE_LOGTOSTDERR \\
      \$KUBE_LOG_LEVEL \\
      \$KUBECONFIG \\
      \$KUBE_CONTROLLER_MANAGER_ARGS
Restart=on-failure
LimitNOFILE=65536

[Install]
WantedBy=multi-user.target
EOF

配置文件

cat >/etc/kubernetes/controller-manager<<EOF
KUBECONFIG="--kubeconfig=/etc/kubernetes/kube-controller-manager.conf"
KUBE_CONTROLLER_MANAGER_ARGS="--address=127.0.0.1 --cluster-cidr=10.0.0.0/8 --cluster-name=kubernetes --cluster-signing-cert-file=/etc/kubernetes/pki/ca.pem --cluster-signing-key-file=/etc/kubernetes/pki/ca-key.pem --service-account-private-key-file=/etc/kubernetes/pki/sa.key --root-ca-file=/etc/kubernetes/pki/ca.pem --leader-elect=true --use-service-account-credentials=true --node-monitor-grace-period=10s --pod-eviction-timeout=10s --allocate-node-cidrs=true --controllers=*,bootstrapsigner,tokencleaner"
EOF

啓動

systemctl daemon-reload
systemctl enable kube-controller-manager
systemctl start kube-controller-manager
systemctl status kube-controller-manager

配置啓動kube-schedulerweb

systemctl啓動文件

cat > /etc/systemd/system/kube-scheduler.service << EOF
[Unit]
Description=Kubernetes Scheduler Plugin
Documentation=https://github.com/kubernetes/kubernetes

[Service]
EnvironmentFile=-/etc/kubernetes/config
EnvironmentFile=-/etc/kubernetes/scheduler
ExecStart=/usr/local/kubernetes/server/bin/kube-scheduler \\
          \$KUBE_LOGTOSTDERR \\
          \$KUBE_LOG_LEVEL \\
          \$KUBECONFIG \\
          \$KUBE_SCHEDULER_ARGS
Restart=on-failure
LimitNOFILE=65536

[Install]
WantedBy=multi-user.target
EOF

配置文件

cat > /etc/kubernetes/scheduler << EOF
KUBECONFIG="--kubeconfig=/etc/kubernetes/kube-scheduler.conf"
KUBE_SCHEDULER_ARGS="--leader-elect=true --address=127.0.0.1"
EOF

啓動

systemctl daemon-reload
systemctl enable kube-scheduler
systemctl start kube-scheduler
systemctl status kube-scheduler

配置kubectl

rm -rf $HOME/.kube
mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config
kubectl get node

查看各個組件的狀態

kubectl get componentstatuses   

[root@master ~]# kubectl get componentstatuses
NAME                 STATUS    MESSAGE              ERROR
controller-manager   Healthy   ok
scheduler            Healthy   ok
etcd-1               Healthy   {"health": "true"}
etcd-0               Healthy   {"health": "true"}
etcd-2               Healthy   {"health": "true"}

配置kubelet使用bootstrap

kubectl create clusterrolebinding kubelet-bootstrap \
--clusterrole=system:node-bootstrapper \
--user=kubelet-bootstrap

5、配置cni和kubelet

master端操做docker

下載cni包

cd /root
wget https://github.com/containernetworking/plugins/releases/download/v0.7.1/cni-plugins-amd64-v0.7.1.tgz
mkdir /opt/cni/bin -p
tar -xf cni-plugins-amd64-v0.7.1.tgz -C /opt/cni/bin

配置啓動kubelet

#配置啓動文件

cat > /etc/systemd/system/kubelet.service << EOF
[Unit]
Description=Kubernetes Kubelet Server
Documentation=https://github.com/kubernetes/kubernetes
After=docker.service
Requires=docker.service

[Service]
EnvironmentFile=-/etc/kubernetes/config
EnvironmentFile=-/etc/kubernetes/kubelet
ExecStart=/usr/local/kubernetes/server/bin/kubelet \\
          \$KUBE_LOGTOSTDERR \\
          \$KUBE_LOG_LEVEL \\
          \$KUBELET_CONFIG \\
          \$KUBELET_HOSTNAME \\
          \$KUBELET_POD_INFRA_CONTAINER \\
          \$KUBELET_ARGS
Restart=on-failure

[Install]
WantedBy=multi-user.target
EOF

cat > /etc/kubernetes/config << EOF
KUBE_LOGTOSTDERR="--logtostderr=true"
KUBE_LOG_LEVEL="--v=2"
EOF

cat > /etc/kubernetes/kubelet << EOF
KUBELET_HOSTNAME="--hostname-override=192.168.1.1"
KUBELET_POD_INFRA_CONTAINER="--pod-infra-container-image=registry.cn-hangzhou.aliyuncs.com/google_containers/pause-amd64:3.1"
KUBELET_CONFIG="--config=/etc/kubernetes/kubelet-config.yml"
KUBELET_ARGS="--bootstrap-kubeconfig=/etc/kubernetes/kubelet-bootstrap.conf --kubeconfig=/etc/kubernetes/kubelet.conf --cert-dir=/etc/kubernetes/pki --network-plugin=cni --cni-bin-dir=/opt/cni/bin --cni-conf-dir=/etc/cni/net.d"
EOF

cat > /etc/kubernetes/kubelet-config.yml << EOF
kind: KubeletConfiguration
apiVersion: kubelet.config.k8s.io/v1beta1
address: 192.168.1.1
port: 10250
cgroupDriver: cgroupfs
clusterDNS:
- 10.96.0.10
clusterDomain: cluster.local.
hairpinMode: promiscuous-bridge
serializeImagePulls: false
authentication:
  x509:
     clientCAFile: /etc/kubernetes/pki/ca.pem
 anonymous:
    enabled: false
 webhook:
    enabled: false
EOF

啓動kubelet

systemctl daemon-reload
systemctl enable kubelet
systemctl restart kubelet
systemctl status kubelet

在node1上操做json

下載cni包

cd /root
wget https://dl.k8s.io/v1.11.1/kubernetes-node-linux-amd64.tar.gz
wget https://github.com/containernetworking/plugins/releases/download/v0.7.1/cni-plugins-amd64-v0.7.1.tgz

tar -xf kubernetes-node-linux-amd64.tar.gz -C /usr/local/
mv /usr/local/kubernetes /usr/local/kubernetes-v1.11
ln -s kubernetes-v1.11 /usr/local/kubernetes
mkdir /opt/cni/bin -p
tar -xf cni-plugins-amd64-v0.7.1.tgz -C /opt/cni/bin

配置啓動kubelet

#配置systemctl啓動文件
cat > /etc/systemd/system/kubelet.service << EOF
[Unit]
Description=Kubernetes Kubelet Server
Documentation=https://github.com/kubernetes/kubernetes
After=docker.service
Requires=docker.service

[Service]
EnvironmentFile=-/etc/kubernetes/config
EnvironmentFile=-/etc/kubernetes/kubelet
ExecStart=/usr/local/kubernetes/node/bin/kubelet \\
          \$KUBE_LOGTOSTDERR \\
          \$KUBE_LOG_LEVEL \\
          \$KUBELET_CONFIG \\
          \$KUBELET_HOSTNAME \\
          \$KUBELET_POD_INFRA_CONTAINER \\
          \$KUBELET_ARGS
Restart=on-failure

[Install]
WantedBy=multi-user.target
EOF

cat > /etc/kubernetes/config << EOF
KUBE_LOGTOSTDERR="--logtostderr=true"
KUBE_LOG_LEVEL="--v=2"
EOF

cat > /etc/kubernetes/kubelet << EOF
KUBELET_HOSTNAME="--hostname-override=192.168.1.2"
KUBELET_POD_INFRA_CONTAINER="--pod-infra-container-image=registry.cn-hangzhou.aliyuncs.com/google_containers/pause-amd64:3.1"
KUBELET_CONFIG="--config=/etc/kubernetes/kubelet-config.yml"
KUBELET_ARGS="--bootstrap-kubeconfig=/etc/kubernetes/kubelet-bootstrap.conf --kubeconfig=/etc/kubernetes/kubelet.conf --cert-dir=/etc/kubernetes/pki --network-plugin=cni --cni-bin-dir=/opt/cni/bin --cni-conf-dir=/etc/cni/net.d"
EOF

cat > /etc/kubernetes/kubelet-config.yml << EOF
kind: KubeletConfiguration
apiVersion: kubelet.config.k8s.io/v1beta1
address: 192.168.1.2
port: 10250
cgroupDriver: cgroupfs
clusterDNS:
- 10.96.0.10
clusterDomain: cluster.local.
hairpinMode: promiscuous-bridge
serializeImagePulls: false
authentication:
  x509:
      clientCAFile: /etc/kubernetes/pki/ca.pem
  anonymous:
      enabled: false
  webhook:
      enabled: false
EOF

啓動kubelet

systemctl daemon-reload
systemctl enable kubelet
systemctl restart kubelet
systemctl status kubelet

在node2上操做bootstrap

下載cni包

cd /root
wget https://dl.k8s.io/v1.11.1/kubernetes-node-linux-amd64.tar.gz
wget https://github.com/containernetworking/plugins/releases/download/v0.7.1/cni-plugins-amd64-v0.7.1.tgz

tar -xf kubernetes-node-linux-amd64.tar.gz -C /usr/local/
mv /usr/local/kubernetes /usr/local/kubernetes-v1.11
ln -s kubernetes-v1.11 /usr/local/kubernetes
mkdir /opt/cni/bin -p
tar -xf cni-plugins-amd64-v0.7.1.tgz -C /opt/cni/bin

配置啓動kubelet

#配置systemctl啓動文件

cat > /etc/systemd/system/kubelet.service << EOF
[Unit]
Description=Kubernetes Kubelet Server
Documentation=https://github.com/kubernetes/kubernetes
After=docker.service
Requires=docker.service

[Service]
EnvironmentFile=-/etc/kubernetes/config
EnvironmentFile=-/etc/kubernetes/kubelet
ExecStart=/usr/local/kubernetes/node/bin/kubelet \\
          \$KUBE_LOGTOSTDERR \\
          \$KUBE_LOG_LEVEL \\
          \$KUBELET_CONFIG \\
          \$KUBELET_HOSTNAME \\
          \$KUBELET_POD_INFRA_CONTAINER \\
          \$KUBELET_ARGS
Restart=on-failure

[Install]
WantedBy=multi-user.target
EOF

cat > /etc/kubernetes/config << EOF
KUBE_LOGTOSTDERR="--logtostderr=true"
KUBE_LOG_LEVEL="--v=2"
EOF

cat > /etc/kubernetes/kubelet << EOF
KUBELET_HOSTNAME="--hostname-override=192.168.1.3"
KUBELET_POD_INFRA_CONTAINER="--pod-infra-container-image=registry.cn-hangzhou.aliyuncs.com/google_containers/pause-amd64:3.1"
KUBELET_CONFIG="--config=/etc/kubernetes/kubelet-config.yml"
KUBELET_ARGS="--bootstrap-kubeconfig=/etc/kubernetes/kubelet-bootstrap.conf --kubeconfig=/etc/kubernetes/kubelet.conf --cert-dir=/etc/kubernetes/pki --network-plugin=cni --cni-bin-dir=/opt/cni/bin --cni-conf-dir=/etc/cni/net.d"
EOF

cat > /etc/kubernetes/kubelet-config.yml << EOF
kind: KubeletConfiguration
apiVersion: kubelet.config.k8s.io/v1beta1
address: 192.168.1.3
port: 10250
cgroupDriver: cgroupfs
clusterDNS:
- 10.96.0.10
clusterDomain: cluster.local.
hairpinMode: promiscuous-bridge
serializeImagePulls: false
authentication:
 x509:
     clientCAFile: /etc/kubernetes/pki/ca.pem
 anonymous:
     enabled: false
 webhook:
     enabled: false
EOF

啓動kubelet

systemctl daemon-reload
systemctl enable kubelet
systemctl restart kubelet
systemctl status kubelet

經過證書驗證添加各個節點

#在master節點操做

kubectl get csr

#經過驗證並添加進集羣

kubectl get csr | awk '/node/{print $1}' | xargs kubectl certificate approve

###單獨執行命令例子：
    kubectl certificate approve node-csr-Yiiv675wUCvQl3HH11jDr0cC9p3kbrXWrxvG3EjWGoE

#查看節點
#此時節點狀態爲 NotReady，由於尚未配置網絡

kubectl get nodes

[root@master ~]#kubectl get nodes   
NAME          STATUS     ROLES     AGE       VERSION
192.168.1.1   NotReady   <none>    6s        v1.11.1
192.168.1.2   NotReady   <none>    7s        v1.11.1
192.168.1.3   NotReady   <none>    7s        v1.11.1

# 在node節點查看生成的文件

ls -l /etc/kubernetes/kubelet.conf
ls -l /etc/kubernetes/pki/kubelet*

6、配置kube-proxy

- 全部節點都要配置kube-proxy!!!

在master節點操做

安裝conntrack-tools

yum install -y conntrack-tools

啓動文件

cat > /etc/systemd/system/kube-proxy.service << EOF
[Unit]
Description=Kubernetes Kube-Proxy Server
Documentation=https://github.com/kubernetes/kubernetes
After=network.target

[Service]
EnvironmentFile=-/etc/kubernetes/config
EnvironmentFile=-/etc/kubernetes/proxy
ExecStart=/usr/local/kubernetes/server/bin/kube-proxy \\
      \$KUBE_LOGTOSTDERR \\
      \$KUBE_LOG_LEVEL \\
      \$KUBECONFIG \\
      \$KUBE_PROXY_ARGS
Restart=on-failure
LimitNOFILE=65536

[Install]
WantedBy=multi-user.target
EOF

#啓用ipvs主要就是把kube-proxy的--proxy-mode配置選項修改成ipvs
#而且要啓用--masquerade-all，使用iptables輔助ipvs運行

cat > /etc/kubernetes/proxy << EOF
KUBECONFIG="--kubeconfig=/etc/kubernetes/kube-proxy.conf"
KUBE_PROXY_ARGS="--proxy-mode=ipvs  --masquerade-all=true --cluster-cidr=10.0.0.0/8"
EOF

啓動

systemctl daemon-reload
systemctl enable kube-proxy
systemctl restart kube-proxy
systemctl status kube-proxy

在全部的node上操做

安裝

yum install -y conntrack-tools

啓動文件

cat > /etc/systemd/system/kube-proxy.service << EOF
[Unit]
Description=Kubernetes Kube-Proxy Server
Documentation=https://github.com/kubernetes/kubernetes
After=network.target

[Service]
EnvironmentFile=-/etc/kubernetes/config
EnvironmentFile=-/etc/kubernetes/proxy
ExecStart=/usr/local/kubernetes/node/bin/kube-proxy \\
      \$KUBE_LOGTOSTDERR \\
      \$KUBE_LOG_LEVEL \\
      \$KUBECONFIG \\
      \$KUBE_PROXY_ARGS
Restart=on-failure
LimitNOFILE=65536

[Install]
WantedBy=multi-user.target
EOF

#啓用ipvs主要就是把kube-proxy的--proxy-mode配置選項修改成ipvs
#而且要啓用--masquerade-all，使用iptables輔助ipvs運行

cat > /etc/kubernetes/proxy << EOF
KUBECONFIG="--kubeconfig=/etc/kubernetes/kube-proxy.conf"
KUBE_PROXY_ARGS="--proxy-mode=ipvs --masquerade-all=true --cluster-cidr=10.0.0.0/8"
EOF

啓動

systemctl daemon-reload
systemctl enable kube-proxy
systemctl restart kube-proxy
systemctl status kube-proxy

7、設置集羣角色

在master節點操做

設置 192.168.1.1 爲 master

kubectl label nodes 192.168.1.1 node-role.kubernetes.io/master=

設置 192.168.1.2 - 3 爲 node

kubectl label nodes 192.168.1.2 node-role.kubernetes.io/node=
kubectl label nodes 192.168.1.3 node-role.kubernetes.io/node=

設置 master 通常狀況下不接受負載

kubectl taint nodes 192.168.1.1 node-role.kubernetes.io/master=true:NoSchedule

查看節點

#此時節點狀態爲 NotReady
#ROLES已經標識出了master和node

kubectl get node

NAME          STATUS     ROLES     AGE       VERSION
192.168.1.1   NotReady   master    1m        v1.11.1
192.168.1.2   NotReady   node      1m        v1.11.1
192.168.1.3   NotReady   node      1m        v1.11.1

8、配置網絡

如下網絡二選一：

使用flannel網絡

cd /root/
mkdir flannel
cd flannel
wget https://raw.githubusercontent.com/coreos/flannel/v0.10.0/Documentation/kube-flannel.yml

sed -ri 's#("Network": ")10.244.0.0/16#\110.0.0.0/8#' kube-flannel.yml
#修改kube-flannel文件中的網段爲咱們須要的網段

kubectl apply -f .

使用canal網絡

cd /root/
mkdir canal
cd canal

wget https://docs.projectcalico.org/v3.1/getting-started/kubernetes/installation/hosted/canal/rbac.yaml
wget https://docs.projectcalico.org/v3.1/getting-started/kubernetes/installation/hosted/canal/canal.yaml

sed -ri 's#("Network": ")10.244.0.0/16#\110.0.0.0/8#' canal.yaml
#修改cannl文件中的網段爲咱們須要的網段

kubectl apply -f .

查看網絡容器是否爲running狀態

kubectl get -n kube-system pod -o wide  

[root@master ~]# kubectl get -n kube-system pod -o wide     
NAME          READY     STATUS    RESTARTS   AGE       IP            NODE
canal-74zhp   3/3       Running   0          7m        192.168.1.3   192.168.1.3
canal-cmz2p   3/3       Running   0          7m        192.168.1.1   192.168.1.1
canal-mkcg2   3/3       Running   0          7m        192.168.1.2   192.168.1.2

查看各個節點是否爲Ready狀態

kubectl get node 

[root@master ~]# 
NAME          STATUS    ROLES     AGE       VERSION
192.168.1.1   Ready     master    5h        v1.11.1
192.168.1.2   Ready     node      5h        v1.11.1
192.168.1.3   Ready     node      5h        v1.11.1

9、配置使用coredns

#10.96.0.10 是kubelet中配置的dns
#安裝coredns

cd /root && mkdir coredns && cd coredns
wget https://raw.githubusercontent.com/coredns/deployment/master/kubernetes/coredns.yaml.sed
wget https://raw.githubusercontent.com/coredns/deployment/master/kubernetes/deploy.sh
chmod +x deploy.sh
./deploy.sh -i 10.96.0.10 > coredns.yml
kubectl apply -f coredns.yml

#查看

kubectl get svc,pods -n kube-system

[root@master coredns]# kubectl get svc,pods -n kube-system
NAME               TYPE        CLUSTER-IP   EXTERNAL-IP   PORT(S)         AGE
service/kube-dns   ClusterIP   10.96.0.10   <none>        53/UDP,53/TCP   2m

NAME                           READY     STATUS    RESTARTS   AGE
pod/canal-5wkkd                3/3       Running   0          17h
pod/canal-6mhhz                3/3       Running   0          17h
pod/canal-k7ccs                3/3       Running   2          17h
pod/coredns-6975654877-jpqg4   1/1       Running   0          2m
pod/coredns-6975654877-lgz9n   1/1       Running   0          2m

10、測試

建立一個nginx 應用，測試應用和dns是否正常

cd /root && mkdir nginx && cd nginx

cat << EOF > nginx.yaml
---
apiVersion: v1
kind: Service
metadata:
  name: nginx
spec:
  selector:
    app: nginx
  type: NodePort
  ports:
  - port: 80
    nodePort: 31000
    name: nginx-port
    targetPort: 80
    protocol: TCP

---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: nginx
spec:
  replicas: 2
  selector:
    matchLabels:
      app: nginx
  template:
    metadata:
      name: nginx
      labels:
        app: nginx
    spec:
      containers:
      - name: nginx
        image: nginx
        ports:
        - containerPort: 80
EOF

建立一個pod用來測試dns

kubectl run curl --image=radial/busyboxplus:curl -i --tty
nslookup kubernetes
nslookup nginx
curl nginx
exit

[ root@curl-87b54756-qf7l9:/ ]$ nslookup kubernetes
Server:    10.96.0.10
Address 1: 10.96.0.10 kube-dns.kube-system.svc.cluster.local

Name:      kubernetes
Address 1: 10.96.0.1 kubernetes.default.svc.cluster.local
[ root@curl-87b54756-qf7l9:/ ]$ nslookup nginx
Server:    10.96.0.10
Address 1: 10.96.0.10 kube-dns.kube-system.svc.cluster.local

Name:      nginx
Address 1: 10.105.93.85 nginx.default.svc.cluster.local
[ root@curl-87b54756-qf7l9:/ ]$ curl nginx
<!DOCTYPE html>
<html>
<head>
<title>Welcome to nginx!</title>
<style>
    body {
        width: 35em;
        margin: 0 auto;
        font-family: Tahoma, Verdana, Arial, sans-serif;
    }
</style>
</head>
...

[ root@curl-87b54756-qf7l9:/ ]$ exit
Session ended, resume using 'kubectl attach curl-87b54756-qf7l9 -c curl -i -t' command when the pod is running

到etcd節點上執行curl nodeIp:31000 測試集羣外部是否能訪問nginx

curl 192.168.1.2:31000

[root@node5 ~]# curl 192.168.1.2:31000
<!DOCTYPE html>
<html>
<head>
<title>Welcome to nginx!</title>
<style>
    body {
        width: 35em;
        margin: 0 auto;
        font-family: Tahoma, Verdana, Arial, sans-serif;
    }
</style>
</head>
<body>
<h1>Welcome to nginx!</h1>
<p>If you see this page, the nginx web server is successfully installed and
working. Further configuration is required.</p>

<p>For online documentation and support please refer to
<a href="http://nginx.org/">nginx.org</a>.<br/>
Commercial support is available at
<a href="http://nginx.com/">nginx.com</a>.</p>

<p><em>Thank you for using nginx.</em></p>
</body>
</html>

安裝ipvsadm查看ipvs規則

yum install -y ipvsadm

ipvsadm

[root@master ~]# ipvsadm
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
  -> RemoteAddress:Port           Forward Weight ActiveConn InActConn
TCP  master:31000 rr
  -> 10.0.0.8:http                Masq    1      0          0
  -> 10.0.1.9:http                Masq    1      0          0
TCP  master:31000 rr
  -> 10.0.0.8:http                Masq    1      0          0
  -> 10.0.1.9:http                Masq    1      0          0
TCP  master:31000 rr
  -> 10.0.0.8:http                Masq    1      0          0
  -> 10.0.1.9:http                Masq    1      0          0
TCP  master:https rr
  -> master:sun-sr-https          Masq    1      2          0
TCP  master:domain rr
  -> 10.0.0.3:domain              Masq    1      0          0
  -> 10.0.1.3:domain              Masq    1      0          0
TCP  master:http rr
  -> 10.0.0.8:http                Masq    1      0          0
  -> 10.0.1.9:http                Masq    1      0          0
TCP  localhost:31000 rr
  -> 10.0.0.8:http                Masq    1      0          0
  -> 10.0.1.9:http                Masq    1      0          0
UDP  master:domain rr
  -> 10.0.0.3:domain              Masq    1      0          0
  -> 10.0.1.3:domain              Masq    1      0          0