28-2kubernenets
5.持久化存儲
5.1pv,pvc
k8s中的副本控制器保證了pod的始終存儲，卻保證不了pod中的數據。只有啓動一個新pod的，以前pod中的數據會隨着容器的刪掉而丟失！ pv和pvc的概念: PersistentVolume（一些簡稱PV）：由管理員添加的的一個存儲的描述，是一個全局資源，包含存儲的類型，存儲的大小和訪問模式等。它的生命週期獨立於Pod，例如當使用它的Pod銷燬時對PV沒有影響。 PersistentVolumeClaim（一些簡稱PVC）：是Namespace裏的資源，描述對PV的一個請求。請求信息包含存儲大小，訪問模式等。
#1.全部節點安裝nfs-utils
[root@k8s-master ~]# yum install -y nfs-utils
#2.master配置nfs服務端
[root@k8s-master ~]#yum install -y rpcbind
[root@k8s-master ~]# vim /etc/exports
[root@k8s-master ~]# systemctl restart nfs
[root@k8s-master ~]# cat /etc/exports
/data 10.0.0.0/24(rw,async,no_root_squash,no_all_squash)
[root@k8s-master ~]#
建立掛載目錄
[root@k8s-master volume]# mkdir -p /data/k8s
#3.node節點查看nfs掛載目錄
[root@k8s-node1 ~]# showmount -e 10.0.0.11
Export list for 10.0.0.11:
/data 10.0.0.0/24
[root@k8s-node2 ~]# showmount -e 10.0.0.11
Export list for 10.0.0.11:
/data 10.0.0.0/24
[root@k8s-node2 ~]#
#啓動nfs服務
全部節點systemctl start nfs
master節點
systemctl start rpcbind
#4.建立PV
## 4.1準備yml
[root@k8s-master volume]# cat test-py.yml
apiVersion: v1
kind: PersistentVolume
metadata:
name: test
labels:
type: test
spec:
capacity:
storage: 10Gi
accessModes:
- ReadWriteMany
persistentVolumeReclaimPolicy: Recycle
nfs:
path: "/data/k8s"
server: 10.0.0.11
readOnly: false
[root@k8s-master volume]# cat test-py2.yml
apiVersion: v1
kind: PersistentVolume
metadata:
name: test2
labels:
type: test
spec:
capacity:
storage: 5Gi
accessModes:
- ReadWriteMany
persistentVolumeReclaimPolicy: Recycle
nfs:
path: "/data/k8s"
server: 10.0.0.11
readOnly: false
##4.2建立pv
[root@k8s-master volume]# kubectl create -f test-py.yml
persistentvolume "test" created
[root@k8s-master volume]# kubectl create -f test-py2.yml
persistentvolume "test2" created
##4.3查看pv
[root@k8s-master volume]# kubectl get pv
NAME CAPACITY ACCESSMODES RECLAIMPOLICY STATUS CLAIM REASON AGE
test 10Gi RWX Recycle Available 2m
test2 5Gi RWX Recycle Available 1m
[root@k8s-master volume]#
#5.建立pvc
##5.1準備yml
[root@k8s-master volume]# cat test-pvc.yml
kind: PersistentVolumeClaim
apiVersion: v1
metadata:
name: nfs
spec:
accessModes:
- ReadWriteMany
resources:
requests:
storage: 1Gi
[root@k8s-master volume]# kubectl create test-pvc.yml
##5.2建立pvc
[root@k8s-master volume]# kubectl create -f test-pvc.yml
persistentvolumeclaim "nfs" created
[root@k8s-master volume]#
##5.3查看，綁定在了test2這個pv上，選擇容量夠本身使用的pv
[root@k8s-master volume]# kubectl get pvc
NAME STATUS VOLUME CAPACITY ACCESSMODES AGE
nfs Bound test2 5Gi RWX 30s
[root@k8s-master volume]#
5.2持久化實戰
#1.刪除以前建立的內容
[root@k8s-master volume]# kubectl get all
NAME REFERENCE TARGET CURRENT MINPODS MAXPODS AGE
hpa/myweb ReplicationController/myweb 10% 0% 1 8 1h
NAME DESIRED CURRENT READY AGE
rc/myweb 1 1 1 1h
NAME CLUSTER-IP EXTERNAL-IP PORT(S) AGE
svc/kubernetes 10.254.0.1 <none> 443/TCP 23h
NAME READY STATUS RESTARTS AGE
po/myweb-9khsv 1/1 Running 0 1h
[root@k8s-master volume]# kubectl delete hpa myweb
horizontalpodautoscaler "myweb" deleted
[root@k8s-master volume]# kubectl delete rc myweb
replicationcontroller "myweb" deleted
[root@k8s-master volume]# kubectl get all
NAME CLUSTER-IP EXTERNAL-IP PORT(S) AGE
svc/kubernetes 10.254.0.1 <none> 443/TCP 23h
[root@k8s-master volume]#
#2.tomcat-mysql實驗的內容從新賦值一份，建立Pod
因爲構建了nfs服務，因此tomcat-rc中 name: MYSQL_SERVICE_HOST value: 'mysql' 能夠直接使用svc的name。
[root@k8s-master k8s]# cp -r tomcat_demo tomcat_demo2
[root@k8s-master k8s]# cd tomcat_demo2/
[root@k8s-master tomcat_demo2]# ll
total 16
-rw-r--r-- 1 root root 416 Aug 29 13:42 mysql-rc.yml
-rw-r--r-- 1 root root 145 Aug 29 13:42 mysql-svc.yml
-rw-r--r-- 1 root root 483 Aug 29 13:42 tomcat-rc.yml
-rw-r--r-- 1 root root 162 Aug 29 13:42 tomcat-svc.yml
[root@k8s-master tomcat_demo2]# vim tomcat-rc.yml
[root@k8s-master tomcat_demo2]# kubectl create -f .
replicationcontroller "mysql" created
service "mysql" created
replicationcontroller "myweb" created
service "myweb" created
[root@k8s-master tomcat_demo2]# kubectl get all
NAME DESIRED CURRENT READY AGE
rc/mysql 1 1 1 7s
rc/myweb 1 1 1 6s
NAME CLUSTER-IP EXTERNAL-IP PORT(S) AGE
svc/kubernetes 10.254.0.1 <none> 443/TCP 23h
svc/mysql 10.254.81.140 <none> 3306/TCP 7s
svc/myweb 10.254.206.74 <nodes> 8080:30008/TCP 6s
NAME READY STATUS RESTARTS AGE
po/mysql-nq9r3 1/1 Running 0 7s
po/myweb-n8h93 1/1 Running 0 6s
[root@k8s-master tomcat_demo2]#
#沒有作持久化時，刪除Pod，數據就沒有了，由於pod會從新建立
#因爲本項目只有數據庫中保存數據，因此只對mysql作持久化
#3.建立pv,pvc
##3.1準備yml
[root@k8s-master tomcat_demo2]# mkdir -p /data/mysql
[root@k8s-master tomcat_demo2]# cat mysql-pv.yml
apiVersion: v1
kind: PersistentVolume
metadata:
name: mysql
labels:
type: mysql
spec:
capacity:
storage: 10Gi
accessModes:
- ReadWriteMany
persistentVolumeReclaimPolicy: Recycle
nfs:
path: "/data/mysql"
server: 10.0.0.11
readOnly: false
[root@k8s-master tomcat_demo2]
[root@k8s-master tomcat_demo2]# cat mysql-pvc.yml
kind: PersistentVolumeClaim
apiVersion: v1
metadata:
name: mysql
spec:
accessModes:
- ReadWriteMany
resources:
requests:
storage: 9Gi
##3.2建立pv和PVC
[root@k8s-master tomcat_demo2]# kubectl create -f mysql-pv.yml
persistentvolume "mysql" created
[root@k8s-master tomcat_demo2]# kubectl create -f mysql-pvc.yml
persistentvolumeclaim "mysql" created
確保mysql pvc綁定在mysql pv上
[root@k8s-master tomcat_demo2]# kubectl get pv
NAME CAPACITY ACCESSMODES RECLAIMPOLICY STATUS CLAIM REASON AGE
mysql 10Gi RWX Recycle Bound default/mysql 26m
test2 5Gi RWX Recycle Bound default/nfs 46m
[root@k8s-master tomcat_demo2]# kubectl get pvc
NAME STATUS VOLUME CAPACITY ACCESSMODES AGE
mysql Bound mysql 10Gi RWX 6m
nfs Bound test2 5Gi RWX 42m
[root@k8s-master tomcat_demo2]#
##3.3修改MySQL的rc.yml文件
[root@k8s-master tomcat_demo2]# cp mysql-rc.yml mysql-rc-pvc.yml
[root@k8s-master tomcat_demo2]# cat mysql-rc-pvc.yml
apiVersion: v1
kind: ReplicationController
metadata:
name: mysql
spec:
replicas: 1
selector:
app: mysql
template:
metadata:
labels:
app: mysql
spec:
containers:
- name: mysql
image: 10.0.0.11:5000/mysql:5.7
ports:
- containerPort: 3306
env:
- name: MYSQL_ROOT_PASSWORD
value: '123456'
volumeMounts:
- name: data
mountPath: /var/lib/mysql
volumes:
- name: data
persistentVolumeClaim:
claimName: mysql
[root@k8s-master tomcat_demo2]#
##3.4應用
[root@k8s-master tomcat_demo2]# kubectl apply -f mysql-rc-pvc.yml
replicationcontroller "mysql" configured
##3.5刪除現有的pod，自動新建的mysql pod會應用pvc
[root@k8s-master tomcat_demo2]# kubectl get all
NAME DESIRED CURRENT READY AGE
rc/mysql 1 1 1 15m
rc/myweb 1 1 1 15m
NAME CLUSTER-IP EXTERNAL-IP PORT(S) AGE
svc/kubernetes 10.254.0.1 <none> 443/TCP 23h
svc/mysql 10.254.81.140 <none> 3306/TCP 15m
svc/myweb 10.254.206.74 <nodes> 8080:30008/TCP 15m
NAME READY STATUS RESTARTS AGE
po/mysql-nq9r3 1/1 Running 0 15m
po/myweb-n8h93 1/1 Running 0 15m
[root@k8s-master tomcat_demo2]# kubectl delete pod mysql-nq9r3
pod "mysql-nq9r3" deleted
[root@k8s-master tomcat_demo2]#
[root@k8s-master tomcat_demo2]# kubectl get all -o wide
NAME DESIRED CURRENT READY AGE CONTAINER(S) IMAGE(S) SELECTOR
rc/mysql 1 1 1 11m mysql 10.0.0.11:5000/mysql:5.7 app=mysql
rc/myweb 1 1 1 36m myweb 10.0.0.11:5000/tomcat-app:v2 app=myweb
NAME CLUSTER-IP EXTERNAL-IP PORT(S) AGE SELECTOR
svc/kubernetes 10.254.0.1 <none> 443/TCP 1d <none>
svc/mysql 10.254.81.140 <none> 3306/TCP 36m app=mysql
svc/myweb 10.254.206.74 <nodes> 8080:30008/TCP 36m app=myweb
NAME READY STATUS RESTARTS AGE IP NODE
po/mysql-tr6bc 1/1 Running 0 5m 172.16.48.3 k8s-node2
po/myweb-n8h93 1/1 Running 0 36m 172.16.79.2 k8s-node1
##3.6master上查看/data/mysql目錄下是否有內容
##注意，這裏若是看不到數據，可能有幾種緣由
1.你的mysql pvc沒有綁定在mysql pv上
2.nfs服務是否運行正常,master端運行nfs和rpcbind服務，node端運行nfs服務
[root@k8s-master tomcat_demo2]# ll /data/mysql
total 188484
-rw-r----- 1 polkitd ssh_keys 56 Aug 29 14:14 auto.cnf
-rw------- 1 polkitd ssh_keys 1675 Aug 29 14:14 ca-key.pem
-rw-r--r-- 1 polkitd ssh_keys 1107 Aug 29 14:14 ca.pem
-rw-r--r-- 1 polkitd ssh_keys 1107 Aug 29 14:14 client-cert.pem
-rw------- 1 polkitd ssh_keys 1679 Aug 29 14:14 client-key.pem
drwxr-x--- 2 polkitd ssh_keys 58 Aug 29 14:14 HPE_APP
-rw-r----- 1 polkitd ssh_keys 699 Aug 29 14:20 ib_buffer_pool
-rw-r----- 1 polkitd ssh_keys 79691776 Aug 29 14:20 ibdata1
-rw-r----- 1 polkitd ssh_keys 50331648 Aug 29 14:20 ib_logfile0
-rw-r----- 1 polkitd ssh_keys 50331648 Aug 29 14:14 ib_logfile1
-rw-r----- 1 polkitd ssh_keys 12582912 Aug 29 14:20 ibtmp1
drwxr-x--- 2 polkitd ssh_keys 4096 Aug 29 14:14 mysql
drwxr-x--- 2 polkitd ssh_keys 8192 Aug 29 14:14 performance_schema
-rw------- 1 polkitd ssh_keys 1675 Aug 29 14:14 private_key.pem
-rw-r--r-- 1 polkitd ssh_keys 451 Aug 29 14:14 public_key.pem
-rw-r--r-- 1 polkitd ssh_keys 1107 Aug 29 14:14 server-cert.pem
-rw------- 1 polkitd ssh_keys 1675 Aug 29 14:14 server-key.pem
drwxr-x--- 2 polkitd ssh_keys 8192 Aug 29 14:14 sys
[root@k8s-master tomcat_demo2]#
##3.7因爲MySQL在node2節點上，因此在node2節點上查看掛載狀況
[root@k8s-node2 ~]# df -h
Filesystem Size Used Avail Use% Mounted on
10.0.0.11:/data/mysql 98G 12G 87G 12% /var/lib/kubelet/pods/4539b360-ca24-11e9-90d0-000c29e15b21/volumes/kubernetes.io~nfs/mysql
overlay 98G 3.4G 95G 4% /var/lib/docker/overlay2/5e84e1093fab4dce4ce314f8ce6cfbfb35e3af6428be837a2c40f617ab627561/merged
shm 64M 0 64M 0% /var/lib/docker/containers/3324fee122006e54388c132976752343824fb7a91a438a0c5344e5fcdc0e917e/shm
overlay 98G 3.4G 95G 4% /var/lib/docker/overlay2/0be4a6ee94804a6d0b8ff5f90874a11ec25ef02bdd6e878f901d5c671be75530/merged
[root@k8s-node2 ~]#
##3.8測試刪除Pod，新加的數據仍然存在
[root@k8s-master tomcat_demo2]# kubectl delete pod mysql-wz4ds
pod "mysql-wz4ds" deleted
5.3分佈式文件系統glusterfs
nfs服務不方便擴容和管理，生產環境更多使用glusterfs Glusterfs是一個開源分佈式文件系統，具備強大的橫向擴展能力，可支持數PB存儲容量和數千客戶端，經過網絡互聯成一個並行的網絡文件系統。 具備可擴展性、高性能、高可用性等特色。
#1.安裝glusterfs
全部節點：
yum install centos-release-gluster -y
yum install install glusterfs-server -y
systemctl start glusterd.service
systemctl enable glusterd.service
mkdir -p /gfs/test1
mkdir -p /gfs/test2
#2.master節點添加存儲資源池
detach是刪除節點
probe是添加節點
[root@k8s-master tomcat_demo2]# gluster pool list
UUID Hostname State
e3966f16-295f-4dd4-99db-0facec4ad990 localhost Connected
[root@k8s-master tomcat_demo2]# gluster peer probe k8s-node1
peer probe: success.
[root@k8s-master tomcat_demo2]# gluster peer probe k8s-node2
peer probe: success.
[root@k8s-master tomcat_demo2]# gluster pool list
UUID Hostname State
04afc91b-5c3d-4a07-a86f-e59288ae7ed2 k8s-node1 Connected
234ac958-4da1-4ae1-ab72-df825026465d k8s-node2 Connected
e3966f16-295f-4dd4-99db-0facec4ad990 localhost Connected
[root@k8s-master tomcat_demo2]#
#3.glusterfs卷管理
##3.1建立分佈式複製卷使用最多，最穩定
因爲複製-分佈式，因此replica 2,最少須要4個目錄，因此使用了master和node1上各兩個
force是因爲/gfs/test1目錄再root分區下，建議在其餘分區更加安全，因此不加force會建立失敗。我這裏就強制建立了
[root@k8s-master tomcat_demo2]#gluster volume create qiangge replica 2 k8s-master:/gfs/test1 k8s-master:/gfs/test2 k8s-node1:/gfs/test1 k8s-node1:/gfs/test2 force
##3.2啓動卷
[root@k8s-master tomcat_demo2]# gluster volume start qiangge
volume start: qiangge: success
##3.3查看卷
[root@k8s-master tomcat_demo2]# gluster volume info qiangge
Volume Name: qiangge
Type: Distributed-Replicate
Volume ID: 4ab2f3fb-ffcb-4213-8455-2f9ddf7b443a
Status: Started
Snapshot Count: 0
Number of Bricks: 2 x 2 = 4
Transport-type: tcp
Bricks:
Brick1: k8s-master:/gfs/test1
Brick2: k8s-master:/gfs/test2
Brick3: k8s-node1:/gfs/test1
Brick4: k8s-node1:/gfs/test2
Options Reconfigured:
transport.address-family: inet
nfs.disable: on
performance.client-io-threads: off
[root@k8s-master tomcat_demo2]#
##3.4掛載卷
mount -t glusterfs
10.0.0.11:任意一個節點
/qiangge 選擇卷
/mnt 選擇掛載點
[root@k8s-master tomcat_demo2]# mount -t glusterfs 10.0.0.11:/qiangge /mnt
容量爲98G
[root@k8s-master tomcat_demo2]# df -h
10.0.0.11:/qiangge 98G 12G 87G 12% /mnt
##3.5分佈式複製卷擴容
[root@k8s-master tomcat_demo2]# gluster volume add-brick qiangge k8s-node2:/gfs/test1 k8s-node2:/gfs/test2 force
volume add-brick: success
##3.6擴容後查看
[root@k8s-master tomcat_demo2]# gluster volume info qiangge
Volume Name: qiangge
Type: Distributed-Replicate
Volume ID: 4ab2f3fb-ffcb-4213-8455-2f9ddf7b443a
Status: Started
Snapshot Count: 0
Number of Bricks: 3 x 2 = 6
Transport-type: tcp
Bricks:
Brick1: k8s-master:/gfs/test1
Brick2: k8s-master:/gfs/test2
Brick3: k8s-node1:/gfs/test1
Brick4: k8s-node1:/gfs/test2
Brick5: k8s-node2:/gfs/test1
Brick6: k8s-node2:/gfs/test2
Options Reconfigured:
transport.address-family: inet
nfs.disable: on
performance.client-io-threads: off
[root@k8s-master tomcat_demo2]#
容量變爲了147G
[root@k8s-master tomcat_demo2]# df -h
10.0.0.11:/qiangge 147G 14G 134G 10% /mnt
##3.7往qiangge卷中寫入數據
[root@k8s-master tomcat_demo2]# cd /mnt/
[root@k8s-master mnt]# cp /opt/k8s/jenkins-k8s/xiaoniaofeifei.zip .
[root@k8s-master mnt]# ll
total 89
-rw-r--r-- 1 root root 91014 Aug 29 15:02 xiaoniaofeifei.zip
[root@k8s-master mnt]# unzip xiaoniaofeifei.zip
Archive: xiaoniaofeifei.zip
inflating: sound1.mp3
creating: img/
inflating: img/bg1.jpg
inflating: img/bg2.jpg
inflating: img/number1.png
inflating: img/number2.png
inflating: img/s1.png
inflating: img/s2.png
inflating: 21.js
inflating: 2000.png
inflating: icon.png
inflating: index.html
#master節點上查看
[root@k8s-master mnt]# tree /gfs/
/gfs/
├── test1
│ ├── 2000.png
│ ├── 21.js
│ ├── icon.png
│ ├── img
│ │ ├── number1.png
│ │ ├── number2.png
│ │ └── s1.png
│ ├── index.html
│ ├── sound1.mp3
│ └── xiaoniaofeifei.zip
└── test2
├── 2000.png
├── 21.js
├── icon.png
├── img
│ ├── number1.png
│ ├── number2.png
│ └── s1.png
├── index.html
├── sound1.mp3
└── xiaoniaofeifei.zip
4 directories, 18 files
[root@k8s-master mnt]# ^C
[root@k8s-master mnt]#
#node1節點上查看
[root@k8s-node1 ~]# tree /gfs/
/gfs/
├── test1
│ └── img
│ ├── bg1.jpg
│ └── s2.png
└── test2
└── img
├── bg1.jpg
└── s2.png
4 directories, 4 files
[root@k8s-node1 ~]#
#node2節點上查看
[root@k8s-node2 ~]# tree /gfs/
/gfs/
├── test1
│ └── img
│ └── bg2.jpg
└── test2
└── img
└── bg2.jpg
4 directories, 2 files
[root@k8s-node2 ~]#
5.4k8s對接glusterfs
#1.建立endpoint
[root@k8s-master glusterfs]#vi glusterfs-ep.yaml
apiVersion: v1
kind: Endpoints
metadata:
name: glusterfs
namespace: default
subsets:
- addresses:
- ip: 10.0.0.11
- ip: 10.0.0.12
- ip: 10.0.0.13
ports:
- port: 49152
protocol: TCP
[root@k8s-master glusterfs]# kubectl create -f glusterfs-ep.yaml
endpoints "glusterfs" created
[root@k8s-master glusterfs]# kubectl get ep
NAME ENDPOINTS AGE
glusterfs 10.0.0.11:49152,10.0.0.12:49152,10.0.0.13:49152 12s
kubernetes 10.0.0.11:6443 1d
mysql 172.16.48.3:3306 1h
myweb 172.16.79.2:8080 1h
[root@k8s-master glusterfs]#
#2.建立service
service與endpoint是經過name關聯的，name要相同
[root@k8s-master glusterfs]# cat glusterfs-svc.yaml
apiVersion: v1
kind: Service
metadata:
name: glusterfs
namespace: default
spec:
ports:
- port: 49152
protocol: TCP
targetPort: 49152
sessionAffinity: None
type: ClusterIP
[root@k8s-master glusterfs]#
[root@k8s-master glusterfs]# kubectl create -f glusterfs-svc.yaml
service "glusterfs" created
[root@k8s-master glusterfs]# kubectl get svc
NAME CLUSTER-IP EXTERNAL-IP PORT(S) AGE
glusterfs 10.254.192.201 <none> 49152/TCP 9s
kubernetes 10.254.0.1 <none> 443/TCP 1d
mysql 10.254.81.140 <none> 3306/TCP 1h
myweb 10.254.206.74 <nodes> 8080:30008/TCP 1h
[root@k8s-master glusterfs]#
#3.查看service與endpoint是否關聯上
[root@k8s-master glusterfs]# kubectl describe svc glusterfs
Name: glusterfs
Namespace: default
Labels: <none>
Selector: <none>
Type: ClusterIP
IP: 10.254.192.201
Port: <unset> 49152/TCP
Endpoints: 10.0.0.11:49152,10.0.0.12:49152,10.0.0.13:49152
Session Affinity: None
No events.
[root@k8s-master glusterfs]#
#4.建立gluster類型pv
path: "qiangge"就是卷的名字，在搭建glusterfs的時候建立的
[root@k8s-master glusterfs]# cat gluster-pv.yml
apiVersion: v1
kind: PersistentVolume
metadata:
name: gluster
labels:
type: glusterfs
spec:
capacity:
storage: 50Gi
accessModes:
- ReadWriteMany
glusterfs:
endpoints: "glusterfs"
path: "qiangge"
readOnly: false
[root@k8s-master glusterfs]#
[root@k8s-master glusterfs]# kubectl create -f gluster-pv.yml
persistentvolume "gluster" created
[root@k8s-master glusterfs]# kubectl get pv
NAME CAPACITY ACCESSMODES RECLAIMPOLICY STATUS CLAIM REASON AGE
gluster 50Gi RWX Retain Available 10s
mysql 10Gi RWX Recycle Bound default/mysql 1h
test2 5Gi RWX Recycle Bound default/nfs 1h
[root@k8s-master glusterfs]#
#5.建立pvc
[root@k8s-master glusterfs]# cat gluster-pvc.yml
kind: PersistentVolumeClaim
apiVersion: v1
metadata:
name: gluster
spec:
accessModes:
- ReadWriteMany
resources:
requests:
storage: 50Gi
[root@k8s-master glusterfs]# kubectl create -f gluster-pvc.yml
persistentvolumeclaim "gluster" created
[root@k8s-master glusterfs]# kubectl get pvc
NAME STATUS VOLUME CAPACITY ACCESSMODES AGE
gluster Bound gluster 50Gi RWX 7s
mysql Bound mysql 10Gi RWX 1h
nfs Bound test2 5Gi RWX 1h
[root@k8s-master glusterfs]#
#6.建立pod
[root@k8s-master glusterfs]# cat nginx_pod.yaml
apiVersion: v1
kind: Pod
metadata:
name: nginx
labels:
app: web
spec:
containers:
- name: nginx
image: 10.0.0.11:5000/nginx:1.13
ports:
- containerPort: 80
hostPort: 80
volumeMounts:
- name: nfs-vol2
mountPath: /usr/share/nginx/html
volumes:
- name: nfs-vol2
persistentVolumeClaim:
claimName: gluster
[root@k8s-master glusterfs]#
[root@k8s-master glusterfs]# kubectl create -f nginx_pod.yaml
pod "nginx" created
#查看在node2節點上
[root@k8s-master glusterfs]# kubectl get all -o wide
NAME DESIRED CURRENT READY AGE CONTAINER(S) IMAGE(S) SELECTOR
rc/mysql 1 1 1 1h mysql 10.0.0.11:5000/mysql:5.7 app=mysql
rc/myweb 1 1 1 1h myweb 10.0.0.11:5000/tomcat-app:v2 app=myweb
NAME CLUSTER-IP EXTERNAL-IP PORT(S) AGE SELECTOR
svc/glusterfs 10.254.192.201 <none> 49152/TCP 26m <none>
svc/kubernetes 10.254.0.1 <none> 443/TCP 1d <none>
svc/mysql 10.254.81.140 <none> 3306/TCP 1h app=mysql
svc/myweb 10.254.206.74 <nodes> 8080:30008/TCP 1h app=myweb
NAME READY STATUS RESTARTS AGE IP NODE
po/mysql-snhd1 1/1 Running 1 1h 172.16.48.3 k8s-node2
po/myweb-n8h93 1/1 Running 0 1h 172.16.79.2 k8s-node1
po/nginx 1/1 Running 0 1m 172.16.48.2 k8s-node2
6.k8s的持續發佈
6.1安裝gitlab
內存至少要2G呀，要不卡到502呀！ #1.node2安裝gitlab [root@k8s-node2 jenkins-k8s]#wget https://mirror.tuna.tsinghua.edu.cn/gitlab-ce/yum/el7/gitlab-ce-11.9.11-ce.0.el7.x86_64.rpm [root@k8s-node2 jenkins-k8s]#yum localinstall gitlab-ce-11.9.11-ce.0.el7.x86_64.rpm -y 配置gitlab [root@k8s-node2 jenkins-k8s]#vim /etc/gitlab/gitlab.rb 13行 external_url 'http://10.0.0.13' 1535行 prometheus_monitoring['enable'] = false 啓動gitlab [root@k8s-node2 jenkins-k8s]#gitlab-ctl reconfigure 訪問gitlab http://10.0.0.13 密碼爲12345678
6.2gitlab建立倉庫並上傳代碼
#2.gitlab建立倉庫並上傳代碼 ##2.1準備文件 [root@k8s-node2 jenkins-k8s]# mkdir /opt/xiaoniao [root@k8s-node2 jenkins-k8s]# cd /opt/xiaoniao/ [root@k8s-node2 xiaoniao]# cp /opt/jenkins-k8s/xiaoniaofeifei.zip . [root@k8s-node2 xiaoniao]# unzip xiaoniaofeifei.zip Archive: xiaoniaofeifei.zip inflating: sound1.mp3 creating: img/ inflating: img/bg1.jpg inflating: img/bg2.jpg inflating: img/number1.png inflating: img/number2.png inflating: img/s1.png inflating: img/s2.png inflating: 21.js inflating: 2000.png inflating: icon.png inflating: index.html [root@k8s-node2 xiaoniao]# ll total 232 -rw-r--r-- 1 root root 15329 Aug 2 2014 2000.png -rw-r--r-- 1 root root 51562 Aug 2 2014 21.js -rw-r--r-- 1 root root 254 Aug 2 2014 icon.png drwxr-xr-x 2 root root 102 Aug 8 2014 img -rw-r--r-- 1 root root 3049 Aug 2 2014 index.html -rw-r--r-- 1 root root 63008 Aug 2 2014 sound1.mp3 -rw-r--r-- 1 root root 91014 Aug 29 16:54 xiaoniaofeifei.zip [root@k8s-node2 xiaoniao]# rm -rf xiaoniaofeifei.zip [root@k8s-node2 xiaoniao]# ##2.2gitlab建立倉庫並上傳代碼 [root@k8s-node2 xiaoniao]# git config --global user.name "Administrator" [root@k8s-node2 xiaoniao]# git config --global user.email admin@example.com [root@k8s-node2 xiaoniao]# git init Initialized empty Git repository in /opt/xiaoniao/.git/ [root@k8s-node2 xiaoniao]# git remote add origin http://10.0.0.13/root/xiaoniao.git [root@k8s-node2 xiaoniao]# git push -u origin master Username for 'http://10.0.0.13': ^C [root@k8s-node2 xiaoniao]# git add . [root@k8s-node2 xiaoniao]# git commit -m "Initial commit" [master (root-commit) 0212ea4] Initial commit 11 files changed, 184 insertions(+) create mode 100644 2000.png create mode 100644 21.js create mode 100644 icon.png create mode 100644 img/bg1.jpg create mode 100644 img/bg2.jpg create mode 100644 img/number1.png create mode 100644 img/number2.png create mode 100644 img/s1.png create mode 100644 img/s2.png create mode 100644 index.html create mode 100644 sound1.mp3 [root@k8s-node2 xiaoniao]# git push -u origin master Username for 'http://10.0.0.13': root Password for 'http://root@10.0.0.13': 12345678 Counting objects: 14, done. Compressing objects: 100% (14/14), done. Writing objects: 100% (14/14), 88.15 KiB | 0 bytes/s, done. Total 14 (delta 0), reused 0 (delta 0) To http://10.0.0.13/root/xiaoniao.git * [new branch] master -> master Branch master set up to track remote branch master from origin. [root@k8s-node2 xiaoniao]#
6.3安裝jenkins並拉取代碼
#1.node1上安裝jenkins
##1.1安裝jenkins
[root@k8s-node1 jenkins-k8s]# rpm -ivh jdk-8u102-linux-x64.rpm
Preparing... ################################# [100%]
Updating / installing...
1:jdk1.8.0_102-2000:1.8.0_102-fcs ################################# [100%]
Unpacking JAR files...
tools.jar...
plugin.jar...
javaws.jar...
deploy.jar...
rt.jar...
jsse.jar...
charsets.jar...
localedata.jar...
[root@k8s-node1 jenkins-k8s]# mkdir -p /app
[root@k8s-node1 jenkins-k8s]# tar xf apache-tomcat-8.0.27.tar.gz -C /app
[root@k8s-node1 jenkins-k8s]# rm -rf /app/apache-tomcat-8.0.27/webapps/*
[root@k8s-node1 jenkins-k8s]# mv jenkins.war /app/apache-tomcat-8.0.27/webapps/ROOT.war
[root@k8s-node1 jenkins-k8s]#
##1.2jenkin的數據包
[root@k8s-node1 jenkins-k8s]# tar xf jenkin-data.tar.gz -C /root
[root@k8s-node1 jenkins-k8s]# ls /root/.jenkins/
config.xml jenkins.install.UpgradeWizard.state nodes updates
hudson.model.UpdateCenter.xml jenkins.model.JenkinsLocationConfiguration.xml plugins userContent
hudson.plugins.git.GitTool.xml jenkins.telemetry.Correlator.xml queue.xml users
identity.key.enc jobs secret.key workflow-libs
jenkins.install.InstallUtil.installingPlugins logs secret.key.not-so-secret
jenkins.install.InstallUtil.lastExecVersion nodeMonitors.xml secrets
[root@k8s-node1 jenkins-k8s]#
##1.3啓動tomcat
[root@k8s-node1 jenkins-k8s]# /app/apache-tomcat-8.0.27/bin/startup.sh
Using CATALINA_BASE: /app/apache-tomcat-8.0.27
Using CATALINA_HOME: /app/apache-tomcat-8.0.27
Using CATALINA_TMPDIR: /app/apache-tomcat-8.0.27/temp
Using JRE_HOME: /usr
Using CLASSPATH: /app/apache-tomcat-8.0.27/bin/bootstrap.jar:/app/apache-tomcat-8.0.27/bin/tomcat-juli.jar
Tomcat started.
[root@k8s-node1 jenkins-k8s]# netstat -antlp|grep 8080
tcp 0 0 10.0.0.12:48266 10.0.0.11:8080 ESTABLISHED 124115/kubelet
tcp 0 0 10.0.0.12:49970 10.0.0.11:8080 ESTABLISHED 124115/kubelet
tcp 0 0 10.0.0.12:42178 10.0.0.11:8080 ESTABLISHED 26169/kube-proxy
tcp 0 0 10.0.0.12:42182 10.0.0.11:8080 ESTABLISHED 26169/kube-proxy
tcp 0 0 10.0.0.12:44416 10.0.0.11:8080 ESTABLISHED 124115/kubelet
tcp 0 0 10.0.0.12:48388 10.0.0.11:8080 ESTABLISHED 124115/kubelet
tcp6 0 0 :::8080 :::* LISTEN 60694/java
[root@k8s-node1 jenkins-k8s]#
admin/123456
在jenkins上建立構建項目 #1.在node1上生成祕鑰對 [root@k8s-node1 jenkins-k8s]# ssh-key ssh-keygen ssh-keyscan [root@k8s-node1 jenkins-k8s]# ssh-keygen -t rsa Generating public/private rsa key pair. Enter file in which to save the key (/root/.ssh/id_rsa): Enter passphrase (empty for no passphrase): Enter same passphrase again: Your identification has been saved in /root/.ssh/id_rsa. Your public key has been saved in /root/.ssh/id_rsa.pub. The key fingerprint is: SHA256:BpstuSGo4XaoPG7u1uzVw07fmkuEm3yDdgQEKsnEtDQ root@k8s-node1 The key s randomart image is: +---[RSA 2048]----+ |oE ... | |+.+ . . | | = . .. | | .. *o | |. . . *.So | |.o. .+=* | |.+o. ..@ = | |++.o. + * + | |B=o. . =o. | +----[SHA256]-----+ [root@k8s-node1 jenkins-k8s]# #2.複製公鑰到github中 [root@k8s-node1 ~]# cat /root/.ssh/id_rsa.pub ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCsZ08H1iFE2/Vwy8Rgx8hfUMuspYIh6TptlwbDd2vGANkLfJlVodhDRSpAeOI51i/VETR+7XJtMQ12AN5tM/s0tHNJxw8FptRJ8sOBcVxjkDjiq2BC5YPsJC6RUm2Ze64yu0wF+yr5yaya/mhOlC58Ri9qCR+1mAL6FZrN3Mi/3KEBeNU9Vuojpi2+Hd4MJt0Vnii+xps1ZlfdHCzynraeyzDDT5BTwuvwDx78HpwBSc3GKtH90AkkINOqPoyJmbG+YC1p7jhc6a9saRm2okNo5LYP/6xPi+XhKoVALmLl/iPJuL90a3Lmg2lgASrrBbtWa40jRh/PliWEh+oWexBj root@k8s-node1 [root@k8s-node1 ~]#
[root@k8s-node1 ~]# ll /root/.jenkins/workspace/xiaoniao/ total 140 -rw-r--r-- 1 root root 15329 Aug 29 19:04 2000.png -rw-r--r-- 1 root root 51562 Aug 29 19:04 21.js -rw-r--r-- 1 root root 254 Aug 29 19:04 icon.png drwxr-xr-x 2 root root 102 Aug 29 19:04 img -rw-r--r-- 1 root root 3049 Aug 29 19:04 index.html -rw-r--r-- 1 root root 63008 Aug 29 19:04 sound1.mp3 [root@k8s-node1 ~]#
6.4構建docker鏡像並測試
#1.構建docker鏡像並測試 [root@k8s-node2 xiaoniao]# pwd /opt/xiaoniao [root@k8s-node2 xiaoniao]# ll total 144 -rw-r--r-- 1 root root 15329 Aug 2 2014 2000.png -rw-r--r-- 1 root root 51562 Aug 2 2014 21.js -rw-r--r-- 1 root root 59 Aug 29 19:13 dockerfile -rw-r--r-- 1 root root 254 Aug 2 2014 icon.png drwxr-xr-x 2 root root 102 Aug 8 2014 img -rw-r--r-- 1 root root 3049 Aug 2 2014 index.html -rw-r--r-- 1 root root 63008 Aug 2 2014 sound1.mp3 [root@k8s-node2 xiaoniao]# cat dockerfile FROM 10.0.0.11:5000/nginx:1.13 ADD . /usr/share/nginx/html [root@k8s-node2 xiaoniao]# vi .dockerignore #不要把dockerfile這個文件上傳到容器中 [root@k8s-node2 xiaoniao]# cat .dockerignore dockerfile [root@k8s-node2 xiaoniao]# #建立鏡像 [root@k8s-node2 xiaoniao]# docker build -t xiaoniao:v1 . Sending build context to Docker daemon 328.2 kB Step 1/2 : FROM 10.0.0.11:5000/nginx:1.13 ---> ae513a47849c Step 2/2 : ADD . /usr/share/nginx/html ---> 7ac78e054c58 Removing intermediate container 3fe76618255c Successfully built 7ac78e054c58 [root@k8s-node2 xiaoniao]# docker images REPOSITORY TAG IMAGE ID CREATED SIZE xiaoniao v1 7ac78e054c58 42 seconds ago 109 MB #建立容器 [root@k8s-node2 xiaoniao]# docker run -d -p 88:80 xiaoniao:v1 ab6195eb8e19a0505ad8a4c41a9ea116fa962fd0a971bd4fb3fd37614e0c667b [root@k8s-node2 xiaoniao]# docker ps -a CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES ab6195eb8e19 xiaoniao:v1 "nginx -g 'daemon ..." 5 seconds ago Up 4 seconds 0.0.0.0:88->80/tcp competent_easley [root@k8s-node2 xiaoniao]#
#把新加的內容傳到github上 [root@k8s-node2 xiaoniao]# git add . [root@k8s-node2 xiaoniao]# git commit -m "first update" [master 747d882] first update 2 files changed, 3 insertions(+) create mode 100644 .dockerignore create mode 100644 dockerfile [root@k8s-node2 xiaoniao]# git push -u origin master Username for 'http://10.0.0.13': root Password for 'http://root@10.0.0.13': Counting objects: 5, done. Compressing objects: 100% (3/3), done. Writing objects: 100% (4/4), 377 bytes | 0 bytes/s, done. Total 4 (delta 1), reused 0 (delta 0) To http://10.0.0.13/root/xiaoniao.git 0212ea4..747d882 master -> master Branch master set up to track remote branch master from origin. [root@k8s-node2 xiaoniao]#
#查看node1上workspace下的內容 [root@k8s-node1 ~]# ll /root/.jenkins/workspace/xiaoniao/ total 144 -rw-r--r-- 1 root root 15329 Aug 29 19:04 2000.png -rw-r--r-- 1 root root 51562 Aug 29 19:04 21.js -rw-r--r-- 1 root root 59 Aug 29 19:34 dockerfile -rw-r--r-- 1 root root 254 Aug 29 19:04 icon.png drwxr-xr-x 2 root root 102 Aug 29 19:04 img -rw-r--r-- 1 root root 3049 Aug 29 19:04 index.html -rw-r--r-- 1 root root 63008 Aug 29 19:04 sound1.mp3 [root@k8s-node1 ~]#
6.5自動化構建docker鏡像版本問題
使用jenkins環境變量 BUILD_ID 修改構建後腳本爲 docker build -t 10.0.0.11:5000/xiaoniao:v$BUILD_ID . docker push 10.0.0.11:5000/xiaoniao:v$BUILD_ID
#查看私有倉庫中的版本 [root@k8s-master ~]# ls /opt/myregistry/docker/registry/v2/repositories/xiaoniao/_manifests/tags/ v1 v3 v4 v5 [root@k8s-master ~]#
#防止誤操做當即構建,沒有更新，就不上傳新版本了
#!/bin/bash
if [ $GIT_PREVIOUS_SUCCESSFUL_COMMIT == $GIT_COMMIT ];then
echo "no change，skip build"
exit 0
else
echo "git pull commmit id not equals to current commit id trigger build"
fi
[root@k8s-master ~]# ls /opt/myregistry/docker/registry/v2/repositories/xiaoniao/_manifests/tags/ v1 v3 v4 v5 [root@k8s-master ~]#
6.6jenkins實現k8s應用的版本升級和迴歸
#1.k8s集羣部署項目
#1.1建立deployment
[root@k8s-master ~]# kubectl run xiaoniao --image=10.0.0.11:5000/xiaoniao:v1 --replicas=2 --record
deployment "xiaoniao" created
#--record 是爲了查看歷史記錄
[root@k8s-master ~]# kubectl rollout history deployment
deployments "xiaoniao"
REVISION CHANGE-CAUSE
1 kubectl run xiaoniao --image=10.0.0.11:5000/xiaoniao:v1 --replicas=2 --record
[root@k8s-master ~]#
#1.2查看，建立了兩個副本
[root@k8s-master ~]# kubectl get all
NAME DESIRED CURRENT UP-TO-DATE AVAILABLE AGE
deploy/xiaoniao 2 2 2 2 2m
NAME CLUSTER-IP EXTERNAL-IP PORT(S) AGE
svc/kubernetes 10.254.0.1 <none> 443/TCP 1d
NAME DESIRED CURRENT READY AGE
rs/xiaoniao-562012648 2 2 2 2m
NAME READY STATUS RESTARTS AGE
po/xiaoniao-562012648-jwk09 1/1 Running 0 2m
po/xiaoniao-562012648-px0fb 1/1 Running 0 2m
[root@k8s-master ~]#
#1.3暴露端口，讓外部訪問
[root@k8s-master ~]# kubectl expose deployment xiaoniao --port=80 --type=NodePort
service "xiaoniao" exposed
[root@k8s-master ~]# kubectl get svc
NAME CLUSTER-IP EXTERNAL-IP PORT(S) AGE
kubernetes 10.254.0.1 <none> 443/TCP 1d
xiaoniao 10.254.10.198 <nodes> 80:32169/TCP 10s
[root@k8s-master ~]#
#1.4訪問
[root@k8s-master ~]# curl 10.0.0.12:32169|grep '小鳥'
% Total % Received % Xferd Average Speed Time Time Time Current
Dload Upload Total Spent Left Speed
100 3049 100 3049 0 0 1315k 0 --:--:-- --:--:-- --:--:-- 2977k
<title>小鳥飛飛飛-文章庫小遊戲</title>
"tTitle": "小鳥飛飛飛-文章庫小遊戲",
document.title = "我玩小鳥飛飛飛過了"+t+"關！你能超過灑家我嗎？";
[root@k8s-master ~]#
#2.k8s 版本升級和回滾
##2.1版本更新
[root@k8s-master ~]# kubectl set image deploy xiaoniao xiaoniao=10.0.0.11:5000/xiaoniao:v3
deployment "xiaoniao" image updated
[root@k8s-master ~]# kubectl set image deploy xiaoniao xiaoniao=10.0.0.11:5000/xiaoniao:v6
deployment "xiaoniao" image updated
##2.2查看歷史版本
[root@k8s-master ~]# kubectl rollout history deployment xiaoniao
deployments "xiaoniao"
REVISION CHANGE-CAUSE
1 kubectl run xiaoniao --image=10.0.0.11:5000/xiaoniao:v1 --replicas=2 --record
2 kubectl set image deploy xiaoniao xiaoniao=10.0.0.11:5000/xiaoniao:v3
3 kubectl set image deploy xiaoniao xiaoniao=10.0.0.11:5000/xiaoniao:v6
[root@k8s-master ~]#
##2.3回滾：
[root@k8s-master ~]# kubectl rollout undo deploy xiaoniao
deployment "xiaoniao" rolled back
#3.jenkins實現一鍵k8s版本升級
修改構建執行shell腳本內容
if [ $GIT_PREVIOUS_SUCCESSFUL_COMMIT == $GIT_COMMIT ];then
echo "no change，skip build"
exit 0
else
echo "git pull commmit id not equals to current commit id trigger build"
docker build -t 10.0.0.11:5000/xiaoniao:v$BUILD_ID .
docker push 10.0.0.11:5000/xiaoniao:v$BUILD_ID
kubectl -s 10.0.0.11:8080 set image deploy xiaoniao xiaoniao=10.0.0.11:5000/xiaoniao:v$BUILD_ID
fi
#更新git中內容測試 [root@k8s-node2 xiaoniao]# echo "jiu yao jie shu le">over.html [root@k8s-node2 xiaoniao]# git add . [root@k8s-node2 xiaoniao]# git commit -m "update" [master 76930bb] update 1 file changed, 1 insertion(+) create mode 100644 over.html [root@k8s-node2 xiaoniao]# git push -u origin master Username for 'http://10.0.0.13': root Password for 'http://root@10.0.0.13': Counting objects: 4, done. Compressing objects: 100% (2/2), done. Writing objects: 100% (3/3), 282 bytes | 0 bytes/s, done. Total 3 (delta 1), reused 0 (delta 0) To http://10.0.0.13/root/xiaoniao.git 747d882..76930bb master -> master Branch master set up to track remote branch master from origin. [root@k8s-node2 xiaoniao]#
[root@k8s-master ~]# kubectl get all -o wide NAME DESIRED CURRENT UP-TO-DATE AVAILABLE AGE deploy/xiaoniao 2 2 2 2 23m NAME CLUSTER-IP EXTERNAL-IP PORT(S) AGE SELECTOR svc/kubernetes 10.254.0.1 <none> 443/TCP 1d <none> svc/xiaoniao 10.254.10.198 <nodes> 80:32169/TCP 19m run=xiaoniao NAME DESIRED CURRENT READY AGE CONTAINER(S) IMAGE(S) SELECTOR rs/xiaoniao-1161273840 2 2 2 59s xiaoniao 10.0.0.11:5000/xiaoniao:v9 pod-template-hash=1161273840,run=xiaoniao rs/xiaoniao-562012648 0 0 0 23m xiaoniao 10.0.0.11:5000/xiaoniao:v1 pod-template-hash=562012648,run=xiaoniao rs/xiaoniao-711827946 0 0 0 16m xiaoniao 10.0.0.11:5000/xiaoniao:v3 pod-template-hash=711827946,run=xiaoniao rs/xiaoniao-783916636 0 0 0 8m xiaoniao 10.0.0.11:5000/v7 pod-template-hash=783916636,run=xiaoniao rs/xiaoniao-936550893 0 0 0 16m xiaoniao 10.0.0.11:5000/xiaoniao:v6 pod-template-hash=936550893,run=xiaoniao NAME READY STATUS RESTARTS AGE IP NODE po/xiaoniao-1161273840-28277 1/1 Running 0 58s 172.16.79.2 k8s-node1 po/xiaoniao-1161273840-r2ff1 1/1 Running 0 58s 172.16.48.3 k8s-node2
回滾
每日一句
-
每一个你不满意的现在，都有一个你没有努力的曾经。
最新文章
- 1. python的安裝和Hello，World編寫
- 2. 重磅解讀：K8s Cluster Autoscaler模塊及對應華爲雲插件Deep Dive
- 3. 鴻蒙學習筆記2（永不斷更）
- 4. static關鍵字 和構造代碼塊
- 5. JVM筆記
- 6. 無法啓動 C/C++ 語言服務器。IntelliSense 功能將被禁用。錯誤: Missing binary at c:\Users\MSI-NB\.vscode\extensions\ms-vsc
- 7. 【Hive】Hive返回碼狀態含義
- 8. Java樹形結構遞歸（以時間換空間）和非遞歸（以空間換時間）
- 9. 數據預處理---缺失值
- 10. 都要2021年了，現代C++有什麼值得我們學習的？
歡迎關注本站公眾號,獲取更多信息
