部署docker鏡像倉庫及高可用

 

下載地址： https://github.com/goharbor/harbor/releases

 

安裝harbor服務器：

安裝harbor

root@harbor-vm1:/usr/local/src# ls

harbor-offline-installer-v1.7.5.tgz

root@harbor-vm1:/usr/local/src# tar -xvf harbor-offline-installer-v1.7.5.tgz  -C /usr/local/src/

root@harbor-vm1:/usr/local/src# cd harbor/

root@harbor-vm1:/usr/local/src/harbor# mkdir certs

root@harbor-vm1:/usr/local/src/harbor# vim harbor.cfg

hostname =  harbor1.dexter.com

ui_url_protocol = https

ssl_cert = /usr/local/src/harbor/cert/server.crt

ssl_cert_key = /usr/local/src/harbor/cert/server.key

harbor_admin_password = 123456

 

生成證書

root@harbor-vm1:~# mkdir  /usr/local/src/harbor/cert

root@harbor-vm1:~# cd  /usr/local/src/harbor/cert

root@harbor-vm1:/usr/local/src/harbor/cert# openssl genrsa -out server.key 2048  #生成私有key

root@harbor-vm1:/usr/local/src/harbor/cert# openssl req -x509 -new -nodes -key  server.key  -subj "/CN= harbor1.dexter.com" -days 7120 -out server.crt   #建立有效期時間的自簽名證書

root@harbor-vm2:/usr/local/src/harbor/cert# openssl req -x509 -new -nodes -key server.key -subj "/CN= harbor2.dexter.com" -days 7120 -out server.crt   #建立有效期時間的自簽名證書

注：若是沒法在ubuntu系統上生成server.crt，能夠嘗試在centos上生成後在複製到ubuntu上。

 

安裝docker

使用官方安裝腳本自動安裝 （僅適用於公網環境）

curl -fsSL https://get.docker.com | bash -s docker --mirror Aliyun

 

安裝docker ce

ubuntu

apt-get install docker-compose -y

 

centos

yum install -y docker-ce

yum install -y python-pip

yum install -y docker-compose

參考文檔： https://yq.aliyun.com/articles/110806

 

#配置Harbor

# ./prepare  

在當前目錄下啓動harbor實例

# ./install.sh

 

配置客戶端使用harbor：

mkdir /etc/docker/certs.d/ harbor1.dexter.com -pv

注：客戶端主要指master和node

 

[root@k8s-harbor1 harbor]# scp cert/server.crt  172.16.99.121:/etc/docker/certs.d/ harbor1.dexter.com/

 

#測試登陸

[root@k8s-m1 ~]# docker login harbor1.dexter.com

Username: admin

Password:

Login Succeeded

 

 

修改本機C:\Windows\System32\drivers\etc\hosts文件，添加以下兩行

172.16.99.127   harbor1.dexter.com

 

嘗試使用瀏覽器打開harbor,帳號：admin，密碼：123456。

順便新建一個基礎鏡像庫

 

測試push鏡像到harbor：

root@k8s-m1:~# docker pull alpine

root@k8s-m1:~# docker tag alpine:latest harbor1.dexter.com/baseimages/alpine:latest

root@k8s-m1:~# docker push harbor1.dexter.com/baseimages/alpine:latest

 

測試pull鏡像

前提是客戶端有證書

root@k8s-n1:~# docker pull harbor1.dexter.com/baseimages/alpine:latest

latest: Pulling from baseimages/alpine

c9b1b535fdd9: Pull complete

Digest: sha256:ddba4d27a7ffc3f86dd6c2f92041af252a1f23a8e742c90e6e1297bfa1bc0c45

Status: Downloaded newer image for harbor1.dexter.com/baseimages/alpine:latest

harbor1.dexter.com/baseimages/alpine:latest

 

設置開機啓動倉庫

echo 'cd /usr/local/src/harbor && ./install.sh' >>/etc/rc.local

chmod +x /etc/rc.local

 

注：若是隻使用http，就簡單的多了，不須要證書了。

 

擴展1：使用http配置harbor

修改harbor.cfg文件

hostname = 172.16.99.127

ui_url_protocol = http

harbor_admin_password = 123456

 

啓動harbor後，能夠使用IP訪問

 

 

客戶端須要修改服務配置文件/lib/systemd/system/docker.service

ExecStart=/usr/bin/dockerd -H fd:// --containerd=/run/containerd/containerd.sock  --insecure-registry 172.16.99.127

# systemctl daemon-reload

# systemctl restart docker

 

# ps -ef | grep dockerd

root      3385     1  0 13:15 ?        00:00:00 /usr/bin/dockerd -H fd:// --containerd=/run/containerd/containerd.sock --insecure-registry 172.16.99.127

 

# docker login 172.16.99.127

 

擴展2：使用單獨的磁盤（/dev/vdb）存儲鏡像

 

好像在centos7.6上默認建立的就是ftype=1

# mkfs.xfs -n ftype=1 /dev/vdb  

 

掛載磁盤

# mount /dev/vdb /var/lib/docker/

 

開機掛載

# blkid  /dev/vdb

/dev/vdb: UUID="36d3a4e3-8ff7-4eb3-b75f-6a30c0eaf802" TYPE="xfs"

# echo 'UUID=36d3a4e3-8ff7-4eb3-b75f-6a30c0eaf802 /var/lib/docker/ xfs defaults 0 0' >>/etc/fstab

 

 

掛載完磁盤後再次安裝docker

# yum install -y docker-ce

 

擴展3：harbor鏡像倉庫高可用

從新部署了2臺harbor鏡像倉庫

172.16.99.152

172.16.99.153

 

VIP

172.16.99.148

 

harbor.cfg配置以下

# grep -v '^#\|^$' /usr/local/src/harbor/harbor.cfg

hostname = 172.16.99.152

ui_url_protocol = http

harbor_admin_password = 123456

 

# grep -v '^#\|^$' /usr/local/src/harbor/harbor.cfg

hostname = 172.16.99.153

ui_url_protocol = http

harbor_admin_password = 123456

 

 

 

 

172.16.99.153

 

 

客戶端

# cat /lib/systemd/system/docker.service

ExecStart=/usr/bin/dockerd -H fd:// --containerd=/run/containerd/containerd.sock --insecure-registry 172.16.99.152 --insecure-registry 172.16.99.153 --insecure-registry 172.16.99.148

 

# docker login 172.16.99.153

Username: admin

Password:

WARNING! Your password will be stored unencrypted in /root/.docker/config.json.

Configure a credential helper to remove this warning. See

https://docs.docker.com/engine/reference/commandline/login/#credentials-store

 

Login Succeeded

# docker tag nginx:v2 172.16.99.153/baseimages/nginx:v2

# docker push 172.16.99.153/baseimages/nginx:v2

 

 

配置高可用

172.16.99.153

 

 

172.16.99.152

 

查看結果

172.16.99.152拿到了172.16.99.153上的鏡像

 

 

 

keepalived+harpoxy(VIP)

 

# cat /etc/keepalived/keepalived.conf

global_defs {

    router_id lb-master-172.16.99.126

}

 

vrrp_script check-haproxy {

    script "killall -0 haproxy"

    interval 5

    weight -60

}

 

vrrp_instance VI-kube-master {

    state MASTER

    priority 120

    unicast_src_ip 172.16.99.126

    unicast_peer {

        172.16.99.125

    }

    dont_track_primary

    interface eth0

    virtual_router_id 111

    advert_int 3

    track_script {

        check-haproxy

    }

    virtual_ipaddress {

        172.16.99.148

    }

}

# cat /etc/keepalived/keepalived.conf

global_defs {

    router_id lb-backup-172.16.99.125

}

 

vrrp_script check-haproxy {

    script "killall -0 haproxy"

    interval 5

    weight -60

}

 

vrrp_instance VI-kube-master {

    state BACKUP

    priority 63

    unicast_src_ip 172.16.99.125

    unicast_peer {

        172.16.99.126

    }

    dont_track_primary

    interface eth0

    virtual_router_id 111

    advert_int 3

    track_script {

        check-haproxy

    }

    virtual_ipaddress {

        172.16.99.148

    }

}

 

# cat /etc/haproxy/haproxy.cfg

global

        log /dev/log    local0

        log /dev/log    local1 notice

        chroot /var/lib/haproxy

        stats socket /run/haproxy/admin.sock mode 660 level admin

        stats timeout 30s

        user haproxy

        group haproxy

        daemon

        nbproc 1

 

defaults

        log     global

        timeout connect 5000

        timeout client  10m

        timeout server  10m

 

listen web

        bind 0.0.0.0:80

        mode tcp

        option tcplog

        balance roundrobin

        server 172.16.99.152 172.16.99.152:80  check inter 2000 fall 2 rise 2 weight 1

        server 172.16.99.153 172.16.99.153:80  check inter 2000 fall 2 rise 2 weight 1

 

客戶端直接從VIP地址訪問和拉取鏡像

root@host-172-16-99-151:~# docker login 172.16.99.148

Username: admin

Password:

WARNING! Your password will be stored unencrypted in /root/.docker/config.json.

Configure a credential helper to remove this warning. See

https://docs.docker.com/engine/reference/commandline/login/#credentials-store

 

Login Succeeded

root@host-172-16-99-151:~# docker pull 172.16.99.148/baseimages/nginx:v2

 

在客戶端上傳鏡像並查看結果

# docker tag mysql:5.6 172.16.99.148/baseimages/mysql:5.6

# docker push 172.16.99.148/baseimages/mysql:5.6

 

咱們在harbor的web段發現2臺harbor機器都有了mysql:5.6這個鏡像，這樣咱們就完成了docker鏡像倉庫的高可用