10: docker 跨主機的容器間通訊(macvlan / overlay )

docker 跨主機的容器間通訊（macvlan）

 做用：

虛擬多個mac地址，虛擬出多個網卡給容器用。

 

#建立macvlan網絡

docker network create --driver macvlan（要建立的網絡類型） --subnet  子網IP段  --gateway 本機網關 -o parent=本機網卡  建立的macvlan網絡名稱

[root@k8s129 ~]# docker network create --driver macvlan --subnet 192.168.6.0/24 --gateway 192.168.6.254 -o parent=eth0  macvlan_xj

7e4729ca45692d2f2a5e4a2129a39027ac1b3f97331129a5dc07093e12edc9f7

[root@k8s129 ~]# docker network ls  #查看建立好的網絡

NETWORK ID          NAME                DRIVER              SCOPE

391cca4ceb1e        bridge              bridge              local

ebb5492e53f3        host                host                local

7e4729ca4569        macvlan_xj          macvlan             local

b087e09b1e13        none                null                local

 

#在另一臺機器，也建立相同的網絡

[root@k8s130 yum.repos.d]# docker network create --driver macvlan --subnet 192.168.6.0/24 --gateway 192.168.6.254 -o parent=eth0  macvlan_xj

34d43e62d44f26cee3124124a87187f89a81d8ef65cb7cd2f313f8c856bef7f3

[root@k8s130 yum.repos.d]# docker network ls

NETWORK ID          NAME                DRIVER              SCOPE

922766952baf        bridge              bridge              local

45355ea7ab2b        host                host                local

34d43e62d44f        macvlan_xj          macvlan             local

1de35a0fe6bd        none                null                local

[root@k8s130 yum.repos.d]#

 

#建立使用macvlan 網絡的 容器

[root@k8s129 ~]# ping 192.168.6.3  #找到一個沒有被使用的IP

PING 192.168.6.3 (192.168.6.3) 56(84) bytes of data.

From 192.168.6.129 icmp_seq=1 Destination Host Unreachable

 

#128機器上面起一個容器,IP地址是：192.168.6.3

[root@k8s129 ~]# docker run -it --network macvlan_xj --ip=192.168.6.3 busybox:latest /bin/sh

/ # ip a

1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue qlen 1000

    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00

    inet 127.0.0.1/8 scope host lo

       valid_lft forever preferred_lft forever

7: eth0@if2: <BROADCAST,MULTICAST,UP,LOWER_UP,M-DOWN> mtu 1500 qdisc noqueue

    link/ether 02:42:c0:a8:06:03 brd ff:ff:ff:ff:ff:ff

    inet 192.168.6.3/24 brd 192.168.6.255 scope global eth0

       valid_lft forever preferred_lft forever

#ping 一下另一臺宿主機IP，發現是能夠ping通的

#而且這時候能夠直接使用xshell，登陸這臺容器，就和宿主機同樣的效果

/ # ping 192.168.6.130

PING 192.168.6.130 (192.168.6.130): 56 data bytes

64 bytes from 192.168.6.130: seq=0 ttl=64 time=1.104 ms

^C

--- 192.168.6.130 ping statistics ---

2 packets transmitted, 2 packets received, 0% packet loss

round-trip min/avg/max = 0.505/0.804/1.104 ms

/ #

#130機器上面也起一個容器，ip地址是:192.168.6.4

[root@k8s130 yum.repos.d]# docker run -it --network macvlan_xj --ip=192.168.6.4 busybox:latest /bin/sh

/ # ip a

1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue qlen 1000

    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00

    inet 127.0.0.1/8 scope host lo

       valid_lft forever preferred_lft forever

5: eth0@if2: <BROADCAST,MULTICAST,UP,LOWER_UP,M-DOWN> mtu 1500 qdisc noqueue

    link/ether 02:42:c0:a8:06:04 brd ff:ff:ff:ff:ff:ff

    inet 192.168.6.4/24 brd 192.168.6.255 scope global eth0

       valid_lft forever preferred_lft forever

/ # ping 192.168.6.3  # ping 192.168.6.3容器，發現是能夠互相ping的，實現了容器互聯

PING 192.168.6.3 (192.168.6.3): 56 data bytes

64 bytes from 192.168.6.3: seq=0 ttl=64 time=1.059 ms

^C

--- 192.168.6.3 ping statistics ---

1 packets transmitted, 1 packets received, 0% packet loss

round-trip min/avg/max = 1.059/1.059/1.059 ms

/ #

 缺點：

每次須要手動指定IP地址,分配IP地址（還要去手動的查看這個IP地址有沒有被使用）

優勢：

性能好，和局域網其它服務器處於同一個網段

-------------------------------

docker跨主機通訊之 overlay(重疊網絡，vxlan)

 

爲支持容器跨主機通訊，Docker 提供了 overlay driver，使用戶能夠建立基於 VxLAN 的 overlay 網絡。

VxLAN 可將二層數據封裝到 UDP 進行傳輸，VxLAN 提供與 VLAN 相同的以太網二層服務，可是擁有更強的擴展性和靈活性。

Docerk overlay 網絡須要一個 key-value 數據庫用於保存網絡狀態信息(這樣就能夠知道哪些IP目前是被使用的)

包括 Network、Endpoint、IP 等。Consul、Etcd 和 ZooKeeper 都是 Docker 支持的 key-vlaue 軟件，咱們這裏使用 Consul。

Consul 相似於redis的功能。

 

#安裝Consul  (直接起一個Consul 的容器就能夠了)

[root@k8s129 ~]# docker run -d -p 8500:8500 -h consul --name consul  --restart=always progrium/consul -server -bootstrap

容器啓動後，能夠經過 http://192.168.6.129:8500 訪問 Consul。

 

#在原有的基礎上面增長以下三行：

  "hosts": ["tcp://0.0.0.0:2376","unix:///var/run/docker.sock"],#起2376端口，和sock

  "cluster-store": "consul://192.168.6.129:8500",           #consul網絡地址

  "cluster-advertise": "192.168.6.129:2376"                

[root@k8s129 ~]# cat /etc/docker/daemon.json    

{

  "registry-mirrors": ["https://aeckruos.mirror.aliyuncs.com"],

  "insecure-registries": ["192.168.6.129:5000"],

  "hosts": ["tcp://0.0.0.0:2376","unix://var/run/docker.sock"],

  "cluster-store": "consul://192.168.6.129:8500",

   "cluster-advertise": "192.168.6.130:2376"

}

[root@k8s129 ~]# vim /usr/lib/systemd/system/docker.service

# for containers run by docker

#ExecStart=/usr/bin/dockerd -H fd:// --containerd=/run/containerd/containerd.sock

ExecStart=/usr/bin/dockerd  #上面的這句改爲當前的這句

 

#重啓 docker daemon。

[root@k8s129 ~]# systemctl daemon-reload  

[root@k8s129 ~]# systemctl restart docker.service

[root@k8s129 ~]# netstat -lntup

Active Internet connections (only servers)

Proto Recv-Q Send-Q Local Address           Foreign Address         State       PID/Program name    

tcp        0      0 0.0.0.0:22              0.0.0.0:*               LISTEN      6120/sshd           

tcp        0      0 127.0.0.1:25            0.0.0.0:*               LISTEN      6209/master         

tcp6       0      0 :::5000                 :::*                    LISTEN      9063/docker-proxy   

tcp6       0      0 :::2376                 :::*                    LISTEN      8921/dockerd        

tcp6       0      0 :::22                   :::*                    LISTEN      6120/sshd           

tcp6       0      0 ::1:25                  :::*                    LISTEN      6209/master         

[root@k8s129 ~]#

#在把剛纔的容器起一下

[root@k8s129 ~]# docker rm `docker ps  -a  -q`

[root@k8s129 ~]# docker run -d -p 8500:8500 -h consul --name consul  --restart=always progrium/consul -server -bootstrap

這個時候網頁點擊：key.value 在點擊docker而後 nodes多點擊幾回，就會出現咱們129 的節點了

 

 

 

 以後再130機器修改以下：

[root@k8s130 yum.repos.d]# cat /etc/docker/daemon.json

{

  "registry-mirrors": ["https://aeckruos.mirror.aliyuncs.com"],

  "insecure-registries": ["192.168.6.129:5000"],

  "hosts": ["tcp://0.0.0.0:2376","unix:///var/run/docker.sock"],

  "cluster-store": "consul://192.168.6.129:8500",

  "cluster-advertise": "192.168.6.130:2376"  #只須要把這個IP改爲咱們的本機130IP就能夠

}

[root@k8s130 yum.repos.d]# vim /usr/lib/systemd/system/docker.service

# for containers run by docker

#ExecStart=/usr/bin/dockerd -H fd:// --containerd=/run/containerd/containerd.sock

ExecStart=/usr/bin/dockerd  #上面的這句改爲當前的這句

[root@k8s130 yum.repos.d]# systemctl daemon-reload  

[root@k8s130 yum.repos.d]# systemctl restart docker.service

 

以後網頁：

就會出現兩個節點了

 

 

 

 

到此咱們的環境就準備好了，下面overlay(重疊網絡，vxlan)的用法：

 ==

建立overlay網絡 

#129或者130建立均可以，在任何一個節點建立網絡，會自動同步到另一臺

[root@k8s139 / 130 ~]# docker network ls  

NETWORK ID          NAME                DRIVER              SCOPE

aaa40ad04d54        bridge              bridge              local

45355ea7ab2b        host                host                local

34d43e62d44f        macvlan_xj          macvlan             local

1de35a0fe6bd        none                null                local

[root@k8s130 ~]# docker network rm 34d43e62d44f   #刪掉以前建立的macvlan_xj 網絡，由於咱們要使用192的網段，若是你有兩塊網卡，好比172網段，就不須要刪除，執行下面的指定成172網段的就行s

[root@k8s130 ~]# docker network create -d overlay --subnet 192.168.6.0/24 --gateway 192.168.6.254 oll

[root@k8s130 ~]# docker network ls  

NETWORK ID          NAME                DRIVER              SCOPE

aaa40ad04d54        bridge              bridge              local

45355ea7ab2b        host                host                local

1de35a0fe6bd        none                null                local

be79115dff0b        oll                 overlay             global

[root@k8s130 ~]#

 

#啓動容器測試，跨主機ping， ping百度均可以上外網

[root@k8s129 ~]# docker run -it --network oll --name xujin01 busybox:latest /bin/sh

/ # ip a

1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue qlen 1000

    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00

    inet 127.0.0.1/8 scope host lo

       valid_lft forever preferred_lft forever

16: eth0@if17: <BROADCAST,MULTICAST,UP,LOWER_UP,M-DOWN> mtu 1450 qdisc noqueue

    link/ether 02:42:c0:a8:06:01 brd ff:ff:ff:ff:ff:ff

    inet 192.168.6.1/24 brd 192.168.6.255 scope global eth0

       valid_lft forever preferred_lft forever

19: eth1@if20: <BROADCAST,MULTICAST,UP,LOWER_UP,M-DOWN> mtu 1500 qdisc noqueue

    link/ether 02:42:ac:12:00:02 brd ff:ff:ff:ff:ff:ff

    inet 172.18.0.2/16 brd 172.18.255.255 scope global eth1

       valid_lft forever preferred_lft forever

/ # ping 192.168.6.2

PING 192.168.6.2 (192.168.6.2): 56 data bytes

64 bytes from 192.168.6.2: seq=0 ttl=64 time=0.996 ms

64 bytes from 192.168.6.2: seq=1 ttl=64 time=1.023 ms

^C

--- 192.168.6.2 ping statistics ---

2 packets transmitted, 2 packets received, 0% packet loss

round-trip min/avg/max = 0.996/1.009/1.023 ms

/ #

 

[root@k8s130 ~]# docker run -it --network oll --name xujin02 busybox:latest /bin/sh

/ # ip a

1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue qlen 1000

    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00

    inet 127.0.0.1/8 scope host lo

       valid_lft forever preferred_lft forever

6: eth0@if7: <BROADCAST,MULTICAST,UP,LOWER_UP,M-DOWN> mtu 1450 qdisc noqueue

    link/ether 02:42:c0:a8:06:02 brd ff:ff:ff:ff:ff:ff

    inet 192.168.6.2/24 brd 192.168.6.255 scope global eth0

       valid_lft forever preferred_lft forever

9: eth1@if10: <BROADCAST,MULTICAST,UP,LOWER_UP,M-DOWN> mtu 1500 qdisc noqueue

    link/ether 02:42:ac:12:00:02 brd ff:ff:ff:ff:ff:ff

    inet 172.18.0.2/16 brd 172.18.255.255 scope global eth1

       valid_lft forever preferred_lft forever

/ # ping 192.168.6.1

PING 192.168.6.1 (192.168.6.1): 56 data bytes

64 bytes from 192.168.6.1: seq=0 ttl=64 time=1.245 ms

^C

--- 192.168.6.1 ping statistics ---

1 packets transmitted, 1 packets received, 0% packet loss

round-trip min/avg/max = 1.245/1.245/1.245 ms

/ # ping baidu.com

PING baidu.com (220.181.38.148): 56 data bytes

64 bytes from 220.181.38.148: seq=0 ttl=127 time=35.268 ms

^C

--- baidu.com ping statistics ---

2 packets transmitted, 1 packets received, 50% packet loss

round-trip min/avg/max = 35.268/35.268/35.268 ms

/ #  

 原理圖：(來源互聯網強哥)

 

 擴展：

 #查看網絡命名空間namespace

容器的網絡環境隔離，就是靠網絡命名空間隔離的

[root@k8s130 ~]# cd /var/run/docker/netns/

[root@k8s130 netns]# ls

256cde1c8c4e  2-be79115dff

[root@k8s130 netns]# ln -s /var/run/docker/netns/  /var/run/netns   # 默認是看不到網絡命名空間，須要作一個軟鏈接

[root@k8s130 netns]# ip netns

256cde1c8c4e (id: 0)

2-be79115dff (id: 1)

[root@k8s130 netns]# ip netns exec 2-be79115dff /bin/bash  #進入網絡命名空間

[root@k8s130 netns]# ifconfig   # 此時會發現有個vxlan0,還有一個bro的網絡,結合上圖就好理解了

br0: flags=4163<UP,BROADCAST,RUNNING,MULTICAST>  mtu 1450

        inet 192.168.6.254  netmask 255.255.255.0  broadcast 192.168.6.255

        ether 0e:15:bd:9d:12:a5  txqueuelen 0  (Ethernet)

        RX packets 0  bytes 0 (0.0 B)

        RX errors 0  dropped 0  overruns 0  frame 0

        TX packets 0  bytes 0 (0.0 B)

        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0

 

lo: flags=73<UP,LOOPBACK,RUNNING>  mtu 65536

        inet 127.0.0.1  netmask 255.0.0.0

        loop  txqueuelen 1000  (Local Loopback)

        RX packets 0  bytes 0 (0.0 B)

        RX errors 0  dropped 0  overruns 0  frame 0

        TX packets 0  bytes 0 (0.0 B)

        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0

 

veth0: flags=4163<UP,BROADCAST,RUNNING,MULTICAST>  mtu 1450

        ether 0e:15:bd:9d:12:a5  txqueuelen 0  (Ethernet)

        RX packets 0  bytes 0 (0.0 B)

        RX errors 0  dropped 0  overruns 0  frame 0

        TX packets 0  bytes 0 (0.0 B)

        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0

 

vxlan0: flags=4163<UP,BROADCAST,RUNNING,MULTICAST>  mtu 1450

        ether ba:c0:e6:bc:58:7d  txqueuelen 0  (Ethernet)

        RX packets 0  bytes 0 (0.0 B)

        RX errors 0  dropped 0  overruns 0  frame 0

        TX packets 0  bytes 0 (0.0 B)

        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0

 

#在129上面也進入命令空間。也會發現vxlan0，就是這個實現了兩個宿主機的容器間的互聯