18.11 LVS DR模式搭建 18.12 keepalived + LVS

時間 2019-11-13

標籤 18.11 lvs 模式搭建 18.12 keepalived 欄目負載均衡简体版

原文原文鏈接

18.11 LVS DR模式搭建

準備工做

三臺機器，三臺機器均有公網IP。html

調度器（director）
IP：192.168.230.135
real server1(real1)
IP：192.168.230.130
real server2(real2)
IP：192.168.230.145
VIP：192.168.230.200

開始搭建linux

配置director 算法

[root@cham002 ~]# vim /usr/local/sbin/lvs_dr.sh

#! /bin/bash
echo 1 > /proc/sys/net/ipv4/ip_forward
ipv=/usr/sbin/ipvsadm
vip=192.168.230.200
rs1=192.168.230.130
rs2=192.168.230.145
#注意這裏的網卡名字
ifdown ens33
ifup ens33
ifconfig ens33:2 $vip broadcast $vip netmask 255.255.255.255 up
route add -host $vip dev ens33:2
$ipv -C
$ipv -A -t $vip:80 -s rr
$ipv -a -t $vip:80 -r $rs1:80 -g -w 1
$ipv -a -t $vip:80 -r $rs2:80 -g -w 1

執行腳本：
[root@cham002 ~]# sh /usr/local/sbin/lvs_dr.sh 
成功斷開設備 'ens33'。
成功激活的鏈接（D-Bus 激活路徑：/org/freedesktop/NetworkManager/ActiveConnection/6）
[root@cham002 ~]# ip add
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN qlen 1
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host 
       valid_lft forever preferred_lft forever
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP qlen 1000
    link/ether 00:0c:29:b6:9f:e3 brd ff:ff:ff:ff:ff:ff
    inet 192.168.230.135/24 brd 192.168.230.255 scope global ens33
       valid_lft forever preferred_lft forever
    inet 192.168.230.200/32 brd 192.168.230.200 scope global ens33:2
       valid_lft forever preferred_lft forever
    inet 192.168.230.150/24 brd 192.168.230.255 scope global secondary ens33:0
       valid_lft forever preferred_lft forever
    inet6 fe80::6f15:52d3:ebeb:e193/64 scope link 
       valid_lft forever preferred_lft forever
3: ens37: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP qlen 1000
    link/ether 00:0c:29:b6:9f:ed brd ff:ff:ff:ff:ff:ff
    inet 192.168.118.147/24 brd 192.168.118.255 scope global ens37
       valid_lft forever preferred_lft forever
    inet 192.168.100.1/24 brd 192.168.100.255 scope global ens37
       valid_lft forever preferred_lft forever
    inet6 fe80::1801:cbbb:ebcc:89a3/64 scope link 
       valid_lft forever preferred_lft forever

注： VIP綁定到了ens33網卡上。shell

腳本解釋vim

#! /bin/bash
echo 1 > /proc/sys/net/ipv4/ip_forward
#開啓端口轉發
ipv=/usr/sbin/ipvsadm
vip=192.168.230.200
rs1=192.168.230.130
rs2=192.168.230.145
#注意這裏的網卡名字
ifdown ens33
ifup ens33
#在此重啓網卡的目的是避免重複設置命令行提供的IP
ifconfig ens33:2 $vip broadcast $vip netmask 255.255.255.255 up
#綁定VIP到dir的虛擬網卡ens33:2
route add -host $vip dev ens33:2
#添加網關
$ipv -C
$ipv -A -t $vip:80 -s wrr
$ipv -a -t $vip:80 -r $rs1:80 -g -w 1
$ipv -a -t $vip:80 -r $rs2:80 -g -w 1
#設置ipvsadm規則，-g=gateway：使用默認網關（DR模式）

配置real server

分別在real一、real2配置下面的腳本：瀏覽器

##real1（130）
[root@cham1 ~]# vi /usr/local/sbin/lvs_rs.sh

#/bin/bash
vip=192.168.230.200
#把vip綁定在lo上，是爲了實現rs直接把結果返回給客戶端
ifdown lo
ifup lo
ifconfig lo:0 $vip broadcast $vip netmask 255.255.255.255 up
route add -host $vip lo:0
#如下操做爲更改arp內核參數，目的是爲了讓rs順利發送mac地址給客戶端
#參考文檔www.cnblogs.com/lgfeng/archive/2012/10/16/2726308.html
echo "1" >/proc/sys/net/ipv4/conf/lo/arp_ignore
echo "2" >/proc/sys/net/ipv4/conf/lo/arp_announce
echo "1" >/proc/sys/net/ipv4/conf/all/arp_ignore
echo "2" >/proc/sys/net/ipv4/conf/all/arp_announce

執行腳本：
[root@cham1 ~]# vi /usr/local/sbin/lvs_rs.sh
[root@cham1 ~]# sh !$
sh /usr/local/sbin/lvs_rs.sh

[root@cham1 ~]# route -n
Kernel IP routing table
Destination     Gateway         Genmask         Flags Metric Ref    Use Iface
0.0.0.0         192.168.230.2   0.0.0.0         UG    100    0        0 ens33
192.168.230.0   0.0.0.0         255.255.255.0   U     100    0        0 ens33
192.168.230.200 0.0.0.0         255.255.255.255 UH    0      0        0 lo

[root@cham1 ~]# ip add
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN qlen 1
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet 192.168.230.200/32 brd 192.168.230.200 scope global lo:0
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host 
       valid_lft forever preferred_lft forever
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP qlen 1000
    link/ether 00:0c:29:90:35:d9 brd ff:ff:ff:ff:ff:ff
    inet 192.168.230.130/24 brd 192.168.230.255 scope global ens33
       valid_lft forever preferred_lft forever
    inet6 fe80::7fe3:4489:d9af:a1ed/64 scope link 
       valid_lft forever preferred_lft forever
    inet6 fe80::6f15:52d3:ebeb:e193/64 scope link tentative dadfailed 
       valid_lft forever preferred_lft forever


##rs2(145)
[root@test ~]# vim /usr/local/sbin/lvs_rs.sh
[root@test ~]# sh !$
sh /usr/local/sbin/lvs_rs.sh

[root@test ~]# route -n
Kernel IP routing table
Destination     Gateway         Genmask         Flags Metric Ref    Use Iface
0.0.0.0         192.168.230.2   0.0.0.0         UG    100    0        0 ens33
192.168.230.0   0.0.0.0         255.255.255.0   U     100    0        0 ens33
192.168.230.200 0.0.0.0         255.255.255.255 UH    0      0        0 lo

[root@test ~]# ip add
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN qlen 1
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet 192.168.230.200/32 brd 192.168.230.200 scope global lo:0
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host 
       valid_lft forever preferred_lft forever
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP qlen 1000
    link/ether 00:0c:29:50:f9:44 brd ff:ff:ff:ff:ff:ff
    inet 192.168.230.145/24 brd 192.168.230.255 scope global ens33
       valid_lft forever preferred_lft forever
    inet6 fe80::9b07:b28d:f5e9:d107/64 scope link 
       valid_lft forever preferred_lft forever

135 bash

[root@cham002 ~]# ipvsadm -ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
  -> RemoteAddress:Port           Forward Weight ActiveConn InActConn
TCP  192.168.230.200:80 rr
  -> 192.168.230.130:80           Route   1      0          1         
  -> 192.168.230.145:80           Route   1      0          1         
[root@cham002 ~]# ipvsadm -ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
  -> RemoteAddress:Port           Forward Weight ActiveConn InActConn
TCP  192.168.230.200:80 rr
  -> 192.168.230.130:80           Route   1      2          1         
  -> 192.168.230.145:80           Route   1      1          1         
[root@cham002 ~]#

測試

在瀏覽器訪問VIP：192.168.230.200，刷新網頁，訪問結果由real一、real2交替回覆。服務器

開另外一臺同網段虛擬機測試！網絡

Type `help' to learn how to use Xshell prompt.
[d:\~]$ ssh 192.168.230.140


Connecting to 192.168.230.140:22...
Connection established.
To escape to local shell, press 'Ctrl+Alt+]'.

Last login: Fri Nov 17 15:14:29 2017 from 192.168.230.1
[root@cham3 ~]# curl 192.168.230.200
cham  145 backup backup.
[root@cham3 ~]# curl 192.168.230.200
cham  130

[root@cham3 ~]# curl 192.168.230.200
cham  145 backup backup.
[root@cham3 ~]# curl 192.168.230.200
cham  130

[root@cham3 ~]# curl 192.168.230.200
cham  145 backup backup.
[root@cham3 ~]# curl 192.168.230.200
cham  130

[root@cham3 ~]# curl 192.168.230.200
cham  145 backup backup.
[root@cham3 ~]# curl 192.168.230.200
cham  130

18.12 Keepalived LVS

完整的架構須要兩臺服務器（角色爲dir），分別安裝Keepalived工具，目的是實現高可用，但Keepalived自己也有負載均衡功能，因此本次使用能夠只安裝一臺Keepalived。Keepalived內置了ipvsadm的功能，因此不須要安裝ipvsadm包，也不用編寫和執行lvs_dr腳本。架構

準備工做

三臺機器：

調度器director：
IP：192.168.230.135；安裝Keepalived
real server（real1）：
IP：192.168.230.130
real server（real2）：
IP：192.168.230.145
VIP：192.168.230.200

•兩臺rs上，依然要執行/usr/local/sbin/lvs_rs.sh腳本

• keepalived有一個比較好的功能，能夠在一臺rs宕機時，再也不把請求轉發過去

• 測試

配置director (由於咱們以前作keepalived高可用的時候已經安裝過了）

[root@cham002 ~]# vim /etc/keepalived/keepalived.conf 

vrrp_instance VI_1 {
    #備用服務器上爲 BACKUP
    state MASTER
    #綁定vip的網卡爲ens33，你的網卡和阿銘的可能不同，這裏須要你改一下
    interface ens33
    virtual_router_id 51
    #備用服務器上爲90
    priority 100
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass aminglinux
    }
    virtual_ipaddress {
        192.168.230.200
    }
}
virtual_server 192.168.230.200 80 {
    #(每隔10秒查詢realserver狀態)
    delay_loop 10
    #(lvs 算法)
    lb_algo wlc
    #(DR模式)
    lb_kind DR
    #(同一IP的鏈接60秒內被分配到同一臺realserver)
    persistence_timeout 0
    #(用TCP協議檢查realserver狀態)
    protocol TCP
    real_server 192.168.230.130 80 {
        #(權重)
        weight 100
        TCP_CHECK {
        #(10秒無響應超時)
        connect_timeout 10
        nb_get_retry 3
        delay_before_retry 3
        connect_port 80
        }
    }
    real_server 192.168.230.145 80 {
        weight 100
        TCP_CHECK {
        connect_timeout 10
        nb_get_retry 3
        delay_before_retry 3
        connect_port 80
        }
     }
}

執行ipvsadm -C  把以前的ipvsadm規則清空掉
[root@cham002 ~]# ipvsadm -C

重啓網絡能夠把以前的vip關掉
[root@cham002 ~]# systemctl restart network


啓動Keepalived服務：
[root@cham002 ~]# systemctl start keepalived
[root@cham002 ~]# ps aux |grep keep
root     11296  0.0  0.1 120720  1404 ?        Ss   1月24   0:00 /usr/sbin/keepalived -D
root     11297  0.0  0.3 127460  3272 ?        S    1月24   0:00 /usr/sbin/keepalived -D
root     11298  0.0  0.3 131656  3032 ?        S    1月24   0:05 /usr/sbin/keepalived -D
root     47689  0.0  0.0 112680   976 pts/1    S+   00:31   0:00 grep --color=auto keep

查看網卡信息：
[root@cham002 ~]# ip addr
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN qlen 1
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host 
       valid_lft forever preferred_lft forever
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP qlen 1000
    link/ether 00:0c:29:b6:9f:e3 brd ff:ff:ff:ff:ff:ff
    inet 192.168.230.135/24 brd 192.168.230.255 scope global ens33
       valid_lft forever preferred_lft forever
    inet 192.168.230.200/32 scope global ens33
       valid_lft forever preferred_lft forever
    inet 192.168.230.150/24 brd 192.168.230.255 scope global secondary ens33:0
       valid_lft forever preferred_lft forever
    inet6 fe80::6f15:52d3:ebeb:e193/64 scope link 
       valid_lft forever preferred_lft forever
3: ens37: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP qlen 1000
    link/ether 00:0c:29:b6:9f:ed brd ff:ff:ff:ff:ff:ff
    inet 192.168.118.147/24 brd 192.168.118.255 scope global ens37
       valid_lft forever preferred_lft forever
    inet 192.168.100.1/24 brd 192.168.100.255 scope global ens37
       valid_lft forever preferred_lft forever
    inet6 fe80::1801:cbbb:ebcc:89a3/64 scope link 

[root@cham002 ~]# ipvsadm -ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
  -> RemoteAddress:Port           Forward Weight ActiveConn InActConn
TCP  192.168.230.200:80 wlc
  -> 192.168.230.130:80           Route   100    0          0         
  -> 192.168.230.145:80           Route   100    0          0

添加Keepalived後會自動將宕機的real server清除出rs列表。

若有一方的ngixn 宕掉了，能自動檢測，開啓後自動加載回來
[root@cham002 ~]# ipvsadm -ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
  -> RemoteAddress:Port           Forward Weight ActiveConn InActConn
TCP  192.168.230.200:80 wlc
  -> 192.168.230.145:80           Route   100    0          0         

[root@cham002 ~]# ipvsadm -ln
IP Virtual Server version 1.2.1 (size=4096)
Prot LocalAddress:Port Scheduler Flags
  -> RemoteAddress:Port           Forward Weight ActiveConn InActConn
TCP  192.168.230.200:80 wlc
  -> 192.168.230.130:80           Route   100    0          0         
  -> 192.168.230.145:80           Route   100    0          0

測試

To escape to local shell, press 'Ctrl+Alt+]'.

Last login: Tue Jan 30 23:15:25 2018 from 192.168.230.1
130宕機
[root@cham3 ~]# curl 192.168.230.200
cham  145 backup backup.
[root@cham3 ~]# curl 192.168.230.200
cham  145 backup backup.
[root@cham3 ~]# curl 192.168.230.200
cham  145 backup backup.
[root@cham3 ~]# curl 192.168.230.200
cham  145 backup backup.
[root@cham3 ~]# curl 192.168.230.200
cham  145 backup backup.
[root@cham3 ~]# curl 192.168.230.200
cham  145 backup backup.
[root@cham3 ~]# curl 192.168.230.200
cham  145 backup backup.
[root@cham3 ~]# curl 192.168.230.200
cham  145 backup backup.
[root@cham3 ~]# curl 192.168.230.200
cham  145 backup backup.

145宕機
[root@cham3 ~]# curl 192.168.230.200
cham  130

[root@cham3 ~]# curl 192.168.230.200
cham  130

[root@cham3 ~]# curl 192.168.230.200
cham  130

[root@cham3 ~]# curl 192.168.230.200
cham  130