lvs fullnat+ECMP【4】後端nginx成功解析真實ip

lvs配置：

[root@lvs-fullnat-one keepalived]# cat keepalived.conf 
! Configuration File for keepalived
global_defs {
   notification_email {
     shanks@51cto.com
   }
   notification_email_from Alexandre.Cassen@firewall.loc
   smtp_server 192.168.200.1
   smtp_connect_timeout 30
   router_id LVS_DEVEL
}

local_address_group laddr_g1 {
  #172.16.98.11
  172.16.98.10
}

virtual_server_group shanks1 {
  2.2.2.3 80
}

virtual_server 2.2.2.3 80 {
    delay_loop 6
    lb_algo rr
    lb_kind FNAT
    protocol TCP
    syn_proxy
    laddr_group_name laddr_g1    #local address group
    #alpha
    #omega
    #quorum 1
    #hysteresis 0
    #quorum_up " ip addr add 10.255.255.123/32 dev lo;"    #add
    #quorum_down "ip addr del 10.255.255.123/32 dev lo;"    #del vip
    real_server 172.16.97.30 80 {
        weight 100
        TCP_CHECK {
            connect_timeout 3
            nb_get_retry 3
            delay_before_retry 3
            connect_port 80
        }
    }
}

include /etc/keepalived/hosts/*.conf

[root@lvs-fullnat-one keepalived]#

說明：

global_defs：這個部分不重要，能夠不用填寫內容，但不能少這個部分，不然可能出現沒法啓動的狀況。
local_address_group：snat的源地址，這裏能夠寫多個，一個IP能轉換65536個session。此地方寫的地址，要求在網卡中存在。
virtual_server_group：這裏須要將vip和vport都聲明。
virtual_server：vip和realserver對應關係配置。

網卡多ip配置方法：

[root@lvs-fullnat-one keepalived]# cat /etc/sysconfig/network-scripts/ifcfg-em2
DEVICE=em2
HWADDR=90:B1:1C:5A:37:4E
TYPE=Ethernet
UUID=2b1947f4-1725-4197-abf5-6c8fed750adc
ONBOOT=yes
NM_CONTROLLED=yes
BOOTPROTO=static
IPADDR=172.16.99.11
NETMASK=255.255.255.0
IPADDR1=2.2.2.2
NETMASK1=255.255.255.255
[root@lvs-fullnat-one keepalived]#

或者在ospfd中，經過命令添加（命令相似於思科命令，詳細方法問一下百度）

開啓keepalived

service keepalived restart
輸入一下命令查看輸出信息：

[root@lvs-fullnat-one keepalived]# service keepalived restart
Stopping keepalived:                                       [  OK  ]
Starting keepalived:                                       [  OK  ]
[root@lvs-fullnat-one keepalived]# ipvsadm -ln
IP Virtual Server version 1.2.1 (size=4194304)
Prot LocalAddress:Port Scheduler Flags
  -> RemoteAddress:Port           Forward Weight ActiveConn InActConn
TCP  2.2.2.3:80 rr synproxy
  -> 172.16.97.30:80              FullNat 100    0          0         
[root@lvs-fullnat-one keepalived]# ipvsadm -G
VIP:VPORT            TOTAL    SNAT_IP              CONFLICTS  CONNS     
2.2.2.3:80           1        
                              172.16.98.10         0          0         
[root@lvs-fullnat-one keepalived]#

server端查看nginx的日誌文件

作了三次測試，配別是在
一、後端開啓toa模塊
二、後端沒開啓toa模塊，且lvs配置local_address_group laddr_g1 {172.16.98.11}
三、後端沒開啓toa模塊，且lvs配置local_address_group laddr_g1 {172.16.98.10}
三種狀況下的日誌。
在開啓toa模塊的狀況下，正確識別真實ip，沒開啓toa模塊的狀況下，識別到了snat後的源地址。
如下是截取的日誌文件：

172.16.99.4 - - [30/Aug/2018:16:13:24 +0800] "GET /nginx-logo.png HTTP/1.1" 200 368 "http://2.2.2.3/" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.140 Safari/537.36 Edge/17.17134" "-"
172.16.98.11 - - [30/Aug/2018:16:15:39 +0800] "GET /poweredby.png HTTP/1.1" 200 2811 "http://2.2.2.3/" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.140 Safari/537.36 Edge/17.17134" "-"
172.16.98.10 - - [30/Aug/2018:16:16:58 +0800] "GET / HTTP/1.1" 200 3700 "-" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.140 Safari/537.36 Edge/17.17134" "-"