OpenResty+Keepalived組建高可用集羣+雙主模式

* 環境：

system version：CentOS Linux release 7.5.1804 (Core) 3.10.0-862.9.1.el7.x86_64
nginx version：openresty/1.13.6.1
keepalive version：keepalived-2.0.6

---

OpenResty安裝

[root@localhost soft]# yum install yum-fastestmirror     #更新源
[root@localhost soft]# yum -y update

* OpenResty所需依賴的包安裝

[root@localhost soft]# yum install gcc gcc-c++ libreadline-dev libncurses5-dev libpcre3-dev libssl-dev pcre pcre-devel zlib zlib-devel openssl openssl-devel readline-devel  perl -y

* 下載nginx_upstream_check_module模塊，該模塊用於ustream健康檢查

OpenResty在這裏用於反向代理，若是後端服務器宕掉的話，nginx是不能把這臺realserver踢出upstream，還會有請求轉發到後端的這臺realserver上面去，雖然能夠在localtion中啓用proxy_next_upstream來解決返回給用戶的錯誤頁面，但請求仍是會先把請求轉發給這臺服務器，而後再轉發給別的服務器，這樣就浪費了一次轉發，對於這種狀況能夠藉助淘寶技術團隊開發的nginx模快nginx_upstream_check_module來檢測後方realserver的健康狀態，若是後端服務器不可用，全部的請求不轉發到這臺服務器。
nginx_upstream_check_module模塊地址：https://github.com/yaoweibin/nginx_upstream_check_module

[root@localhost soft]# wget https://github.com/yaoweibin/nginx_upstream_check_module/archive/v0.3.0.tar.gz
[root@localhost soft]# tar -zxvf v0.3.0.tar.gz

* 下載ngx_cache_purge模塊，該模塊用於清理nginx緩存

[root@localhost soft]# wget http://labs.frickle.com/files/ngx_cache_purge-2.3.tar.gz
[root@localhost soft]# tar zxvf ngx_cache_purge-2.3.tar.gz

* 編譯安裝OpenResty

[root@localhost soft]# wget https://openresty.org/download/openresty-1.13.6.2.tar.gz
[root@localhost soft]# tar -zxvf openresty-1.13.6.2.tar.gz
[root@localhost soft]# cd openresty-1.13.6.2
[root@localhost openresty-1.13.6.2]# groupadd www
[root@localhost openresty-1.13.6.2]# useradd -M -g www -s /sbin/nologin www
[root@localhost openresty-1.13.6.2]# ./configure --prefix=/app/OpenResty \
> --user=www \
> --group=www \
> --with-luajit \
> --without-http_redis2_module \
> --with-http_iconv_module \
> --with-http_realip_module \       #獲取用戶真實ip模塊
> --with-pcre \                              #Perl兼容的達式模塊
> --with-luajit \                             #集成luajit模塊
> --add-module=../ngx_cache_purge-2.3/ \     #緩存模塊
> --add-module=../nginx_upstream_check_module-0.3.0/ \   #upstream健康檢查模塊
> --with-http_stub_status_module \         #狀態信息
> --with-http_ssl_module \                      #ssl模塊
> -j2
[root@localhost openresty-1.13.6.2]# gmake && gmake install
[root@localhost openresty-1.13.6.2]# cd /app/OpenResty/nginx/sbin/
[root@localhost sbin]# ./nginx -V
nginx version: openresty/1.13.6.2
built by gcc 4.8.5 20150623 (Red Hat 4.8.5-28) (GCC) 
built with OpenSSL 1.0.2k-fips  26 Jan 2017
TLS SNI support enabled
configure arguments: --prefix=/app/OpenResty/nginx --with-cc-opt=-O2 --add-module=../ngx_devel_kit-0.3.0 --add-module=../iconv-nginx-module-0.14 --add-module=../echo-nginx-module-0.61 --add-module=../xss-nginx-module-0.06 --add-module=../ngx_coolkit-0.2rc3 --add-module=../set-misc-nginx-module-0.32 --add-module=../form-input-nginx-module-0.12 --add-module=../encrypted-session-nginx-module-0.08 --add-module=../srcache-nginx-module-0.31 --add-module=../ngx_lua-0.10.13 --add-module=../ngx_lua_upstream-0.07 --add-module=../headers-more-nginx-module-0.33 --add-module=../array-var-nginx-module-0.05 --add-module=../memc-nginx-module-0.19 --add-module=../redis-nginx-module-0.3.7 --add-module=../rds-json-nginx-module-0.15 --add-module=../rds-csv-nginx-module-0.09 --add-module=../ngx_stream_lua-0.0.5 --with-ld-opt=-Wl,-rpath,/app/OpenResty/luajit/lib --user=www --group=www --with-http_realip_module --with-pcre --add-module=/app/soft/openresty-1.13.6.2/../ngx_cache_purge-2.3 --add-module=/app/soft/openresty-1.13.6.2/../nginx_upstream_check_module-0.3.0 --with-http_stub_status_module --with-http_ssl_module --with-stream --with-stream_ssl_module
[root@localhost sbin]#

[root@localhost openresty-1.13.6.2]# ./configure --prefix=/app/OpenResty --user=www --group=www --with-luajit --without-http_redis2_module --with-http_iconv_module --with-http_realip_module --with-pcre --with-luajit --add-module=../ngx_cache_purge-2.3/ --add-module=../nginx_upstream_check_module-0.3.0/ --with-http_stub_status_module --with-http_ssl_module -j2

* 將OpenResty配置成服務，設置開機啓動

[root@localhost nginx]# vim /lib/systemd/system/nginx.service
[Unit]            #服務的說明
Description=nginx      #描述服務
After=network.target              #描述服務類別

[Service]             #服務運行參數的設置
Type=forking       #後臺運行的形式，
ExecStart=/app/OpenResty/nginx/sbin/nginx    #服務的具體運行命令
ExecReload=/app/OpenResty/nginx/sbin/nginx reload     #重啓命令
ExecStop=/app/OpenResty/nginx//sbin/nginx quit     #中止命令
PrivateTmp=true         #給服務分配獨立的臨時空間

[Install]
WantedBy=multi-user.target
[root@localhost nginx]# systemctl enable nginx
nginx.service is not a native service, redirecting to /sbin/chkconfig.
Executing /sbin/chkconfig nginx on
[root@localhost nginx]#

注意：[Service]的啓動、重啓、中止命令所有要求使用絕對路徑
[Install]運行級別下服務安裝的相關設置，可設置爲多用戶，即系統運行級別爲3

* OpenResty配置

[root@localhost soft]# cd /app/OpenResty/nginx/conf/
[root@localhost conf]# cp nginx.conf{,.default20180723bak}
[root@localhost conf]# vim nginx.conf

---

* keepalived安裝

• keepalive官網：http://www.keepalived.org/software/

* keepalived依賴安裝

[root@localhost ~]# cd /app/soft/
[root@localhost soft]# yum -y install kernel kernel-devel*  popt popt-devel libssl-dev libnl libnl-devel openssl openssl-* ipvsadm libnfnetlink-devel

* keepalived安裝

[root@localhost soft]# wget http://www.keepalived.org/software/keepalived-2.0.6.tar.gz
[root@localhost soft]# tar -zxvf keepalived-2.0.6.tar.gz
[root@localhost soft]# cd keepalived-2.0.6
[root@localhost keepalived-2.0.6]# ./configure --prefix=/app/keepalived --sysconf=/etc --with-kernel-dir=/usr/src/kernels/3.10.0-862.11.6.el7.x86_64
checking syslog.h presence... yes
checking for syslog.h... yes
checking for unistd.h... (cached) yes
checking for linux/netlink.h... no
configure: error: Missing/unusable kernel header file <linux/netlink.h>
[root@localhost keepalived-2.0.6] ./configure --prefix=/app/keepalived --sysconf=/etc
Keepalived configuration
------------------------
Keepalived version       : 2.0.6
Compiler                 : gcc
Preprocessor flags       :
Compiler flags           : -Wall -Wunused -Wstrict-prototypes -Wextra -Winit-self -g -D_GNU_SOURCE -fPIE -Wformat -Werror=format-security -Wp,-D_FORTIFY_SOURCE=2 -fexceptions -fsta
Linker flags             :  -pie
Extra Lib                :  -lcrypto  -lssl  -lnl
Use IPVS Framework       : Yes
IPVS use libnl           : Yes
IPVS syncd attributes    : No
IPVS 64 bit stats        : No
fwmark socket support    : Yes
Use VRRP Framework       : Yes
Use VRRP VMAC            : Yes
Use VRRP authentication  : Yes
With ip rules/routes     : Yes
Use BFD Framework        : No
SNMP vrrp support        : No
SNMP checker support     : No
SNMP RFCv2 support       : No
SNMP RFCv3 support       : No
DBUS support             : No
SHA1 support             : No
Use Json output          : No
libnl version            : 1
Use IPv4 devconf         : No
Use libiptc              : No
Use libipset             : No
init type                : systemd
Build genhash            : Yes
Build documentation      : No
[root@localhost keepalived-2.0.6]# make && make install

--sysconf 指定了配置文件的地址.即：/etc/keepalived/keepalived.conf
--prefix 指定了安裝目錄
--with-kernel-dir 指定使用內核源碼中的頭文件,即 include 目錄.只有使用 LVS 時才須要這個參數，getconf LONG_BIT 獲得系統位數。其它的時候不須要。

編譯配置須要確保一下幾項爲Yes狀態：
Use IPVS Framework : Yes
IPVS sync daemon support : Yes
IPVS use libnl : Yes
Use VRRP Framework : Yes

* keepalived配置

• 編譯安裝後，會根據系統環境，生成啓動腳本

[root@localhost keepalived-2.0.6]# cat /usr/lib/systemd/system/keepalived.service
[Unit]
Description=LVS and VRRP High Availability Monitor
After= network-online.target syslog.target
Wants=network-online.target

[Service]
Type=forking
PIDFile=/var/run/keepalived.pid
KillMode=process
EnvironmentFile=-/etc/sysconfig/keepalived
ExecStart=/app/keepalived/sbin/keepalived $KEEPALIVED_OPTIONS
ExecReload=/bin/kill -HUP $MAINPID

[Install]
WantedBy=multi-user.target
[root@localhost keepalived-2.0.6]#

檢查配置文件是否正確。

• 配置日誌路徑
  修改EnvironmentFile=-/etc/sysconfig/keepalived，使用-f指定配置文件路徑、日誌信息。

[root@localhost keepalived-2.0.6]# cat /etc/sysconfig/keepalived
# Options for keepalived. See `keepalived --help' output and keepalived(8) and
# keepalived.conf(5) man pages for a list of all options. Here are the most
# common ones :
#
# --vrrp               -P    Only run with VRRP subsystem.
# --check              -C    Only run with Health-checker subsystem.
# --dont-release-vrrp  -V    Dont remove VRRP VIPs & VROUTEs on daemon stop.
# --dont-release-ipvs  -I    Dont remove IPVS topology on daemon stop.
# --dump-conf          -d    Dump the configuration data.
# --log-detail         -D    Detailed log messages.
# --log-facility       -S    0-7 Set local syslog facility (default=LOG_DAEMON)
#

KEEPALIVED_OPTIONS="-D"

[root@localhost keepalived-2.0.6]# cp /etc/sysconfig/keepalived{,.default}
[root@localhost keepalived-2.0.6]# sed -i 's?KEEPALIVED_OPTIONS=.*?KEEPALIVED_OPTIONS="-d -D -S 0"?' /etc/sysconfig/keepalived
[root@localhost keepalived-2.0.6]# cp /etc/rsyslog.conf{,.default}
[root@localhost keepalived-2.0.6]# sed -i '$a\local0.* /var/log/keepalived.log' /etc/rsyslog.conf
[root@localhost keepalived-2.0.6]# systemctl restart rsyslog

-S 0 配置keepalived日誌輸出到local0，

• 複製keepalived啓動文件到默認路徑，也能夠經過設置環境變量的path實現

[root@localhost keepalived-2.0.6]# ln -s /app/keepalived/sbin/keepalived /sbin/

• 將keepalived配置文件拷貝到默認路徑

[root@localhost keepalived-2.0.6]# ln -s  /app/keepalived/etc/keepalived/keepalived.conf /etc/keepalived/keepalived.conf

• 複製keepalived服務腳本到默認的地址，也經過修改init.d/keepalived文件中的相應配置實現

[root@localhost keepalived-2.0.6]#  cp /app/keepalived/etc/sysconfig/keepalived  /etc/sysconfig/

• 複製服務啓動腳本到，將keepalived註冊成服務

[root@localhost keepalived-2.0.6]#  cp /app/keepalived/etc/rc.d/init.d/keepalived  /etc/init.d/
[root@localhost keepalived-2.0.6]#   systemctl enable keepalived

• 啓動keepalived

[root@justin keepalived-2.0.6]# systemctl start keepalived

---

• 一、Nginx+keepalived 主從配置
  這種模式，使用一個vip地址，前端使用2臺機器，一臺作主，一臺作備，但同時只有一臺機器工做，另外一臺備份機器在主機器不出現故障的時候，永遠處於浪費狀態，對於服務器很少的網站，該方案不經濟實惠。

• 二、Nginx+keepalived 雙主配置
  這種模式，使用兩個vip地址，前端使用2臺機器，互爲主備，同時有兩臺機器工做，當其中一臺機器出現故障，兩臺機器的請求轉移到一臺機器負擔，很是適合於當前架構環境。

* 雙主模式

雙主模式2臺keepalived的配置幾乎同樣，只須要顛倒下vrrp_instance的state、priority值。

[root@justin keepalived]# cat keepalived.conf
! Configuration File for keepalived

global_defs {
   notification_email {
     acassen@firewall.loc
     failover@firewall.loc
     sysadmin@firewall.loc
   }
   notification_email_from Alexandre.Cassen@firewall.loc
   smtp_server 192.168.200.1
   smtp_connect_timeout 30
   router_id LVS_DEVEL
   vrrp_skip_check_adv_addr
#   vrrp_strict
   vrrp_garp_interval 0
   vrrp_gna_interval 0
}

vrrp_script chk_nginx {
    script "/etc/keepalived/check_nginx.sh"
    interval 2
    weight 2
}

vrrp_instance VI_1 {           #定義一個名爲 VI_1 的 VRRP 實例
    state MASTER               #Keepalived服務器角色,設置爲主，只有MASTER和BACKUP 兩種狀態，都必須大寫。
    interface ens33              #設置實例綁定的網卡，通信網卡
    virtual_router_id 51        #虛擬路由標識，同一實例下該值必須相同，即MASTER和BACKUP的virtual_router_id是一致的。
    priority 100                    #節點優先級，0-254，數值越大，權重越大。權重值 MASTRE 必定要高於 BAUCKUP
    advert_int 1                   #MASTER與BACKUP負載均衡器之間同步檢查的時間間隔，單位是秒
    authentication {             #節點之間通訊驗證類型、密碼 ，同一 VRRP 實例中，MASTER/BACKUP使用相同的密碼才能夠通訊
        auth_type PASS       #主從服務器驗證類型，主要有PASS和AH兩種
        auth_pass 1111        #加密的密碼，用於通信主機間驗證，兩臺服務器必定要同樣，否則會出錯
    }
    virtual_ipaddress {       #虛擬IP地址，又稱漂移IP能夠經過ip add在MASTER上查看是否綁定
        10.15.43.15             #VIP 地址，能夠多個虛擬IP，換行便可
    }
    notify_master "/etc/keepalived/clean_arp.sh 10.15.43.15"     #更新虛擬服務器（VIP）地址的arp記錄到網關
}

vrrp_instance VI_2 {
    state BACKUP
    interface ens33
    virtual_router_id 51
    priority 99
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 1111
    }
    virtual_ipaddress {
        10.15.43.13
    }
    notify_master "/etc/keepalived/clean_arp.sh 10.15.43.13"
}
[root@justin keepalived]#

nginx狀態監測：

[root@justin keepalived]# cat check_nginx.sh
#!/bin/bash
export LANG="en_US.UTF-8"
#export LANG="zh_CN.UTF8"
source /etc/rc.d/init.d/functions
[ -f /etc/profile ] && . /etc/profile
[ -f ~/.bash_profile ] && . ~/.bash_profile
if [ $(ps -C nginx --no-header | wc -l) -eq 0 ]; then
   /app/OpenResty/nginx/sbin/nginx -c /app/OpenResty/nginx/conf/nginx.conf
fi
sleep 3
if [ $(ps -C nginx --no-header | wc -l) -eq 0 ]; then
   systemctl stop keepalived
fi
[root@justin keepalived]#

設置更新虛擬服務器（VIP）地址的arp記錄到網關腳本：

[root@justin keepalived]# cat clean_arp.sh
#!/bin/bash
export LANG="en_US.UTF-8"
#export LANG="zh_CN.UTF8"
source /etc/rc.d/init.d/functions
[ -f /etc/profile ] && . /etc/profile
[ -f ~/.bash_profile ] && . ~/.bash_profile
VIP=$1
GATEWAY=10.15.43.254
/sbin/arping -I ens33 -c 5 -s $VIP $GATEWAY &>/dev/null
[root@justin keepalived]#

而後重啓nginx、keepalived。