keepalived+nginx的高可用

時間 2019-11-19

標籤 keepalived+nginx keepalived nginx 可用欄目 Nginx 简体版

原文原文鏈接

keepalived+nginx的高可用
#########
nginx+keepalived環境：
master:10.10.54.61(vip:10.10.54.69)
backup:10.10.54.64(vip:10.10.54.69)
realserver：10.10.54.63
realserver：10.10.54.67

本文不是作lvs，因此realserver不是配置在keepalived.conf
而是在nginx的配置文件中upstream

此架構需考慮的問題
1）Master沒掛，則Master佔有vip且nginx運行在Master上
2）Master掛了，則backup搶佔vip且在backup上運行nginx服務
3）若是master服務器上的nginx服務掛了，則vip資源轉移到backup服務器上
4）檢測後端服務器的健康狀態
Master和Backup兩邊都開啓nginx服務，不管Master仍是Backup，當其中的一個keepalived服務中止後，vip都會漂移到keepalived服務還在的節點上，
若是要想使nginx服務掛了，vip也漂移到另外一個節點，則必須用腳本或者在配置文件裏面用shell命令來控制。
1、安裝keepalived+nginx
10.10.54.61/64
#############################
源碼編譯nginx
1.下載
[root@gyf soft]#wget http://nginx.org/download/nginx-1.4.5.tar.gz
[root@gyf soft]# tar xvf nginx-1.4.5.tar.gz
2.編譯
[root@gyf nginx-1.4.5]# ./configure --prefix=/usr/local/nginx --user=apache --group=apache --with-http_stub_status_module   --with-http_gzip_static_module --with-http_ssl_module
///
--with-http_stub_status_module     enable ngx_http_stub_status_module     ---支持監控
--with-http_gzip_static_module     enable ngx_http_gzip_static_module     ---支持壓縮

3.安裝
[root@gyf nginx-1.4.5]# make && make install
4.啓動
[root@gyf conf]# /usr/local/nginx/sbin/nginx -c /usr/local/nginx/conf/nginx.conf
nginx: [emerg] getpwnam("apache") failed
[root@gyf conf]# useradd apache

[root@gyf conf]# /usr/local/nginx/sbin/nginx -c /usr/local/nginx/conf/nginx.conf
nginx: [emerg] bind() to 0.0.0.0:80 failed (98: Address already in use)
nginx: [emerg] bind() to 0.0.0.0:80 failed (98: Address already in use)
nginx: [emerg] bind() to 0.0.0.0:80 failed (98: Address already in use)
nginx: [emerg] bind() to 0.0.0.0:80 failed (98: Address already in use)
nginx: [emerg] bind() to 0.0.0.0:80 failed (98: Address already in use)
nginx: [emerg] still could not bind()
[root@gyf conf]# netstat -ntlp|grep 80
tcp        0      0 0.0.0.0:80                  0.0.0.0:*                   LISTEN      1572/httpd
[root@gyf conf]# apachectl stop
[root@gyf conf]# /usr/local/nginx/sbin/nginx -c /usr/local/nginx/conf/nginx.conf
5.關閉:給父進程發送一個TERM信號，試圖殺死它和它的子進程。
[root@s01 logs]# cat /usr/local/nginx/logs/nginx.pid | xargs kill -TERM

6.重啓
[root@s01 logs]# cat /usr/local/nginx/logs/nginx.pid | xargs kill -HUP

HUP      重啓
TERM,INT 快速中止
USR1    從新打開日誌文件，用於日誌切割
USR2    平滑升級可執行程序
QUIT     從容關閉
WINCH    從容關閉工做進程

//測試配置文件
[root@s01 html]# /usr/local/nginx/sbin/nginx -t -c /usr/local/nginx/conf/nginx.conf

7.製做nginx自啓動10.10.54.61/64
[root@gyf init.d]# vim /etc/init.d/nginx
#!/bin/bash
#chkconfig: 2345 80 90
#description: nginx
alter=$1
nginx=/usr/local/nginx/sbin/nginx
nginx_conf=/usr/local/nginx/conf/nginx.conf
nginx_pid=/usr/local/nginx/logs/nginx.pid
. /etc/rc.d/init.d/functions
function if_info
{
        if [ $2 == 0 ];then
                echo -n "nginx $1 is ok!" && success && echo
        else
                echo -n "nginx $1 is error!" && success && echo
        fi
}
case $alter in
start)
         if [ -f $nginx_pid ];then

                echo "nginx is already start!"
        else
                $nginx -c $nginx_conf
                if_info start $?
        fi
        ;;
stop)
       if [ ! -f $nginx_pid ];then
                echo "nginx is already stop!"
         else
                kill -TERM `cat $nginx_pid`

                if_info stop $?
        fi
        ;;
restart)
        if [ ! -f $nginx_pid ];then
                echo "nginx is stop,please start nginx!"
        else
                kill -HUP `cat $nginx_pid`
                if_info restart $?
        fi
        ;;
test)
        $nginx -t -c $nginx_conf
#       $nginx -t
        if_info test $?
        ;;
status)
        if [ ! -f $nginx_pid ];then
                echo "nginx is stop"
        else
                echo "nginx is runing"
        fi
        ;;
*)
        echo "Usage: $0 {start|stop|status|restart|test}"
        ;;
esac

chmod +x /etc/init.d/nginx

chkconfig --add nginx
chkconfig    nginx onjavascript

chkconfig nginx --list
css

/etc/init.d/nginx start
[root@Cent64 keepalived]# ps -ef |grep nginx

#########################
編譯ipvsadm10.10.54.61/64

//安裝依賴包
yum -y install wget libnl* popt* gcc.x86_64 gcc-c++.x86_64 gcc-objc++.x86_64 kernel-devel.x86_64 make popt-static.x86_64

//編譯ipvsadm
[root@Cent64 softs]# tar xvf ipvsadm-1.26.tar.gz
[root@tech2 lvs]# cd ipvsadm-1.26
[root@tech2 ipvsadm-1.26]# make && make install

//確認lvs模塊
[root@tech2 ipvsadm-1.26]# modprobe -l|grep ipvs
kernel/net/netfilter/ipvs/ip_vs.ko
kernel/net/netfilter/ipvs/ip_vs_rr.ko
kernel/net/netfilter/ipvs/ip_vs_wrr.ko
kernel/net/netfilter/ipvs/ip_vs_lc.ko
kernel/net/netfilter/ipvs/ip_vs_wlc.ko
kernel/net/netfilter/ipvs/ip_vs_lblc.ko
kernel/net/netfilter/ipvs/ip_vs_lblcr.ko
kernel/net/netfilter/ipvs/ip_vs_dh.ko
kernel/net/netfilter/ipvs/ip_vs_sh.ko
kernel/net/netfilter/ipvs/ip_vs_sed.ko
kernel/net/netfilter/ipvs/ip_vs_nq.ko
kernel/net/netfilter/ipvs/ip_vs_ftp.ko

3.編譯keepalived
[root@tech2 lvs]# tar xvf keepalived-1.2.9.tar.gz
[root@tech2 keepalived-1.2.9]# ls

//基礎軟件包
In order to compile Keepalived needs the following libraries :

* OpenSSL, <www.openssl.org>
* popt

[root@tech2 keepalived-1.2.9]# yum install -y net-snmp.x86_64 net-snmp-devel.x86_64

[root@tech2 keepalived-1.2.9]# ./configure --prefix=/usr/local/keepalived --enable-snmp --sysconfdir=/etc

Keepalived configuration
------------------------
Keepalived version       : 1.2.9
Compiler                 : gcc
Compiler flags           : -g -O2
Extra Lib                : -Wl,-z,relro -Wl,-z,now -L/usr/lib64 -lnetsnmpagent -lnetsnmphelpers -lnetsnmpmibs -lnetsnmp -Wl,-E -Wl,-rpath,/usr/lib64/perl5/CORE -lssl -lcrypto -lcrypt -lnl
Use IPVS Framework       : Yes
IPVS sync daemon support : Yes
IPVS use libnl           : Yes
Use VRRP Framework       : Yes
Use VRRP VMAC            : Yes
SNMP support             : Yes
SHA1 support             : No
Use Debug flags          : No

[root@tech2 keepalived-1.2.9]# make && make install

[root@tech2 sbin]# cp /usr/local/keepalived/sbin/keepalived /sbin/
[root@tech2 bin]# cp /usr/local/keepalived/bin/genhash /bin/
[root@tech2 bin]# chkconfig --add keepalived
[root@centos61 ~]# /etc/init.d/keepalived start
二，修改keepalived配置文件
10.10.54.61
[root@centos61 ~]# vim /etc/keepalived/keepalived.conf

! Configuration File for keepalived

global_defs {
   notification_email {
     yangry@shiwei.com
   }
   notification_email_from yangry@shiwei.com
   smtp_server mail.shiwei.com
   smtp_connect_timeout 30
   router_id LVS_MASTER1 #表示運行keepalived服務器的一個標識，發郵件時顯示在郵件主題中的信息
}
vrrp_script chk_http_port {
script "/usr/local/keepalived/nginx.sh" ####檢測nginx狀態的腳本連接
interval 2
weight 2
}
vrrp_instance VI_2 {   #vrrp實例
    state MASTER     #MASTER/BACKUP
    interface eth0    ####HA 監測網絡接口
    virtual_router_id 51 #虛擬路由標識，是一個數字，同一個VRRP實例使用惟一的標識,master和backup要同樣
    priority 100          #用於主從模式，優先級主高於100，從低於100
    advert_int 1           #主備之間的通告間隔秒數
    authentication {        #認證用於主從模式，mater和backup配置同樣
        auth_type PASS          ###主備切換時的驗證
        auth_pass 1111          #密碼
    }
track_script {
chk_http_port ### 執行監控的服務
}
    virtual_ipaddress {

10.10.54.69/24 dev eth0 label eth0:1   ###########虛擬ip
    }
}

[root@centos61 ~]#vim /usr/local/keepalived/nginx.sh
#!/bin/bash
if [ `ps -C nginx --no-header |wc -l` -eq 0 ];thenhtml

#if [ `ps -ef|grep nginx:mater process|wc -l` -eq 0 ]; then
killall keepalived
fi

三，修改keepalived配置文件
10.10.54.64
[root@centos64 ~]# vim /etc/keepalived/keepalived.conf

! Configuration File for keepalived

global_defs {
   notification_email {
     yangry@shiwei.com
   }
   notification_email_from yangry@shiwei.com
   smtp_server mail.shiwei.com
   smtp_connect_timeout 30
   router_id LVS_SLAVE #表示運行keepalived服務器的一個標識，發郵件時顯示在郵件主題中的信息
}
vrrp_script chk_http_port {
script "/usr/local/keepalived/nginx.sh" ####檢測nginx狀態的腳本連接
interval 2    #腳本執行間隔
weight 2       #腳本結果致使的優先級變動
}
vrrp_instance VI_2 {   #vrrp實例
    state BACKUP     #MASTER/BACKUP
    interface eth0    ####HA 監測網絡接口
    virtual_router_id 51 #虛擬路由標識，是一個數字，同一個VRRP實例使用惟一的標識,master和backup要同樣
    priority 80          #用於主從模式，優先級主高於100，從低於100
    advert_int 1           #主備之間的通告間隔秒數
    authentication {        #認證用於主從模式，mater和backup配置同樣
        auth_type PASS          ###主備切換時的驗證
        auth_pass 1111          #密碼驗證要一致
    }
track_script {
chk_http_port ### 執行監控的服務
}
    virtual_ipaddress {

10.10.54.69/24 dev eth0 label eth0:1   ###########虛擬ip
    }
}

[root@centos64 ~]#vim /usr/local/keepalived/nginx.sh
#!/bin/bash
if [ `ps -C nginx --no-header |wc -l` -eq 0 ];thenjava

#if [ `ps -ef|grep nginx:mater process|wc -l` -eq 0 ];thennginx

killall keepalived
fi
######以上作完測試vip是否能夠飄移，nginx中止vip也能漂移
四.nginx實現後端realserver的負載均衡
10.10.54.61/64
1.配置代理文件
[root@gyf htdocs]# cd /usr/local/nginx/conf/
[root@gyf conf]#mkdir virtual
[root@gyf conf]# vim virtual/bbs.ssr.com.conf
upstream bbs_ssr_com {
    server 10.10.54.63:80 max_fails=3 weight=1 fail_timeout=60s;
    server 10.10.54.67:80 max_fails=3 weight=3 fail_timeout=60s;
    }

server {
     listen      80;
     server_name bbs.ssr.com; #bbs.ssr.com 的dns能解析到10.10.54.69
   access_log logs/www.access.log;
    error_log   logs/www.error.log;
    location / {
        proxy_pass http://bbs_ssr_com;
        proxy_set_header HOST             $host;
        proxy_set_header X-Real-IP                          $remote_addr;
        proxy_set_header X-Forwarded-For             $proxy_add_x_forwarded_for;
                }
}c++

.主配置文件配置

[root@gyf ~]# vi /usr/local/nginx/conf/nginx.conf
user nginx nginx;
worker_processes 2;
error_log logs/error.log info;
pid      logs/nginx.pid;
events {
    worker_connections 1024;
}
http {
    include      mime.types;
    default_type application/octet-stream;
    #日誌格式定義
    log_format main ‘$remote_addr – $remote_user[$time_local] 「$request」 ‘
                      ‘$status $body_bytes_sent」$http_referer」 ‘
                    ‘」$http_user_agent」「$http_x_forwarded_for」‘;
    access_log logs/access.log main;
    sendfile        on;
keepalive_timeout 65;
#gzip壓縮功能設置
    gzip on;
    gzip_min_length 1k;
    gzip_buffers    4 16k;
    gzip_http_version 1.0;
    gzip_comp_level 2;
    gzip_types text/plain application/x-javascripttext/css application/xml;
    gzip_vary on;

include virtual/bbs.ssr.com.conf;

}

五.在10.10.54.63/67上安裝apache 製做網站
10.10.54.63/67
yum install -y httpd.x86_64 httpd-devel.x86_64 httpd-tools.x86_64

六.重啓各類服務

shell