搭建LVS+Keepalived+nginx+tomcat高可用性，高性能jsp集羣

LVS-master:192.168.0.210

LVS-backup:192.168.0.211

LVS-VIP:192.168.0.209

nginx+tomcat:192.168.0.212

nginx+tomcat:192.168.0.227

安裝nginx所需包：

Nginx-1.6.0.tar.gz和pcre-8.35.zip

一.安裝pcre-8.35

1 #unzip pcre-8.35.zip
2 #cd pcre-8.35
3 #./configure
4 #make
5 #make install

二.安裝nginx

1 #tar -zxvf nginx-1.6.0.tar.gz
2 #cd nginx-1.6.0
3 #./configure --with-http_stub_status_module --with-http_ssl_module
4 #make
5 #make install

在第三部檢查編譯安裝環境，提示缺乏什麼包就安裝什麼包

例：提示錯誤：

1 SSL modules require the OpenSSL library

解決辦法爲安裝openssl

1 yum –y install openssl openssl-devel

nginx安裝成功後安裝目錄爲/usr/local/nginx

三.爲nginx單獨配置用戶，併爲tomcat設置單獨的目錄。安全性和可維護性高

1.tomcat的目錄設置爲/home/www/web(沒有該目錄就新建)

更改tomcat的server.xml文件，在Host name="localhost"處將appBase=""的指向路徑改成/home/www/web

2.建立用戶www和用戶組www爲該文件目錄的使用權限者

1 #/usr/sbin/groupadd www
2 #/usr/sbin/useradd -g www www -s /sbin/nologin
3 #mkdir -p /home/www
4 #chmod +w /home/www
5 #chown -R www:www/home/www

四.配置nginx

1 #cd /usr/local/nginx

在conf文件夾中新建proxy.conf，用於配置一些代理參數，內容以下：

 1 # proxy.conf
 2 proxy_redirect            off;
 3 proxy_set_header          Host $host;
 4 proxy_set_header          X-Real-IP $remote_addr;  
 5 client_max_body_size      10m;
 6 client_body_buffer_size   128k;
 7 proxy_connect_timeout     90;
 8 proxy_send_timeout        90;
 9 proxy_read_timeout        90;
10 proxy_buffer_size         4k;
11 proxy_buffers             4 32k;
12 proxy_busy_buffers_size   64k;
13 proxy_temp_file_write_size 64k;

修改conf中的nginx.conf，配置以下

  1 user  www www;
  2 worker_processes  8;
  3 
  4 error_log   /usr/local/nginx/logs/error.log;
  5 error_log   /usr/local/nginx/logs/error.log  notice;
  6 error_log   /usr/local/nginx/logs/error.log  info;
  7 
  8 pid         /usr/local/nginx/logs/nginx.pid;
  9 
 10 #工做模式及鏈接數上限 
 11 events {
 12     use epoll;
 13     
 14     worker_connections  65535;
 15 }
 16 
 17 #設定http服務器，利用它的反向代理功能提供負載均衡支持 
 18 http {
 19     #設定mime類型   
 20     include       mime.types;
 21     default_type  application/octet-stream;
 22     include       /usr/local/nginx/conf/proxy.conf;
 23     #log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
 24     #                  '$status $body_bytes_sent "$http_referer" '
 25     #                  '"$http_user_agent" "$http_x_forwarded_for"';
 26 
 27     #access_log  logs/access.log  main;
 28 
 29     #設定請求緩衝
 30     server_names_hash_bucket_size  128;
 31     client_header_buffer_size   32K;
 32     large_client_header_buffers  4 32k;
 33     # client_max_body_size   8m;
 34 
 35     sendfile        on;
 36     tcp_nopush      on;
 37     tcp_nodelay     on;
 38 
 39     #keepalive_timeout  0;
 40     keepalive_timeout  65;
 41 
 42     gzip  on;
 43     gzip_min_length  1k;
 44     gzip_buffers   4 16k;
 45     gzip_http_version  1.1;
 46     gzip_comp_level  2;
 47     gzip_types  text/plain application/x-javascript text/css  application/xml;
 48     gzip_vary on;    
 49 
 50      server {
 51       server_name _;
 52       return 404;
 53      }
 54     #此處爲你tomcat的地址，能夠寫多個tomcat地址
 55     upstream tomcat_pool {
 56       # server tomcat地址:端口號 weight=4 max_fails=2 fail_timeout=30s;
 57        server 192.168.0.212:8080 weight=4 max_fails=2 fail_timeout=30s;
 58     }    
 59 
 60     server {
 61         listen       80;
 62         server_name  www.yourdomain.com;#此處替換爲你本身的網址，若有多箇中間用空格
 63         index jump.jsp index.htm index.html  index.do;#設定訪問的默認首頁地址     
 64         root /home/www/web/ROOT; #設定網站的資源存放路徑 
 65 
 66         #charset koi8-r;
 67 
 68         #access_log  logs/host.access.log  main;
 69 
 70        # location / {
 71        #     root   html;
 72        #     index  index.html index.htm;
 73        # }
 74 
 75         location ~ \.(jsp|jspx|dp)?$ #全部JSP的頁面均交由tomcat處理
 76         {
 77             proxy_set_header  Host $host;
 78             proxy_set_header  X-Real-IP $remote_addr; 
 79             proxy_pass http://tomcat_pool;#轉向tomcat處理
 80         }
 81         #設定訪問靜態文件直接讀取不通過tomcat
 82         location ~ .*\.(htm|html|gif|jpg|jpeg|png|bmp|swf|ioc|rar|zip|txt|flv|mid|doc|ppt|pdf|xls|mp3|wma)$ 
 83          {
 84              expires  30d;
 85          }
 86 
 87          location ~ .*\.(js|css)?$
 88          {
 89               expires  1h;
 90          }
 91         log_format  access  '$remote_addr - $remote_user [$time_local] "$request" '
 92               '$status $body_bytes_sent "$http_referer" '
 93               '"$http_user_agent" $http_x_forwarded_for';
 94     access_log  /usr/local/nginx/logs/ubitechtest.log access;#設定訪問日誌的存放路徑     
 95 
 96 
 97 
 98         error_page   500 502 503 504  /50x.html;
 99         location = /50x.html {
100             root   html;
101         }
102 
103 
104         # pass the PHP scripts to FastCGI server listening on 127.0.0.1:9000
105         #
106         #location ~ \.php$ {
107         #    root           html;
108         #    fastcgi_pass   127.0.0.1:9000;
109         #    fastcgi_index  index.php;
110         #    fastcgi_param  SCRIPT_FILENAME  /scripts$fastcgi_script_name;
111         #    include        fastcgi_params;
112         #}
113 
114         # deny access to .htaccess files, if Apache's document root
115         # concurs with nginx's one
116         #
117         #location ~ /\.ht {
118         #    deny  all;
119         #}
120     }
121 
122     server {
123         listen       80;
124         server_name  bbs.yourdomain.com;
125         location / {
126             root /home/www/web/springmvc; #設定網站的資源存放路徑 
127             index index.jsp index.htm index.html  index.do welcome.jsp;#設定訪問的默認首頁地址   
128         }
129 
130         location ~ \.(jsp|jspx|dp)?$ #全部JSP的頁面均交由tomcat處理
131         {
132             proxy_set_header  Host $host;
133             proxy_set_header  X-Real-IP $remote_addr; 
134             proxy_pass http://tomcat_pool;#轉向tomcat處理
135         }
136         #設定訪問靜態文件直接讀取不通過tomcat
137         location ~ .*\.(htm|html|gif|jpg|jpeg|png|bmp|swf|ioc|rar|zip|txt|flv|mid|doc|ppt|pdf|xls|mp3|wma)$ 
138          {
139              expires  30d;
140          }
141 
142          location ~ .*\.(js|css)?$
143          {
144               expires  1h;
145          }
146         log_format  jsp  '$remote_addr - $remote_user [$time_local] "$request" '
147               '$status $body_bytes_sent "$http_referer" '
148               '"$http_user_agent" $http_x_forwarded_for';
149         access_log  /usr/local/nginx/logs/ubitechztt.log jsp;#設定訪問日誌的存放路徑     
150 
151 
152 
153         error_page   500 502 503 504  /50x.html;
154         location = /50x.html {
155             root   html;
156         }
157 
158     }
159 
160     # another virtual host using mix of IP-, name-, and port-based configuration
161     #
162     #server {
163     #    listen       8000;
164     #    listen       somename:8080;
165     #    server_name  somename  alias  another.alias;
166 
167     #    location / {
168     #        root   html;
169     #        index  index.html index.htm;
170     #    }
171     #}
172 
173 }

nginx啓動時提示錯誤：

1 /usr/local/nginx/sbin/nginx -t
2 /usr/local/nginx/sbin/nginx: error while loading shared libraries: libpcre.so.1: cannot open shared object file: No such file or directory

解決辦法爲：

ln -s /usr/local/lib/libpcre.so.1 /lib64

32位系統則：

1 ln -s /usr/local/lib/libpcre.so.1 /lib

附nginx經常使用命令

 1 #檢測配置文件
 2 #/usr/local/nginx/sbin/nginx -t 
 3 #啓動nginx
 4  /usr/local/nginx/sbin/nginx 
 5  echo "/usr/local/nginx/sbin/nginx" >> /etc/rc.local
 6 #查看nginx主進程號
 7 #ps -ef | grep "nginx: master process" | grep -v "grep" | awk -F ' ' '{print $2}'
 8 #中止nginx
 9 #/usr/local/nginx/sbin/nginx -s stop
10 #平滑重啓
11 #/usr/local/nginx/sbin/nginx -s reload

測試時爲在/home/www/web中新建一個index.jsp

啓動tomcat，啓動nginx

經過訪問ip地址便可看到，nginx+tomcat配置完畢。

 

 

五.配置lvs+keepalived

1.安裝LVS前系統須要安裝popt-static，kernel-devel，make，gcc，openssl-devel，lftp，libnl*，popt*

1 #yum –y install popt-static，kernel-devel，make，gcc，openssl-devel，lftp，libnl*，popt*
2 #ln -s /usr/src/kernels/2.6.32-431.17.1.el6.x86_64/ /usr/src/linux
3 #tar -zxvf ipvsadm-1.26.tar.gz
4 #cd ipvsadm-1.26
5 #make && make install

2.安裝keepalived

 1 #wget http://www.keepalived.org/software/keepalived-1.2.13.tar.gz
 2 #tar –zxvf keepalived-1.2.13.tar.gz
 3 #cd keepalived-1.2.13
 4 #./configure
 5 #make && make install
 6 ######### 將keepalived作成啓動服務，方便管理##########
 7 # cp /usr/local/etc/rc.d/init.d/keepalived /etc/init.d/
 8 # cp /usr/local/etc/sysconfig/keepalived /etc/sysconfig/
 9 # mkdir /etc/keepalived/
10 # cp /usr/local/etc/keepalived/keepalived.conf /etc/keepalived/
11 # cp /usr/local/sbin/keepalived /usr/sbin/
12 # service keepalived start | stop

3.開啓路由轉發

1 #vi /etc/sysctl.conf
2 #sysctl –p

4.配置keepalived

1 #vi /etc/keepalived/keepalived.conf

keepalive.conf具體以下：

 1 ! Configuration File for keepalived
 2 
 3 global_defs {
 4    notification_email {
 5       cong.chen@czubitech.com
 6    }
 7    notification_email_from cong.chen@czubitech.com
 8    smtp_server send.one.com
 9    smtp_connect_timeout 30
10    router_id LVS_MASTER #備份服務器上將MASTER改成BACKUP 
11 }
12 
13 vrrp_instance VI_1 {
14     state MASTER #備份服務器上將MASTER改成BACKUP 
15     interface eth0  #該網卡名字須要查看具體服務器的網口
16     virtual_router_id 51
17     priority 100 # 備份服務上將100改成90
18     advert_int 1
19     authentication {
20         auth_type PASS
21         auth_pass 1111
22     }
23     virtual_ipaddress {
24         192.168.0.209
25          #(若是有多個VIP，繼續換行填寫.)
26     }
27 }
28 
29 virtual_server 192.168.0.209 80 {
30     delay_loop 6   #(每隔6秒查詢realserver狀態)
31     lb_algo rr   #(rr 算法)
32     lb_kind DR      #(Direct Route)
33     nat_mask 255.255.255.0
34    # persistence_timeout 50   #(同一IP的鏈接60秒內被分配到同一臺realserver)
35     protocol TCP    #(用TCP協議檢查realserver狀態)
36 
37     real_server 192.168.0.212 80 {
38         weight 1   #(權重)
39         TCP_CHECK {
40             connect_timeout 10    #(10秒無響應超時)
41             nb_get_retry 3
42             delay_before_retry 3
43             connect_port 80
44         }
45      }
46      real_server 192.168.0.227 80 {
47         weight 1
48         TCP_CHECK {
49             connect_timeout 10
50             nb_get_retry 3
51             delay_before_retry 3
52             connect_port 80
53         }
54      }
55     
56 }

須要注意的是{前面須要有空格，我在配置時TCP_CHECK沒有空格致使沒法找到real_server

5.配置realserver(即tomcat+nginx那物理機)

1 #vi /etc/init.d/lvs_real.sh

lvs_real.sh具體配置以下：

 1 # description: Config realserver lo and apply noarp
 2  
 3 SNS_VIP=192.168.0.209
 4  
 5 /etc/rc.d/init.d/functions
 6  
 7 case "$1" in
 8 start)
 9        ifconfig lo:0 $SNS_VIP netmask 255.255.255.255 broadcast $SNS_VIP
10        /sbin/route add -host $SNS_VIP dev lo:0
11        echo "1" >/proc/sys/net/ipv4/conf/lo/arp_ignore
12        echo "2" >/proc/sys/net/ipv4/conf/lo/arp_announce
13        echo "1" >/proc/sys/net/ipv4/conf/all/arp_ignore
14        echo "2" >/proc/sys/net/ipv4/conf/all/arp_announce
15        sysctl -p >/dev/null 2>&1
16        echo "RealServer Start OK"
17  
18        ;;
19 stop)
20        ifconfig lo:0 down
21        route del $SNS_VIP >/dev/null 2>&1
22        echo "0" >/proc/sys/net/ipv4/conf/lo/arp_ignore
23        echo "0" >/proc/sys/net/ipv4/conf/lo/arp_announce
24        echo "0" >/proc/sys/net/ipv4/conf/all/arp_ignore
25        echo "0" >/proc/sys/net/ipv4/conf/all/arp_announce
26        echo "

1 #chmod +x /roo/lvs_real.sh
2 #/etc/init.d/lvs_real.sh start

啓動提示權限不夠，賦權便可

查看網絡ifconfig命令

6.測試lvs+keepalived

Master上輸入命令ip add

VIP綁定成功，ipvsadm能夠查看212和227這2臺物理機鏈接狀況

Backup上輸入命令

 

解析域名，測試訪問，LVS轉發,在master上停掉keepalived

經歷延遲後繼續能夠訪問，同時在backup上輸入命令ip add能夠查看vip已經轉至backup機器上

重啓Master上的服務，又從新切換回來

Jsp集羣中session共享使用memcached，具體配置可參考

http://www.cnblogs.com/cc1688/p/3772156.html