Nginx配合keepalived實現LVS負載均衡--生產環境實際案例

       前言：因爲服務器資源有限，keepalived未配主主、只配了一個VIP。本人在香港機房生產環境部署的實際案例，從2013年9月開始運行，服務從未掛過。因爲涉及到生產環境，系統展現就不貼出來了。但願你們給予意見！

Webserver部署的是lnmp環境，採用我本身寫的一鍵編譯安裝腳本，安裝目錄在/data/webserver/。如你們想借鑑個人編譯安裝腳本，請留言向我索取。

1、架構規劃

一、服務器IP地址規劃

VIP：192.168.1.6

real_server1:192.168.1.7

real_server2:192.168.1.8

WebServer1:192.168.1.9

WebServer2:192.168.1.10

memcache: 192.168.1.11

mysql主：192.168.1.12

mysql從：192.168.1.2

二、服務器操做系統

所使用的操做系統均爲CentOS 6.4

三、網絡拓撲圖

2、負載服務器配置 

1.LVS主服務器配置

安裝yum install keepalived ipvsadm

chkconfig keepalived on

vim /etc/ keepalived/keepalived.conf

global_defs {
	notification_email {
        	yunwei@admin.com
   	}
	notification_email_from 
	smtp_server 127.0.0.1
	#smtp_connect_timeout 30
	router_id LVS_DEVEL_1
}
vrrp_instance VI_1 {
	state MASTER
	interface eth1
	virtual_router_id 51
	priority 100
	advert_int 1
	authentication {
		auth_type PASS
		auth_pass 1111
    	}
	virtual_ipaddress {
        	192.168.1.6
    	}
}
virtual_server 192.168.1.6 80 {
	delay_loop 6
	#lb_algo wrr
	lb_algo rr
	lb_kind DR
	#persistence_timeout 300
	protocol TCP
real_server 192.168.1.7 80 {
	weight 3
        TCP_CHECK {
        connect_timeout 10
	nb_get_retry 3
	delay_before_retry 3
	connect_port 80
        }
}
real_server 192.168.1.8 80 {
        weight 3 
        TCP_CHECK {
        connect_timeout 10
        nb_get_retry 3
        delay_before_retry 3
        connect_port 80
        }
}
}

啓動keepalived進程   /etc/init.d/keepalived start

查看本機VIP      

2.LVS前端服務器配置（real_server）

2.1   vim /usr/local/bin/lvs_real

VIP=192.168.1.6
case "$1" in
        start)
                echo " start LVS of REALServer"
                /sbin/ifconfig lo:0 $VIP broadcast $VIP netmask 255.255.255.255 up
                echo "1" >/proc/sys/net/ipv4/conf/lo/arp_ignore
                echo "2" >/proc/sys/net/ipv4/conf/lo/arp_announce
                echo "1" >/proc/sys/net/ipv4/conf/all/arp_ignore
                echo "2" >/proc/sys/net/ipv4/conf/all/arp_announce
                ;;
        stop)
                /sbin/ifconfig lo:0 down
                echo "close LVS Directorserver"
                echo "0" >/proc/sys/net/ipv4/conf/lo/arp_ignore
                echo "0" >/proc/sys/net/ipv4/conf/lo/arp_announce
                echo "0" >/proc/sys/net/ipv4/conf/all/arp_ignore
                echo "0" >/proc/sys/net/ipv4/conf/all/arp_announce
                ;;
        *)
                echo "Usage: $0 {start|stop}"
                exit 1
esac

啓動lvs前端真實IP：  lvs_real start

2.2  編譯安裝nginx，此處省略（使用我本身寫的一鍵編譯安裝腳本安裝）

2.3  LVS前端服務器nginx配置

server {
                           listen 80;
                           server_name sandy.com;
                           location / {
                           proxy_redirect off;
                           proxy_set_header Host $host;
                           proxy_set_header X-Real-IP $remote_addr;
                           proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
                           proxy_pass http://sandy/;
                           }
                           location /coocaa_status {
                           stub_status on;
                           access_log /data/log/nginx_status.log;
                           auth_basic "NginxStatus";
                           allow 192.168.0.58;
                           deny all;
                           }
                           access_log /data/log/access.log access;
                           error_log /data/log/error.log warn;
                       }
                  upstream sandy {
                           sticky;
                           server 192.168.1.9:80;   #後端webserver1內網IP
                           server 192.168.1.10:80;  #後端webserver2內網IP
                           }
                 }

3、Webserver服務器配置

1.編譯安裝nginx、php，此處省略（使用我本身寫的一鍵編譯安裝腳本安裝）

2.webserver服務器nginx配置，以下

user nginx;
worker_processes 8;
worker_cpu_affinity 00000001 00000010 00000100 00001000 00010000 00100000 01000000 10000000;
error_log /data/log/error.log debug;
pid /data/webserver/nginx/logs/nginx.pid;
worker_rlimit_nofile 65535;
   events {
     use epoll;
     worker_connections 65535;
          }
    http {
         include mime.types;
         default_type application/octet-stream;
         charset utf-8;
         server_names_hash_bucket_size 128;
         client_header_buffer_size 32k;
         large_client_header_buffers 4 32k;
         client_max_body_size 50m;
         sendfile on;
         tcp_nopush on;
         keepalive_timeout 60;
         tcp_nodelay on;
         server_tokens off;
         fastcgi_connect_timeout 300;
         fastcgi_send_timeout 3000;
         fastcgi_read_timeout 3000;
         fastcgi_buffer_size 64k;
         fastcgi_buffers 4 64k;
         fastcgi_busy_buffers_size 128k;
         fastcgi_temp_file_write_size 128k;
         gzip on;
         gzip_min_length 1k;
         gzip_buffers 4  16k;
         gzip_http_version 1.0;
         gzip_comp_level 2;
         gzip_types text/plain application/x-javascript text/css application/xml;
         gzip_vary on;
         log_format access '\$remote_addr - \$remote_user [\$time_local] "\$request" ' '\$status \$body_bytes_sent "\$http_referer" '
         '"\$http_user_agent" "\$http_x_forwarded_for" - "\$http_soapaction"';
            server
                {
                   listen 80;
                   server_name 0.0.0.0; #webserver外網ip
                   index index.html index.php;
                   root   /data/www/cloudservice/;
                    #autoindex on;
                   location = /favicon.ico {
                   log_not_found off;
                   access_log off;
                   }
                  location /Framework/tvos/ {
                         try_files \$uri \$uri/ /Framework/tvos/index.php?\$args;
                    }
                  # nginx status
                  location /coocaa_status {
                  stub_status on;
                  access_log /data/log/nginx_status.log;
                  auth_basic "NginxStatus";
                  allow 192.168.0.58;
                  deny all;
                  }
                  location ~ \.php$ {
                  root           /data/www/cloudservice;
                  fastcgi_pass   127.0.0.1:9000;
                  fastcgi_index  index.php;
                  fastcgi_param  SCRIPT_FILENAME  /data/www/cloudservice/$fastcgi_script_name;
                  include        fastcgi_params;
                  }
                 location ~ .*\.(gif|jpg|jpeg|png|bmp|swf)\$
                 {
                 expires 30d;
                 }
                 location ~ .*\.(js|css)?\$
                 {
                 expires 1h;
                 }
             access_log /data/log/access.log access;
           }
    }

3.PHP-fpm配置以下：

[global]
pid = /data/webserver/php/var/run/php-fpm.pid
error_log = /data/log/php-fpm.error.log
log_level = error
[www]
listen = 127.0.0.1:9000
user = nginx
group = nginx
pm = dynamic
;pm= static
pm.max_children = 200
pm.start_servers = 32
pm.min_spare_servers = 32
pm.max_spare_servers = 200
pm.max_requests = 10240
request_terminate_timeout = 900
php_admin_flag[log_errors] = on
php_value[session.save_handler] = files
php_value[session.save_path] = /tmp
catch_workers_output = yes
access.log = /data/log/php-fpm.access.log
request_slowlog_timeout = 5
slowlog = /data/log/www-slow.log
pm.status_path = /status
rlimit_files = 65535

4、Memcache服務器部署

1.安裝memcached

yum -y install libevent libevent-devel make gcc gcc-c++ build-essential make

memcached (){
 pwd
 wget -c http://42.120.20.242/memcached-1.4.15.tar.gz
 tar zxvf memcached-1.4.15.tar.gz
 cd memcached-1.4.15/
 ./configure --with-libevent=/usr/local --prefix=/usr/local/memcached
 make &&make install
 cd ../

2.開啓memcached實例，本項目中開了兩個實例。以下

memcached -d -m 2500 -u root -l 192.168.1.13 -p 11211 -c 20480 -P /tmp/memcached.pid –v

memcached -d -m 2500 -u root -l 192.168.1.13 -p 11212 -c 20480 -P /tmp/memcached2.pid –v

5、Mysql服務器配置

1.編譯安裝mysql5.5.29，此處省略（使用我本身寫的一鍵編譯安裝腳本安裝）

2.mysql配置文件以下：

[client]
port            = 3306
socket          = /tmp/mysqld.sock
[mysqld]
user       = mysql
port            = 3306
socket          = /tmp/mysqld.sock
skip-external-locking  
pid-file=/data/mysql/mysqld.pid
datadir = /data/mysql
log-error=/data/log/error.log
key_buffer_size = 3G
max_allowed_packet = 256M
thread_stack =1M
thread_cache_size = 32
sort_buffer_size = 2M
join_buffer_size =4M
read_buffer_size = 2M
net_buffer_length = 16M
read_rnd_buffer_size = 8M 
myisam_sort_buffer_size = 64M
myisam_max_sort_file_size = 10G 
myisam_repair_threads = 1 
myisam-recover-options=DEFAULT
tmp_table_size = 256M
max_heap_table_size = 4G 
myisam-recover  = BACKUP 
max_connections =3000
max_connect_errors = 100000
query_cache_limit = 2M
query_cache_size =512M
query_cache_type = 1
query_cache_min_res_unit = 2k
default-storage-engine = Innodb
table_open_cache = 1000
transaction_isolation = READ-COMMITTED
innodb_additional_mem_pool_size = 16M    
innodb_autoextend_increment = 8M      
innodb_buffer_pool_size = 4G         
innodb_checksums = 1                 
innodb_commit_concurrency = 0        
innodb_concurrency_tickets = 500      
innodb_doublewrite = 1         
innodb_fast_shutdown = 1       
innodb_file_io_threads = 4     
innodb_file_per_table = 1           
innodb_flush_log_at_trx_commit = 0   
innodb_flush_method = O_DIRECT
innodb_lock_wait_timeout = 30    
innodb_locks_unsafe_for_binlog = 0    
innodb_log_buffer_size = 3M         
innodb_max_dirty_pages_pct = 80       
innodb_open_files = 3000               
innodb_data_file_path=ibdata1:50M:autoextend
innodb_log_file_size = 800M
innodb_log_files_in_group = 2
log_slow_queries  = /data/log/slow.log
long_query_time = 2
log-queries-not-using-indexes  
log_bin_trust_function_creators = 1
expire_logs_days        = 10
max_binlog_size         = 100M
binlog_cache_size = 256K
binlog_stmt_cache_size=128K
max_binlog_cache_size = 512M
max_binlog_stmt_cache_size=512M
binlog_format=mixed
skip-name-resolve
server-id               = 1
log_bin                 = /data/mysql/mysql-bin.log
thread_concurrency= 32   
innodb_thread_concurrency = 32
character-set-server = utf8
innodb_write_io_threads =32
innodb_read_io_threads= 32
innodb_max_dirty_pages_pct= 90
#wait_timeout=600
#interactive_timeout = 600
[mysqldump]
quick
max_allowed_packet = 16M
[mysql]
no-auto-rehash
[myisamchk]
key_buffer_size = 20M
sort_buffer_size = 20M
read_buffer = 2M
write_buffer = 2M
[mysqlhotcopy]
interactive-timeout

3.啓動mysql進程

/etc/init.d/mysql start

至此，該項目完成部署，解析域名在VIP，便可。