centos7使用haproxy1.7.5實現反向代理負載均衡實戰

使用haproxy實現反向代理負載均衡實戰
環境準備：兩臺虛擬機

# yum install -y gcc glibc gcc-c++ make screen tree lrzsz

node1源碼編譯安裝haproxy

[root@node1 ~]# cd /usr/local/src
[root@node1 src]# wget http://www.haproxy.org/download/1.7/src/haproxy-1.7.5.tar.gz
[root@node1 src]# tar zxf haproxy-1.7.5.tar.gz
[root@node1 src]# cd haproxy-1.7.5
[root@node1 haproxy-1.7.5]# make TARGET=linux2628 PREFIX=/usr/local/haproxy-1.7.5

[root@node1 haproxy-1.7.5]# make install
[root@node1 haproxy-1.7.5]# cp /usr/local/sbin/haproxy /usr/sbin/
[root@node1 haproxy-1.7.5]# haproxy -v
HA-Proxy version 1.7.5 2017/04/03
Copyright 2000-2017 Willy Tarreau <willy@haproxy.org>

編輯Haproxy啓動腳本
[root@node1 haproxy-1.7.5]# cp examples/haproxy.init /etc/init.d/haproxy
[root@node1 haproxy-1.7.5]# chmod 755 /etc/init.d/haproxy

針對配置文件的路徑建立如下文件
[root@node1 haproxy-1.7.5]# useradd -r haproxy
[root@node1 haproxy-1.7.5]# mkdir /etc/haproxy
[root@node1 haproxy-1.7.5]# mkdir /var/lib/haproxy
[root@node1 haproxy-1.7.5]# mkdir /var/run/haproxy

編輯haproxy配置文件，配置log，並啓動

[root@linux-node1 haproxy]# vim /etc/haproxy/haproxy.cfg

global
log 127.0.0.1 local3 info
chroot /var/lib/haproxy
user haproxy
group haproxy
daemon
defaults
log global
mode http
option httplog
option dontlognull
timeout connect 5000
timeout client 50000
timeout server 50000

frontend www_chinasoft_com
mode http
bind *:80
stats uri /haproxy?stats
default_backend www_chinasoft_backend

backend www_chinasoft_backend
option httpchk GET /index.html
balance roundrobin
server node1 192.168.3.140:8080 check inter 2000 rise 3 fall 3 weight 5
server node2 192.168.3.200:8080 check inter 2000 rise 3 fall 3 weight 5
server mini1 192.168.3.12:8080 check inter 2000 rise 3 fall 3 weight 5

 

*******************************************************
global #全局配置，在全部配置段中都生效
log 127.0.0.1 local3 info #記錄日誌
chroot /var/lib/haproxy
user haproxy
group haproxy
daemon
defaults #默認配置，能夠被前端和後端繼承
log global #使用global的log設置
mode http #使用http模式，也能夠使用tcp模式
option httplog #啓動http請求的log
option dontlognull #在日誌中不記錄空鏈接（空鏈接：健康檢查的連接）
timeout connect 5000 #長鏈接超時時間
timeout client 50000 #客戶端鏈接超時
timeout server 50000 #RS鏈接超時

frontend www_chinasoft_com #前端配置 + 一個配置段的名字（最好不要亂寫，和項目直接相關最佳）
mode http #使用http模式，也能夠使用tcp模式
bind *:80 #監聽80端口
stats uri /haproxy?stats #狀態頁面dashboard
default_backend www_chinasoft_com_backend #對應的backend名稱
backend www_chinasoft_com_backend #對應的frontend的default_backend
#source cookie SERVERID
option httpchk GET /index.html #檢測url
balance roundrobin #使用rr負載均衡方式
server node1 192.168.3.140:8080 check inter 2000 rise 3 fall 3 weight 5
server node2 192.168.3.200:8080 check inter 2000 rise 3 fall 3 weight 5
server mini1 192.168.3.12:8080 check inter 2000 rise 3 fall 3 weight 1 #RS健康檢測時間間隔2秒，重試三次，失敗三次不可用，權重1
*******************************************************

打開haproxy的日誌

# vim /etc/rsyslog.conf

15 $ModLoad imudp #打開註釋
16 $UDPServerRun 514 #打開註釋
74 local3.* /var/log/haproxy.log #local3的路徑

[root@node1 haproxy-1.7.5]# /etc/init.d/haproxy start
[root@node1 haproxy-1.7.5]# systemctl restart rsyslog.service
[root@node1 haproxy-1.7.5]# touch /var/log/haproxy.log
[root@node1 haproxy-1.7.5]# chown -R haproxy.haproxy /var/log/haproxy.log
[root@node1 haproxy-1.7.5]# /etc/init.d/haproxy restart
Restarting haproxy (via systemctl): [ OK ]
[root@node1 haproxy-1.7.5]# tail -f /var/log/haproxy.log
May 4 03:23:50 localhost haproxy[5793]: Stopping frontend www_chinasoft_com in 0 ms.
May 4 03:23:50 localhost haproxy[5793]: Stopping backend www_chinasoft_backend in 0 ms.
May 4 03:23:50 localhost haproxy[5793]: Proxy www_chinasoft_com stopped (FE: 0 conns, BE: 0 conns).
May 4 03:23:50 localhost haproxy[5793]: Proxy www_chinasoft_backend stopped (FE: 0 conns, BE: 0 conns).
May 4 03:23:50 localhost haproxy[5848]: Proxy www_chinasoft_com started.

[root@node1 ~]# sed -i 's/index.html/chinasoft.html/g' /etc/haproxy/haproxy.cfg
[root@node1 ~]# /etc/init.d/haproxy restart
Restarting haproxy (via systemctl): [ OK ]
[root@node1 ~]#
Message from syslogd@localhost at May 4 12:20:49 ...
haproxy[6076]: backend www_chinasoft_backend has no server available!

下面是檢測url和uri的幾種方式

option httpchk
option httpchk <uri>
option httpchk <method> <uri>
option httpchk <method> <uri> <version>

更改配置文件獲取客戶端的真實ip
在banckend配置段加入一個option

option forwardfor header X-REAL-IP #X-REAL-IP是自定義的一個名稱

經過acl設置虛擬主機，一個前端能夠對應多個後端，而實際生產環境建議一個frontend對應一個backend，並重載（生產不建議restart，restart會斷開現有連接）

[root@node1 ~]# cat /etc/haproxy/haproxy.cfg 
global
log 127.0.0.1 local3 info
chroot /var/lib/haproxy
user haproxy
group haproxy
daemon
defaults
log global
mode http
option httplog
option dontlognull
timeout connect 5000
timeout client 50000
timeout server 50000

frontend www_chinasoft_com
mode http
bind *:80
stats uri /haproxy?stats
default_backend www_chinasoft_backend    # 默認的backend
acl other_chinasoft_com hdr_end(host) other.chinasoft.com    # other_chinasoft_com：給此acl起一個名字；hdr(host)：固定格式，用來識別host，若是沒有匹配到acl，即訪問default的bankcend    
use_backend other_chinasoft_com_backend if other_chinasoft_com

backend www_chinasoft_backend
option forwardfor header X-REAL-IP
option httpchk GET /index.html
balance roundrobin
server node1 192.168.3.140:8080 check inter 2000 rise 3 fall 3 weight 5

backend other_chinasoft_com_backend
option forwardfor header X-REAL-IP
option httpchk GET /index.html
balance roundrobin
server node2 192.168.3.200:8080 check inter 2000 rise 3 fall 3 weight 1
server mini1 192.168.3.12:8080 check inter 2000 rise 3 fall 3 weight 1

 

在本地電腦使用host解析

192.168.3.140 www.chinasoft.com 
192.168.3.140 other.chinasoft.com 
192.168.3.12 other.chinasoft.com

經過瀏覽器訪問不一樣的域名

在fortend添加acl，根據靜態文件，設置不一樣的backend（相似於location），註釋的兩行和前兩行意義相同，分別是經過url正則匹配和url的後綴匹配

frontend www_chinasoft_com
mode http
bind *:80
stats uri /haproxy?stats
default_backend www_chinasoft_backend
acl other_chinasoft_com hdr_end(host) other.chinasoft.com
use_backend other_chinasoft_com_backend if other_chinasoft_com
#acl is_static_reg url_reg /*.(css|jpg|jpeg|png|js|gif)$
#use_backend other_chinasoft_com_backend if is_static_reg
acl is_static_path path_end .gif .png .css .jpg .jpeg
use_backend other_chinasoft_com_backend if is_static_path

[root@mini1 ~]# echo 'this is static test page <br> 192.168.3.12' > /var/www/html/hello.js
[root@mini3 ~]# echo 'this is static test page <br> 192.168.3.200' > /var/www/html/hello.js
[root@node1 html]# /etc/init.d/haproxy restart
Restarting haproxy (via systemctl): [ OK ]

其餘形式的acl，正則或者UA（能夠理解爲nginx的location）,更多形式的acl，請參考：http://cbonte.github.io/haproxy-dconv/configuration-1.6.html#acl

acl is_do_path url_reg /chuck.do
use_backend other_chuck-blog_com_backend if is_do_path
acl is_UA_path hdr_reg(User-Agent) -i andriod
use_backend other_chuck-blog_com_backend if is_UA_path

4、haproxy的動態維護
在配置文件添加socket

[root@node1 html]# head -8 /etc/haproxy/haproxy.cfg
global
log 127.0.0.1 local3 info
chroot /var/lib/haproxy
user haproxy
group haproxy
daemon
stats socket /var/lib/haproxy/haproxy.sock mode 600 level admin # 指定socket文件路徑，權限，管理級別
stats timeout 2m # 指定超時時間
[root@node1 html]# /etc/init.d/haproxy restart
Restarting haproxy (via systemctl): [ OK ]

[root@node1 html]# ll /var/lib/haproxy/
total 0
srw------- 1 root root 0 May 4 14:38 haproxy.sock

安裝socat

[root@node1 ~]# yum install -y socat

使用help查看socat的事情

[root@node1 ~]# echo 'help' | socat stdio /var/lib/haproxy/haproxy.sock 
Unknown command. Please enter one of the following commands only :
help : this message
prompt : toggle interactive mode with prompt
quit : disconnect
disable agent : disable agent checks (use 'set server' instead)
disable health : disable health checks (use 'set server' instead)
disable server : disable a server for maintenance (use 'set server' instead)
enable agent : enable agent checks (use 'set server' instead)
enable health : enable health checks (use 'set server' instead)
enable server : enable a disabled server (use 'set server' instead)
set maxconn server : change a server's maxconn setting
set server : change a server's state, weight or address
get weight : report a server's current weight
set weight : change a server's weight (deprecated)
disable frontend : temporarily disable specific frontend
enable frontend : re-enable specific frontend
set maxconn frontend : change a frontend's maxconn setting
show servers state [id]: dump volatile server information (for backend <id>)
show backend : list backends in the current running config
shutdown frontend : stop a specific frontend
clear table : remove an entry from a table
set table [id] : update or create a table entry's data
show table [id]: report table usage stats or dump this table's contents
show errors : report last request and response errors for each proxy
clear counters : clear max statistics counters (add 'all' for all counters)
show info : report information about the running process
show stat : report counters for each proxy and server
show sess [id] : report the list of current sessions or dump this session
shutdown session : kill a specific session
shutdown sessions server : kill sessions on a server
show pools : report information about the memory pools usage
add acl : add acl entry
clear acl <id> : clear the content of this acl
del acl : delete acl entry
get acl : report the patterns matching a sample for an ACL
show acl [id] : report available acls or dump an acl's contents
add map : add map entry
clear map <id> : clear the content of this map
del map : delete map entry
get map : report the keys and values matching a sample for a map
set map : modify map entry
show map [id] : report available maps or dump a map's contents
show stat resolvers [id]: dumps counters from all resolvers section and
associated name servers
set maxconn global : change the per-process maxconn setting
set rate-limit : change a rate limiting value
set timeout : change a timeout setting
show env [var] : dump environment variables known to the process

 

查看info信息，內容值能夠利用來監控

[root@node1 ~]# echo "show info" |socat stdio /var/lib/haproxy/haproxy.sock 
Name: HAProxy
Version: 1.7.5
Release_date: 2017/04/03
Nbproc: 1
Process_num: 1
Pid: 6902
Uptime: 0d 0h03m26s
Uptime_sec: 206
Memmax_MB: 0
PoolAlloc_MB: 0
PoolUsed_MB: 0
PoolFailed: 0
Ulimit-n: 4034
Maxsock: 4034
Maxconn: 2000
Hard_maxconn: 2000
CurrConns: 0
CumConns: 3
CumReq: 3
Maxpipes: 0
PipesUsed: 0
PipesFree: 0
ConnRate: 0
ConnRateLimit: 0
MaxConnRate: 0
SessRate: 0
SessRateLimit: 0
MaxSessRate: 0
CompressBpsIn: 0
CompressBpsOut: 0
CompressBpsRateLim: 0
Tasks: 9
Run_queue: 1
Idle_pct: 100
node: node1

關閉linux-node2主機

[root@node1 ~]# echo "disable server other_chinasoft_com_backend/node2" |socat stdio /var/lib/haproxy/haproxy.sock

能夠看到node2進入了維護（maintain）狀態

 

打開node2主機(只對現有已經寫到配置文件中的server生效，不能用來新增節點)
[root@node1 ~]# echo "enable server other_chinasoft_com_backend/node2" |socat stdio /var/lib/haproxy/haproxy.sock

 

5、生產環境遇到的問題
haproxy的本地端口可能用盡，解決方案以下4條
1）更改local的端口範圍,調整內核參數
[root@node1 ~]# cat /proc/sys/net/ipv4/ip_local_port_range
32768 60999

2）調整timewait的端口複用，設置爲1

[root@node1 ~]# cat /proc/sys/net/ipv4/tcp_tw_reuse
0
3)縮短tcp_wait的時間，不建議修改

[root@node1 ~]# cat /proc/sys/net/ipv4/tcp_fin_timeout
60
4)終極方案：增長爲多個ip，天然端口數就夠了