Haproxy壓測

時間 2019-12-11

原文原文鏈接

目的：測試Haproxy壓測狀況java

環境：linux

Ha服務器：8核16G虛機，後端6個2核4G，壓測客戶端3個2核4Ggit

安裝和優化：github

1、Haproxyweb

#cd /opt/soft
#wget http://www.haproxy.org/download/1.8/src/haproxy-1.8.14.tar.gz
#tar zxvf haproxy-1.8.14.tar.gz
#cd haproxy-1.8.14
安裝：
#make TARGET=linux2628  ARCH=X86_64 PREFIX=/usr/local/haproxy
#make install PREFIX=/usr/local/haproxy
添加用戶：
#useradd -s /sbin/nologin -M haproxy
id haproxy
#mkdir /etc/haproxy/

配置systemctl
#cp /opt/soft/haproxy-1.8.14/examples/haproxy.init /etc/init.d/haproxy
#chmod +x /etc/init.d/haproxy
#vi /etc/init.d/haproxy

/etc/init.d/haproxyredis

#!/bin/sh
#
# chkconfig: - 85 15
# description: HA-Proxy is a TCP/HTTP reverse proxy which is particularly suited \
#              for high availability environments.
# processname: haproxy
# config: /etc/haproxy/haproxy.cfg
# pidfile: /var/run/haproxy.pid

# Source function library.
if [ -f /etc/init.d/functions ]; then
  . /etc/init.d/functions
elif [ -f /etc/rc.d/init.d/functions ] ; then
  . /etc/rc.d/init.d/functions
else
  exit 0
fi

# Source networking configuration.
. /etc/sysconfig/network

# Check that networking is up.
[ ${NETWORKING} = "no" ] && exit 0

# This is our service name
BASENAME=`basename $0`
if [ -L $0 ]; then
  BASENAME=`find $0 -name $BASENAME -printf %l`
  BASENAME=`basename $BASENAME`
fi

BIN=/usr/sbin/$BASENAME

CFG=/etc/$BASENAME/$BASENAME.cfg
[ -f $CFG ] || exit 1

PIDFILE=/var/run/$BASENAME.pid
LOCKFILE=/var/lock/subsys/$BASENAME

RETVAL=0

start() {
  quiet_check
  if [ $? -ne 0 ]; then
    echo "Errors found in configuration file, check it with '$BASENAME check'."
    return 1
  fi

  echo -n "Starting $BASENAME: "
  daemon $BIN -D -f $CFG -p $PIDFILE
  RETVAL=$?
  echo
  [ $RETVAL -eq 0 ] && touch $LOCKFILE
  return $RETVAL
}

stop() {
  echo -n "Shutting down $BASENAME: "
  killproc $BASENAME -USR1
  RETVAL=$?
  echo
  [ $RETVAL -eq 0 ] && rm -f $LOCKFILE
  [ $RETVAL -eq 0 ] && rm -f $PIDFILE
  return $RETVAL
}

restart() {
  quiet_check
  if [ $? -ne 0 ]; then
    echo "Errors found in configuration file, check it with '$BASENAME check'."
    return 1
  fi
  stop
  start
}

reload() {
  if ! [ -s $PIDFILE ]; then
    return 0
  fi

  quiet_check
  if [ $? -ne 0 ]; then
    echo "Errors found in configuration file, check it with '$BASENAME check'."
    return 1
  fi
  $BIN -D -f $CFG -p $PIDFILE -sf $(cat $PIDFILE)
}

check() {
  $BIN -c -q -V -f $CFG
}

quiet_check() {
  $BIN -c -q -f $CFG
}

rhstatus() {
  status $BASENAME
}

condrestart() {
  [ -e $LOCKFILE ] && restart || :
}

# See how we were called.
case "$1" in
  start)
    start
    ;;
  stop)
    stop
    ;;
  restart)
    restart
    ;;
  reload)
    reload
    ;;
  condrestart)
    condrestart
    ;;
  status)
    rhstatus
    ;;
  check)
    check
    ;;
  *)
    echo $"Usage: $BASENAME {start|stop|restart|reload|condrestart|status|check}"
    exit 1
esac
 
exit $?

　　Haproxy配置：spring

global
group root
    user root
    daemon
    nbproc 7
    log 127.0.0.1 local3
    pidfile /var/run/haproxy/pid/haproxy.pid
    ulimit-n 1000000 ##這裏增大文件打開數
    max-spread-checks  1000ms
    maxconn     3000000 ##這裏放開限制
    maxconnrate 3000000
    maxsessrate 3000000
    #maxsslconn  3000000
    #maxsslrate  3000000
    spread-checks 20
    stats timeout 60s
    stats maxconn 500
    stats socket /var/run/haproxy/sock/haproxy.sock mode 600 level admin process 1

#---------------------------------------------------------------------
# common defaults that all the 'listen' and 'backend' sections will
# use if not designated in their block
#---------------------------------------------------------------------
defaults
    mode http
    #mode tcp
    maxconn     3000000 ##這裏最好與上面一致，不然默認限制2000，找了很久
    option  abortonclose
    option redispatch
    option forwardfor
    balance roundrobin
    log 127.0.0.1 local3 err
    retries 3
    option clitcpka
    option srvtcpka
#---------------------------------------------------------------------
# main frontend which proxys to the backends
#---------------------------------------------------------------------
listen 172.28.20.102:81
    balance roundrobin
    bind *:81  
    option tcp-check
    option httplog
    #option dontlognull
	#test
    timeout http-keep-alive 70s
    timeout http-request    70s 
    timeout connect 100000
    timeout queue   100000
    timeout client  100000
    timeout server  100000
    timeout check   100000
    cookie SERVERID insert indirect nocache
    server  105_1 172.28.20.105:8282 check
    server  105_2 172.28.20.105:8283 check
    server  105_3 172.28.20.105:8284 check
    server  105_4 172.28.20.105:8285 check
    server  105_5 172.28.20.105:8286 check
    server  105_6 172.28.20.105:8287 check
    server  108_1 172.28.20.108:8282 check
    server  108_2 172.28.20.108:8283 check
    server  108_3 172.28.20.108:8284 check
    server  108_4 172.28.20.108:8285 check
    server  108_5 172.28.20.108:8286 check
    server  108_6 172.28.20.108:8287 check
    server  167_1 172.28.16.167:8282 check
    server  167_2 172.28.16.167:8283 check
    server  167_3 172.28.16.167:8284 check
    server  168_1 172.28.16.168:8282 check
    server  168_2 172.28.16.168:8283 check
    server  168_3 172.28.16.168:8284 check
    server  169_1 172.28.16.169:8282 check
    server  169_2 172.28.16.169:8283 check
    server  169_3 172.28.16.169:8284 check
    server  170_1 172.28.16.170:8282 check
    server  170_2 172.28.16.170:8283 check
    server  170_3 172.28.16.170:8284 check

　#內核參數優化（haproxy，後端服務器，客戶端均要）apache

#vi /etc/sysctl.conf後端

### 系統中所容許的文件句柄的最大數目
fs.file-max = 12553500
### 單個進程所容許的文件句柄的最大數目
fs.nr_open = 12453500
### 內核容許使用的共享內存大 Controls the maximum number of shared memory segments, in pages
kernel.shmall = 4294967296
###單個共享內存段的最大值 Controls the maximum shared segment size, in bytes
kernel.shmmax = 68719476736
### 內核中消息隊列中消息的最大值 Controls the maximum size of a message, in bytes
kernel.msgmax = 65536
### 系統救援工具
kernel.sysrq = 0
### 在每一個網絡接口接收數據包的速率比內核處理這些包的速率快時，容許送到緩存隊列的數據包的最大數目
net.core.netdev_max_backlog = 2000000
### 默認的TCP數據接收窗口大小（字節）
net.core.rmem_default = 699040
### 最大的TCP數據接收窗口（字節）
net.core.rmem_max = 50331648
### 默認的TCP數據發送窗口大小（字節）
net.core.wmem_default = 131072
### 最大的TCP數據發送窗口（字節）
net.core.wmem_max = 33554432
### 定義了系統中每個端口最大的監聽隊列的長度，這是個全局的參數
net.core.somaxconn = 65535

### TCP/UDP協議容許使用的本地端口號
net.ipv4.ip_local_port_range = 15000 65000
net.ipv4.ip_nonlocal_bind = 1
### 對於本端斷開的socket鏈接，TCP保持在FIN-WAIT-2狀態的時間（秒）
net.ipv4.tcp_fin_timeout = 7
### TCP發送keepalive探測消息的間隔時間（秒），用於確認TCP鏈接是否有效
net.ipv4.tcp_keepalive_time = 300
net.ipv4.tcp_max_orphans = 3276800
### 對於還未得到對方確認的鏈接請求，可保存在隊列中的最大數目
net.ipv4.tcp_max_syn_backlog = 655360
net.ipv4.tcp_max_tw_buckets = 6000000
### 肯定TCP棧應該如何反映內存使用，每一個值的單位都是內存頁（一般是4KB）
### 第一個值是內存使用的下限；第二個值是內存壓力模式開始對緩衝區使用應用壓力的上限；第三個值是內存使用的上限.
net.ipv4.tcp_mem = 94500000 915000000 927000000
### 爲自動調優定義socket使用的內存。
### 第一個值是爲socket接收緩衝區分配的最少字節數；
### 第二個值是默認值（該值會被rmem_default覆蓋），緩衝區在系統負載不重的狀況下能夠增加到這個值；
### 第三個值是接收緩衝區空間的最大字節數（該值會被rmem_max覆蓋）
net.ipv4.tcp_rmem = 32768 699040 50331648
### 爲自動調優定義socket使用的內存。
### 第一個值是爲socket發送緩衝區分配的最少字節數；
### 第二個值是默認值（該值會被wmem_default覆蓋），緩衝區在系統負載不重的狀況下能夠增加到這個值；
### 第三個值是發送緩衝區空間的最大字節數（該值會被wmem_max覆蓋）
net.ipv4.tcp_wmem = 32768 131072 33554432
net.ipv4.tcp_slow_start_after_idle = 0
net.ipv4.tcp_synack_retries = 2
### 表示是否打開TCP同步標籤（syncookie），同步標籤能夠防止一個套接字在有過多試圖鏈接到達時引發過載
### 內核必須打開了CONFIG_SYN_COOKIES項進行編譯，
net.ipv4.tcp_syncookies = 1
net.ipv4.tcp_syn_retries = 2
### 表示開啓TCP鏈接中TIME-WAIT sockets的快速回收，默認爲0，表示關閉
net.ipv4.tcp_tw_recycle = 1
### 容許將TIME-WAIT sockets從新用於新的TCP鏈接，默認爲0，表示關閉
net.ipv4.tcp_tw_reuse = 1
### 啓用RFC 1323定義的window scaling，要支持超過64KB的TCP窗口，必須啓用該值（1表示啓用），
### TCP窗口最大至1GB，TCP鏈接雙方都啓用時才生效，默認爲1
net.ipv4.tcp_window_scaling = 1
### 最大限度使用物理內存
vm.swappiness = 0

生效下瀏覽器

#/sbin/sysctl -p

vi /etc/security/limits.conf

* soft nofile 65535
* hard nofile 65535
* soft nproc 65535
* hard nproc 65535

2、後端服務端

服務器優化，參考haproxy修改sysctl.conf和limits.conf

簡單寫了個springboot web端，建立springboot工程，加入如下兩個類，編譯成jar包。每一個服務器運行3個實例

啓動腳本runhellos.sh

#!/bin/sh
counter=0
while [ $counter -le 2 ]
do
        port=$((8282+$counter))
        java -jar helloworld-0.0.1-SNAPSHOT.jar --server.port=$port 2>&1 >/dev/null &
        ((counter++))
done
echo "all servers create"

　　關閉：pkill java

主程序HelloController.class

：

package com.wwd;

import javax.servlet.http.HttpServletRequest;

import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.boot.context.embedded.ConfigurableEmbeddedServletContainer;
import org.springframework.boot.context.embedded.EmbeddedServletContainerCustomizer;
import org.springframework.stereotype.Controller;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.ResponseBody;

@Controller
@EnableAutoConfiguration
@SpringBootApplication
public class HelloController {

    @RequestMapping("/hello")
    @ResponseBody
    String home(HttpServletRequest request) {
    	String sleep = request.getHeader("sleep");
    	if(!"".equals(sleep) && null != sleep){
    		System.out.println("sleeptime:"+(long)((Math.random()*Integer.parseInt(sleep))));
	    	try {
				Thread.sleep((long)((Math.random()*Integer.parseInt(sleep))));
			} catch (NumberFormatException e) {
				e.printStackTrace();
			} catch (InterruptedException e) {
				e.printStackTrace();
			}
    	}
    	
        return "Hello ,spring boot!";
    }

    public static void main(String[] args) throws Exception {
        SpringApplication.run(HelloController.class, args);
        //運行以後在瀏覽器中訪問：http://localhost:8080/hello
    }


    
   /* @Override  
    public void customize(ConfigurableEmbeddedServletContainer configurableEmbeddedServletContainer) {  
  
        
    }*/  
    
}

　tomcat參數調整（由於要保持鏈接，設置的超時鏈接時間較長，否則壓不上去。）

/*
 * Project: helloworld
 * 
 * File Created at 2018年10月11日
 * 
 * Copyright 2016 CMCC Corporation Limited.
 * All rights reserved.
 *
 * This software is the confidential and proprietary information of
 * ZYHY Company. ("Confidential Information").  You shall not
 * disclose such Confidential Information and shall use it only in
 * accordance with the terms of the license.
 */
package com.wwd;

import org.apache.catalina.connector.Connector;
import org.apache.coyote.http11.Http11NioProtocol;
import org.springframework.boot.context.embedded.EmbeddedServletContainer;
import org.springframework.boot.context.embedded.ServletContextInitializer;
import org.springframework.boot.context.embedded.tomcat.TomcatEmbeddedServletContainerFactory;
import org.springframework.stereotype.Component;

@Component()
public class MyEmbeddedServletContainerFactory extends TomcatEmbeddedServletContainerFactory {
	public EmbeddedServletContainer getEmbeddedServletContainer(ServletContextInitializer... initializers) {
		// 設置端口
//		this.setPort(8081);
		return super.getEmbeddedServletContainer(initializers);
	}

	protected void customizeConnector(Connector connector) {
		super.customizeConnector(connector);
		Http11NioProtocol protocol = (Http11NioProtocol) connector.getProtocolHandler();
		// 設置最大鏈接數
		protocol.setMaxConnections(8000);
		// 設置最大線程數
		protocol.setMaxThreads(400);
		protocol.setConnectionTimeout(3000000);
	}
}

/**
 * Revision history
 * -------------------------------------------------------------------------
 * 
 * Date Author Note
 * -------------------------------------------------------------------------
 * 2018年10月11日 WWD create
 */

3、壓測工具

開始用jmeter和ab去壓測，只能測試tps信息，鏈接數壓不上去。

ab -c 300 -n 100000 http://172.28.20.102:81/hello

引入貝吉塔Vegeta

wget https://github.com/tsenart/vegeta/releases/download/v8.0.0/vegeta-8.0.0-linux-amd64.tar.gz
tar zxvf vegeta-8.0.0-linux-amd64.tar.gz
ln -s /opt/soft/vegeta /usr/local/sbin/vegeta

　2核4G，壓測命令和參數以下，

echo "GET http://172.28.20.102:81/hello" | vegeta -cpus=2 attack -duration=10m -header="sleep:60000" -rate=2000 -workers=200 -timeout=1800s | tee reports.bin | vegeta report

-duration 10m爲持續時間

-rate 2000爲每秒2000個請求

-workers 200個線程　　

sleep爲設置後端隨機響應時間，以保持鏈接。

4、測試對比結果

調整sleep時間和增長壓測客戶端數量，查看estab數量

查看命令： ss -s&&free -g