emqx服務器調優

時間  2019-11-16
標籤 emqx 服務器 简体版
原文   原文鏈接

優化前架構

優化前架構.jpg

主要問題

  • emqtt 2.x版本問題
  • linux 內核參數
  • erl 配置參數
haproxy問題
  • 單點

  • 配置最大鏈接數問題html

    • 配置文件中TCP最大鏈接數被我設置成2049啦，這就致使TCP同時保持的最大鏈接只有2049個，限制了客戶端鏈接成功率

  • 配置tcp保活時長問題前端

    • TCP心跳最大時長我設置啦30秒
  • 其餘配置參數不合理（分發，重試策略）
  • 服務器端口重用沒有開啓

優化後架構

emqx高可用.jpg

優化功能點

emq版本升級
  • emqx3.1版本
  • 安裝/部署
rpm -ivh emqx-centos7-v3.1.0.x86\_64.rpm
  • 啓動/中止
service emqx start/stop/restart
  • 集羣加入/離開
emqx_ctl cluster join emqx@ip
linux系統調優
haproxy調優
  • 安裝
[root@dz home]# yum install -y pcre-devel  bzip2-devel  gcc gcc-c++ make
[root@dz home]# tar -zxvf haproxy-1.8.13.tar.gz 
[root@dz home]# cd haproxy-1.8.13
[root@dz haproxy-1.8.13]# make TARGET=linux2628 PREFIX=/usr/local/haproxy
[root@dz haproxy-1.8.13]# make install PREFIX=/usr/local/haproxy
install -d "/usr/local/haproxy/sbin"
install haproxy  "/usr/local/haproxy/sbin"
install -d "/usr/local/haproxy/share/man"/man1
install -m 644 doc/haproxy.1 "/usr/local/haproxy/share/man"/man1
install -d "/usr/local/haproxy/doc/haproxy"
for x in configuration management architecture peers-v2.0 cookie-options lua WURFL-device-detection proxy-protocol linux-syn-cookies network-namespaces DeviceAtlas-device-detection 51Degrees-device-detection netscaler-client-ip-insertion-protocol peers close-options SPOE intro; do \
    install -m 644 doc/$x.txt "/usr/local/haproxy/doc/haproxy" ; \
done
[root@dz haproxy-1.8.13]# 
[root@dz haproxy-1.8.13]# /usr/local/haproxy/sbin/haproxy -v
HA-Proxy version 1.8.13 2018/07/30
Copyright 2000-2018 Willy Tarreau <willy@haproxy.org>

[root@dz haproxy-1.8.13]# 
[root@dz haproxy-1.8.13]# mkdir /etc/haproxy
[root@dz haproxy-1.8.13]# groupadd haproxy
[root@dz haproxy-1.8.13]# useradd -s /sbin/nologin -M -g haproxy haproxy //添加haproxy運行haproxy帳號並設置及屬主與屬組
[root@dz haproxy-1.8.13]# cp examples/haproxy.init /etc/init.d/haproxy
[root@dz haproxy-1.8.13]# chmod 755 /etc/init.d/haproxy
[root@dz haproxy-1.8.13]# chkconfig --add haproxy
[root@dz haproxy-1.8.13]# cp /usr/local/haproxy/sbin/haproxy /usr/sbin/
  • 配置參數調優
vim /etc/haproxy/haproxy.cfg
#---------------------------------------------------------------------
#默認配置和全局配置
defaults
    log                     global
    option                  dontlognull
    option http-server-close
    # option forwardfor
    retries                 3
    timeout http-request    10s
    timeout queue           1m
    timeout connect         60s
    timeout client          2m
    timeout server          2m
    timeout http-keep-alive 10s
    timeout check           10s
#配置前端的監聽端口
frontend emqtt-front
    bind *:1883
    maxconn     1000000
    mode tcp
    default_backend emqtt-backend
#配置後端的轉發端口
backend emqtt-backend
    balance roundrobin
    # balance source
    server emq1 10.199.96.149:9883 check inter 100000 fall 2 rise 5 weight 1
    server emq2 10.199.96.150:9883 check inter 100000 fall 2 rise 5 weight 1
    server emq3 10.199.96.152:9883 check inter 100000 fall 2 rise 5 weight 1
    # source 0.0.0.0 usesrc clientip

frontend emqtt-admin-front
    bind *:18083
    mode http
    default_backend emqtt-admin-backend

backend emqtt-admin-backend
    mode http
    balance roundrobin
    server emq1 10.199.96.149:18083 check
    server emq2 10.199.96.150:18083 check
    server emq3 10.199.96.152:18083 check
#控制檯配置
listen admin_stats
        stats   enable
        bind    *:8081
        mode    http
        option  httplog
        log     global
        maxconn 10
        stats   refresh 30s
        stats   uri /admin
        stats   realm haproxy
        stats   auth admin:admin
        stats   hide-version
        stats   admin if TRUE
  • 啓動
systemctl start haproxy
8月 08 09:14:34 dz haproxy[3223]: /etc/rc.d/init.d/haproxy: 第 26 行:[: =: 期待一元表達式
修改/etc/rc.d/init.d/haproxy文件
[ ${NETWORKING} = "no" ] && exit 0
改爲
[ "${NETWORKING}" = "no" ] && exit 0
systemctl daemon-reload
  • 開機自啓動
chkconfig haproxy on
  • 高可用
cat << EOF >> /etc/sysctl.conf
fs.file-max=2097152 
fs.nr_open=2097152
net.core.somaxconn=32768
net.ipv4.tcp_max_syn_backlog=16384
net.core.netdev_max_backlog=16384
net.ipv4.ip_local_port_range=500 65535
net.core.rmem_default=262144
net.core.wmem_default=262144
net.core.rmem_max=16777216
net.core.wmem_max=16777216
net.core.optmem_max=16777216
net.ipv4.tcp_rmem=1024 4096 16777216
net.ipv4.tcp_wmem=1024 4096 16777216
net.nf_conntrack_max=1000000
net.netfilter.nf_conntrack_max=1000000
net.netfilter.nf_conntrack_tcp_timeout_time_wait=30
net.ipv4.tcp_max_tw_buckets=1048576
net.ipv4.tcp_fin_timeout = 15
EOF


cat << EOF >>/etc/security/limits.conf
*      soft   nofile      1048576
*      hard   nofile      1048576
EOF

echo DefaultLimitNOFILE=1048576 >>/etc/systemd/system.conf 

echo session required /usr/lib64/security/pam_limits.so >>/etc/pam.d/login

cat << EOF >> /etc/sysctl.conf
net.ipv4.tcp_tw_reuse=1
net.ipv4.tcp_tw_recycle=1
net.ipv4.tcp_fin_timeout=30
net.ipv4.tcp_syncookies = 1
EOF
  • keepalived部署
yum install keepalived
  • 增長配置文件
### /etc/keepalived/keepalived.conf
! Configuration File for keepalived

global_defs {
   notification_email {
     huangmeng@dyjs.com
    #  failover@firewall.loc
    #  sysadmin@firewall.loc
   }
   notification_email_from huangmeng4520@163.com
   smtp_server smtp.163.com
   smtp_connect_timeout 30
   router_id mqtt40
   vrrp_skip_check_adv_addr
#    vrrp_strict
   vrrp_garp_interval 0
   vrrp_gna_interval 0
}

vrrp_instance VI_1 {
    state MASTER
    interface eth0
    virtual_router_id 51
    priority 100
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 1111
    }
    mcast_src_ip 172.16.40.22
    # unicast_peer {
    # 172.18.40.41 ##（對端IP地址）此地址必定不能忘記,vrrp need use
    # }
    virtual_ipaddress {
        172.16.40.24/24
        # 192.168.200.16
        # 192.168.200.17
        # 192.168.200.18
    }
}
  • 開機啓動
systemctl enable keepalived
  • 172.16.40.24做爲虛擬IPkeepalive的服務器提供外網訪問

測試工具

依賴安裝
yum -y install ncurses-devel openssl-devel unixODBC-devel gcc-c++  
mkdir -p /app/install $$ cd /app/install/
wget http://erlang.org/download/otp_src_21.3.tar.gz
tar -xvzf otp_src_19.0.tar.gz
cd otp_src_19.0
./configure --prefix=/usr/local/erlang --with-ssl -enable-threads -enable-smmp-support -enable-kernel-poll --enable-hipe --without-javac
make && make install
配置erl環境變量
vim /etc/profile
# erlang
export ERLPATH=/usr/local/erlang
export PATH=$ERLPATH/bin:$PATH
source /etc/profile
erl -v
安裝壓測軟件
yum -y install git
cd /app/install
git clone https://github.com/emqtt/emqtt_benchmark.git
cd emqtt_benchmark
make
## 調整系統參數而且開始壓測
sysctl -w net.ipv4.ip_local_port_range="500 65535"
echo 1000000 > /proc/sys/fs/nr_open
ulimit -n 1000000
測試指令與結果展現
[root@zhanghp2 emqtt_benchmark]# ./emqtt_bench pub  -h 192.168.199.132 -p 1883 -c 500 -I 10 -t bench21/%i -s 256
connected: 1
connected: 2
connected: 3
connected: 4
connected: 5
connected: 6
connected: 7
connected: 8
connected: 9
connected: 10
connected: 11
connected: 12
connected: 13
connected: 14
connected: 15
connected: 16
測試命令參數說明
./emqtt_bench pub --help
./emqtt_bench sub --help
報錯總結
`conneted:``138`
`client``49863``EXIT: {shutdown,eaddrnotavail}`
`#分配不了端口`
`[error] [Client <``0.7267``.``0``>] CONNACK Timeout!`
`client``7590``EXIT: {shutdown,connack_timeout}`
`#連接超時`
conneted:`191`
client``49810``EXIT: {shutdown,econnrefused}`
#連接被拒絕
#查看端口號使用命令 
netstat -npta | grep <端口號> 
#查看端口號使用數量 
netstat -npta |grep <端口號> | wc -l

監控頁面

1572517060128.jpg

附件

相關文章
相關標籤/搜索
每日一句
    每一个你不满意的现在，都有一个你没有努力的曾经。
本站公眾號
   歡迎關注本站公眾號,獲取更多信息
聯繫我們 最近搜索 最新文章 沪ICP备13005482号-10 MyBatis教程 SQL 教程 MySQL教程 Java 教程 Thymeleaf 教程 Hibernate教程 Spring教程 Redis教程