第九章·Logstash深刻-Logstash配合rsyslog收集haproxy日誌

rsyslog介紹及安裝配置

在centos 6及以前的版本叫作syslog，centos 7開始叫作rsyslog，根據官方的介紹，rsyslog(2013年版本)能夠達到每秒轉發百萬條日誌的級別，官方網址：http://www.rsyslog.com/

---

安裝配置rsyslog

#安裝rsyslog [root@elkstack03 ~]# yum install -y rsyslog #編輯rsyslog配置文件 [root@elkstack03 ~]# vim /etc/rsyslog.conf $ModLoad imudp $UDPServerRun 514 $ModLoad imtcp $InputTCPServerRun 514 #最後面一行添加，local6對應haproxy配置文件定義的local級別,端口爲Logstash的端口 local6.* @@10.0.0.53:2222 

---

安裝配置haproxy

#安裝haproxy [root@elkstack03 ~]# yum install -y haproxy #編輯haproxy配置文件 [root@elkstack03 ~]# vim /etc/haproxy/haproxy.cfg global maxconn 100000 chroot /var/lib/haproxy uid 99 gid 99 daemon nbproc 1 pidfile /var/run/haproxy.pid log 127.0.0.1 local6 info defaults option http-keep-alive option forwardfor maxconn 100000 mode http timeout connect 300000ms timeout client 300000ms timeout server 300000ms listen stats mode http bind 0.0.0.0:9999 stats enable log global stats uri /haproxy-status stats auth haadmin:123456 #frontend web_port frontend web_port bind 0.0.0.0:80 mode http option httplog log global option forwardfor ###################ACL Setting########################## acl pc hdr_dom(host) -i www.elk.com acl mobile hdr_dom(host) -i m.elk.com ###################USE ACL############################## use_backend pc_host if pc use_backend mobile_host if mobile ######################################################## backend pc_host mode http option httplog balance source server web1 10.0.0.53:8081 check inter 2000 rise 3 fall 2 weight 1 backend mobile_host mode http option httplog balance source server web1 10.0.0.53:8080 check inter 2000 rise 3 fall 2 weight 1 #啓動haproxy [root@elkstack03 ~]# /etc/init.d/haproxy start 正在啓動 haproxy： [肯定] #啓動rsyslog [root@elkstack03 ~]# /etc/init.d/rsyslog start 啓動系統日誌記錄器： #驗證端口 [root@elkstack03 ~]# netstat -lntup tcp 0 0 0.0.0.0:9999 0.0.0.0:* LISTEN 9082/haproxy tcp 0 0 0.0.0.0:80 0.0.0.0:* LISTEN 9631/haproxy #驗證進程 [root@elkstack03 ~]# ps -ef|grep haproxy nobody 9082 1 0 14:04 ? 00:00:00 /usr/sbin/haproxy -D -f /etc/haproxy/haproxy.cfg -p /var/run/haproxy.pid #修改nginx配置文件，將端口改成8081 [root@elkstack03 ~]# vim /usr/local/nginx/conf/nginx.conf worker_processes 1; events { worker_connections 1024; } http { include mime.types; default_type application/octet-stream; sendfile on; keepalive_timeout 65; log_format main '$remote_addr - $remote_user [$time_local] "$request" ' '$status $body_bytes_sent "$http_referer" ' '"$http_user_agent" "$http_x_forwarded_for"'; access_log logs/access.log main; log_format access_json '{"@timestamp":"$time_iso8601",' '"host":"$server_addr",' '"clientip":"$remote_addr",' '"size":$body_bytes_sent,' '"responsetime":$request_time,' '"upstreamtime":"$upstream_response_time",' '"upstreamhost":"$upstream_addr",' '"http_host":"$host",' '"url":"$uri",' '"domain":"$host",' '"xff":"$http_x_forwarded_for",' '"referer":"$http_referer",' '"status":"$status"}'; access_log logs/access_json.log access_json; server { listen 8081; server_name 10.0.0.53; location / { root /code/html; index index.html index.htm; } } } #修改tomcat配置文件，將默認站點目錄改爲/webapps/webdir [root@elkstack03 ~]# vim /usr/local/tomcat/conf/server.xml <Host name="localhost" appBase="webapps" unpackWARs="true" autoDeploy="true"> <Context path="" docBase="/usr/local/tomcat/webapps/webdir" debug="0" reloadable="false" crossContext="true"/> #重啓nginx [root@elkstack03 ~]# /usr/local/nginx/sbin/nginx -t nginx: the configuration file /usr/local/nginx-1.10.3/conf/nginx.conf syntax is ok nginx: configuration file /usr/local/nginx-1.10.3/conf/nginx.conf test is successful [root@elkstack03 ~]# /usr/local/nginx/sbin/nginx -s reload #重啓tomcat [root@elkstack03 ~]# cd /usr/local/tomcat/bin/ [root@elkstack03 bin]# ./catalina.sh stop [root@elkstack03 bin]# ./catalina.sh start #修改本地hosts文件 10.0.0.53 www.elk.com 10.0.0.53 m.elk.com 

測試域名訪問


測試haproxy，打開瀏覽器，訪問：http://www.elk.com/


測試haproxy，打開瀏覽器，訪問：http://m.elk.com/


---

配置Logstash

#編輯Logstash配置文件 [root@elkstack03 conf.d]# vim haproxy.cof input{ syslog { type => "rsyslog_haproxy" port => "2222" }} output{ stdout{ codec => rubydebug }} #啓動Logstash [root@elkstack03 conf.d]# /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/haproxy.conf #檢查Logstash端口 [root@elkstack03 ~]# netstat -lntup|grep 2222 tcp 0 0 :::2222 :::* LISTEN 9867/java udp 0 0 :::2222 :::* 9867/java 

訪問haproxy管理頁面測試數據

打開瀏覽器，訪問：http://10.0.0.53:9999/haproxy-status

輸入haproxy配置文件中的用戶名和密碼
用戶名：haadmin
密碼：123456







---

將輸出改爲ES

#進入Logstash配置文件目錄 [root@elkstack03 ~]# cd /etc/logstash/conf.d #編輯配置文件 [root@elkstack03 conf.d]# vim haproxy.conf input{ syslog { type => "rsyslog_haproxy" port => "2222" } } output{ elasticsearch { hosts => ["10.0.0.51:9200"] index => "logstash_rsyslog-%{+YYYY.MM.dd}" } } #啓動Logstash [root@elkstack03 conf.d]# /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/haproxy.conf & 

打開瀏覽器，訪問：http://10.0.0.51:9100/



---

將ES索引添加到Kibana中

打開瀏覽器，訪問：http://10.0.0.54:5601/