第十一章·Filebeat-使用Filebeat收集日誌

Filebeat介紹及部署

Filebeat介紹

Filebeat附帶預構建的模塊，這些模塊包含收集、解析、充實和可視化各類日誌文件格式數據所需的配置，每一個Filebeat模塊由一個或多個文件集組成，這些文件集包含攝取節點管道、Elasticsearch模板、Filebeat勘探者配置和Kibana儀表盤。

Filebeat模塊很好的入門，它是輕量級單用途的日誌收集工具，用於在沒有安裝java的服務器上專門收集日誌，能夠將日誌轉發到logstash、elasticsearch或redis等場景中進行下一步處理。

---

Filebeat和Logstash使用內存對比

Logstash內存佔用

[root@elkstack03 ~]# ps -ef | grep -v grep | grep logstash | awk '{print $2}' 12628 [root@elkstack03 ~]# cat /proc/12628/status | grep -i vm VmPeak: 6252788 kB VmSize: 6189252 kB VmLck: 0 kB VmHWM: 661168 kB VmRSS: 661168 kB VmData: 6027136 kB VmStk: 88 kB VmExe: 4 kB VmLib: 16648 kB VmPTE: 1888 kB VmSwap: 0 kB 

Filebeat內存佔用

[root@test ~]# cat /proc/12750/status /proc/12751/status | grep -i vm VmPeak: 11388 kB VmSize: 11388 kB VmLck: 0 kB VmHWM: 232 kB VmRSS: 232 kB VmData: 10424 kB VmStk: 88 kB VmExe: 864 kB VmLib: 0 kB VmPTE: 16 kB VmSwap: 0 kB  VmPeak: 25124 kB VmSize: 25124 kB VmLck: 0 kB VmHWM: 15144 kB VmRSS: 15144 kB VmData: 15496 kB VmStk: 88 kB VmExe: 4796 kB VmLib: 0 kB VmPTE: 68 kB VmSwap: 0 kB 

---

Filebeat部署

官方文檔:https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-configuration-details.html

官網下載地址:https://www.elastic.co/downloads/beats/filebeat

#下載Filebeat安裝包 [root@elkstack03 ~]# wget https://artifacts.elastic.co/downloads/beats/filebeat/filebeat-5.3.2-x86_64.rpm #安裝Filebeat [root@elkstack03 ~]# yum localinstall -y filebeat-5.3.2-x86_64.rpm 

Filebeat收集單類型日誌到本地文件

配置Filebeat

#編輯Filebeat配置文件 [root@elkstack03 ~]# vim /etc/filebeat/filebeat.yml filebeat.prospectors: - input_type: log  paths: - /usr/local/nginx/logs/access_json.log #不收集的行  exclude_lines: ["^DBG","^$"] #日誌類型  document_type: ngx_log output.file:  path: "/tmp"  filename: "zls_filebeat.txt" #啓動Filebeat(CentOS6) [root@elkstack03 ~]# /etc/init.d/filebeat start #啓動Filebeat(CentOS7) [root@elkstack03 ~]# systemctl start filebeat #檢測進程 [root@elkstack03 ~]# ps -ef|grep filebeat root 10881 1 0 01:06 pts/1 00:00:00 /usr/share/filebeat/bin/filebeat-god -r / -n -p /var/run/filebeat.pid -- /usr/share/filebeat/bin/filebeat -c /etc/filebeat/filebeat.yml -path.home /usr/share/filebeat -path.config /etc/filebeat -path.data /var/lib/filebeat -path.logs /var/log/filebeat root 10882 10881 0 01:06 pts/1 00:00:00 /usr/share/filebeat/bin/filebeat -c /etc/filebeat/filebeat.yml -path.home /usr/share/filebeat -path.config /etc/filebeat -path.data /var/lib/filebeat -path.logs /var/log/filebeat 



---

檢測本地數據文件

#查看本地/tmp目錄下內容 [root@elkstack03 ~]# ll /tmp/ 總用量 8 -rw------- 1 root root 143953 4月 9 01:06 zls_filebeat.txt #查看日誌內容 [root@elkstack03 ~]# cat /tmp/zls_filebeat.txt {"@timestamp":"2019-04-08T17:06:09.591Z","beat":{"hostname":"elkstack03","name":"elkstack03","version":"5.3.2"},"input_type":"log","message":"{\"@timestamp\":\"2019-04-08T10:47:41+08:00\",\"host\":\"10.0.0.53\",\"clientip\":\"10.0.0.1\",\"size\":0,\"responsetime\":0.000,\"upstreamtime\":\"-\",\"upstreamhost\":\"-\",\"http_host\":\"10.0.0.53\",\"url\":\"/index.html\",\"domain\":\"10.0.0.53\",\"xff\":\"-\",\"referer\":\"-\",\"status\":\"304\"}","offset":256,"source":"/usr/local/nginx/logs/access_json.log","type":"ngx_log"} {"@timestamp":"2019-04-08T17:06:09.591Z","beat":{"hostname":"elkstack03","name":"elkstack03","version":"5.3.2"},"input_type":"log","message":"{\"@timestamp\":\"2019-04-08T10:47:42+08:00\",\"host\":\"10.0.0.53\",\"clientip\":\"10.0.0.1\",\"size\":0,\"responsetime\":0.000,\"upstreamtime\":\"-\",\"upstreamhost\":\"-\",\"http_host\":\"10.0.0.53\",\"url\":\"/index.html\",\"domain\":\"10.0.0.53\",\"xff\":\"-\",\"referer\":\"-\",\"status\":\"304\"}","offset":512,"source":"/usr/local/nginx/logs/access_json.log","type":"ngx_log"} 



Filebeat收集單類型多個日誌到Logstash

配置Filebeat

#編輯Filebeat配置文件 [root@elkstack03 ~]# vim /etc/filebeat/filebeat.yml filebeat.prospectors: - input_type: log  paths: - /usr/local/nginx/logs/access_json.log - /usr/local/nginx/logs/access.log  exclude_lines: ["^DBG","^$"]  document_type: ngx_zls output.logstash: #logstash 服務器地址，能夠是多個  hosts: ["10.0.0.53:6666"] #是否開啓輸出至logstash，默認即爲true  enabled: true #工做線程數  worker: 1 #壓縮級別  compression_level: 3 #多個輸出的時候開啓負載 # loadbalance: true #重啓Filebeat [root@elkstack03 ~]# /etc/init.d/filebeat stop Stopping filebeat: [肯定] [root@elkstack03 ~]# rm -f /var/lib/filebeat/registry [root@elkstack03 ~]# /etc/init.d/filebeat start 

---

配置Logstash輸出到ES

#進入Logstash配置文件目錄 [root@elkstack03 ~]# cd /etc/logstash/conf.d/ #編輯Logstash配置文件 [root@elkstack03 conf.d]# vim beats.conf input { beats { port => 6666 codec => "json" } } output { elasticsearch { hosts => ["10.0.0.51:9200"] index => "%{type}-%{+YYYY.MM.dd}" } } #啓動Logstash [root@elkstack03 conf.d]# /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/beats.conf & 

---

驗證數據

打開瀏覽器，訪問：http://10.0.0.51:9100/



Filebeat收集單類型多個日誌到Redis

配置Filebeat

#編輯Filebeat配置文件 [root@elkstack03 ~]# vim /etc/filebeat/filebeat.yml filebeat.prospectors: - input_type: log  paths: - /usr/local/nginx/logs/access_json.log - /usr/local/nginx/logs/access.log #不收集的行  exclude_lines: ["^DBG","^$"] #日誌類型  document_type: www.driverzeng.com output.redis:  hosts: ["10.0.0.54:6379"] #Redis中的key名稱  key: "nginx" #使用1庫  db: 0 #設置超時時間  timeout: 5 #redis密碼  password: zls #重啓Filebeat [root@elkstack03 ~]# /etc/init.d/filebeat stop Stopping filebeat: [肯定] [root@elkstack03 ~]# rm -f /var/lib/filebeat/registry [root@elkstack03 ~]# /etc/init.d/filebeat start 

---

登陸Redis驗證數據

#登陸Redis [root@elkstack04 ~]# redis-cli -a zls #查看全部key 127.0.0.1:6379> KEYS * 1) "nginx" #查看nginx key長度 127.0.0.1:6379> LLEN nginx (integer) 218 #取出一條日誌 127.0.0.1:6379> LPOP nginx "{\"@timestamp\":\"2019-04-08T17:40:14.675Z\",\"beat\":{\"hostname\":\"elkstack03\",\"name\":\"elkstack03\",\"version\":\"5.3.2\"},\"input_type\":\"log\",\"message\":\"10.0.0.1 - - [08/Apr/2019:10:29:11 +0800] \\\"GET / HTTP/1.1\\\" 404 571 \\\"-\\\" \\\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.86 Safari/537.36\\\"\",\"offset\":194,\"source\":\"/usr/local/nginx/logs/access.log\",\"type\":\"www.driverzeng.com\"}" 



Filebeat收集多類型日誌到Redis

配置Filebeat

#修改Filebeat配置文件 [root@elkstack03 conf.d]# vim /etc/filebeat/filebeat.yml filebeat.prospectors: - input_type: log  paths: - /usr/local/nginx/logs/access_json.log #不收集的行  exclude_lines: ["^DBG","^$"] #日誌類型  document_type: ngx_log - input_type: log  paths: - /usr/local/tomcat/logs/tomcat_access_log.*.log #不收集的行  exclude_lines: ["^DBG","^$"] #日誌類型  document_type: tc_log output.redis:  hosts: ["10.0.0.54:6379"] #Redis中的key名稱  key: "tomcat_nginx" #使用1庫  db: 1 #設置超時時間  timeout: 5 #redis密碼  password: zls #重啓Filebeat [root@elkstack03 ~]# /etc/init.d/filebeat stop Stopping filebeat: [肯定] [root@elkstack03 ~]# rm -f /var/lib/filebeat/registry [root@elkstack03 ~]# /etc/init.d/filebeat start 

---

登陸Redis驗證數據

#登陸Redis [root@elkstack04 ~]# redis-cli -a zls #切換成1庫 127.0.0.1:6379> SELECT 1 OK #查看全部key 127.0.0.1:6379[1]> KEYS * 1) "tomcat_nginx" #查看key長度 127.0.0.1:6379[1]> LLEN tomcat_nginx (integer) 7 #取出日誌 127.0.0.1:6379[1]> LPOP tomcat_nginx "{\"@timestamp\":\"2019-04-08T17:50:48.599Z\",\"beat\":{\"hostname\":\"elkstack03\",\"name\":\"elkstack03\",\"version\":\"5.3.2\"},\"input_type\":\"log\",\"message\":\"{\\\"@timestamp\\\":\\\"2019-04-09T01:50:47+08:00\\\",\\\"host\\\":\\\"10.0.0.53\\\",\\\"clientip\\\":\\\"10.0.0.53\\\",\\\"size\\\":0,\\\"responsetime\\\":0.000,\\\"upstreamtime\\\":\\\"-\\\",\\\"upstreamhost\\\":\\\"-\\\",\\\"http_host\\\":\\\"www.elk.com\\\",\\\"url\\\":\\\"/index.html\\\",\\\"domain\\\":\\\"www.elk.com\\\",\\\"xff\\\":\\\"10.0.0.1\\\",\\\"referer\\\":\\\"-\\\",\\\"status\\\":\\\"304\\\"}\",\"offset\":25894,\"source\":\"/usr/local/nginx/logs/access_json.log\",\"type\":\"ngx_log\"}" 127.0.0.1:6379[1]> LPOP tomcat_nginx "{\"@timestamp\":\"2019-04-08T17:50:48.599Z\",\"beat\":{\"hostname\":\"elkstack03\",\"name\":\"elkstack03\",\"version\":\"5.3.2\"},\"input_type\":\"log\",\"message\":\"{\\\"@timestamp\\\":\\\"2019-04-09T01:50:47+08:00\\\",\\\"host\\\":\\\"10.0.0.53\\\",\\\"clientip\\\":\\\"10.0.0.53\\\",\\\"size\\\":0,\\\"responsetime\\\":0.000,\\\"upstreamtime\\\":\\\"-\\\",\\\"upstreamhost\\\":\\\"-\\\",\\\"http_host\\\":\\\"www.elk.com\\\",\\\"url\\\":\\\"/index.html\\\",\\\"domain\\\":\\\"www.elk.com\\\",\\\"xff\\\":\\\"10.0.0.1\\\",\\\"referer\\\":\\\"-\\\",\\\"status\\\":\\\"304\\\"}\",\"offset\":26162,\"source\":\"/usr/local/nginx/logs/access_json.log\",\"type\":\"ngx_log\"}" 127.0.0.1:6379[1]> LPOP tomcat_nginx "{\"@timestamp\":\"2019-04-08T17:50:48.599Z\",\"beat\":{\"hostname\":\"elkstack03\",\"name\":\"elkstack03\",\"version\":\"5.3.2\"},\"input_type\":\"log\",\"message\":\"{\\\"@timestamp\\\":\\\"2019-04-09T01:50:47+08:00\\\",\\\"host\\\":\\\"10.0.0.53\\\",\\\"clientip\\\":\\\"10.0.0.53\\\",\\\"size\\\":0,\\\"responsetime\\\":0.000,\\\"upstreamtime\\\":\\\"-\\\",\\\"upstreamhost\\\":\\\"-\\\",\\\"http_host\\\":\\\"www.elk.com\\\",\\\"url\\\":\\\"/index.html\\\",\\\"domain\\\":\\\"www.elk.com\\\",\\\"xff\\\":\\\"10.0.0.1\\\",\\\"referer\\\":\\\"-\\\",\\\"status\\\":\\\"304\\\"}\",\"offset\":26430,\"source\":\"/usr/local/nginx/logs/access_json.log\",\"type\":\"ngx_log\"}" 127.0.0.1:6379[1]> LPOP tomcat_nginx "{\"@timestamp\":\"2019-04-08T17:50:58.601Z\",\"beat\":{\"hostname\":\"elkstack03\",\"name\":\"elkstack03\",\"version\":\"5.3.2\"},\"input_type\":\"log\",\"message\":\"{\\\"clientip\\\":\\\"10.0.0.53\\\",\\\"ClientUser\\\":\\\"-\\\",\\\"authenticated\\\":\\\"-\\\",\\\"AccessTime\\\":\\\"[09/Apr/2019:01:50:49 +0800]\\\",\\\"method\\\":\\\"GET / HTTP/1.1\\\",\\\"status\\\":\\\"304\\\",\\\"SendBytes\\\":\\\"-\\\",\\\"Query?string\\\":\\\"\\\",\\\"partner\\\":\\\"-\\\",\\\"AgentVersion\\\":\\\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.86 Safari/537.36\\\"}\",\"offset\":6328,\"source\":\"/usr/local/tomcat/logs/tomcat_access_log.2019-04-09.log\",\"type\":\"tc_log\"}" 127.0.0.1:6379[1]> LPOP tomcat_nginx "{\"@timestamp\":\"2019-04-08T17:50:58.601Z\",\"beat\":{\"hostname\":\"elkstack03\",\"name\":\"elkstack03\",\"version\":\"5.3.2\"},\"input_type\":\"log\",\"message\":\"{\\\"clientip\\\":\\\"10.0.0.53\\\",\\\"ClientUser\\\":\\\"-\\\",\\\"authenticated\\\":\\\"-\\\",\\\"AccessTime\\\":\\\"[09/Apr/2019:01:50:49 +0800]\\\",\\\"method\\\":\\\"GET / HTTP/1.1\\\",\\\"status\\\":\\\"304\\\",\\\"SendBytes\\\":\\\"-\\\",\\\"Query?string\\\":\\\"\\\",\\\"partner\\\":\\\"-\\\",\\\"AgentVersion\\\":\\\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.86 Safari/537.36\\\"}\",\"offset\":6661,\"source\":\"/usr/local/tomcat/logs/tomcat_access_log.2019-04-09.log\",\"type\":\"tc_log\"}" 

看的出來，tomcat日誌和nginx日誌都在一個key中



那麼有人會問了，都在一個key中，日誌不就混亂了麼？咱們該如何查看日誌呢？接下來咱們就來解決這個問題。

---

使用Logstash將beat放入redis的數據輸出到ES

#進入Logstash配置文件目錄 [root@elkstack03 ~]# cd /etc/logstash/conf.d/ #編輯Logstash配置文件 [root@elkstack03 conf.d]# vim beats_redis_es.conf input { redis { host => "10.0.0.54" port => "6379" db => "1" key => "tomcat_nginx" data_type => "list" password => "zls" codec => "json" } } output { elasticsearch { hosts => ["10.0.0.51:9200"] index => "%{type}-%{+YYYY.MM.dd}" } } #啓動Logstash [root@elkstack03 conf.d]# /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/beats_redis_es.conf & 

---

驗證數據

打開瀏覽器，訪問：http://10.0.0.51:9100/



能夠看到，咱們經過Logstash利用type將日誌區分開，分別輸出到ES中，雖然在Redis中沒有區分開，可是最終在ES中區分開了，那麼添加到Kibana中，一樣是兩個日誌。

Filebeat收集多類型日誌輸出到多個目標

配置Filebeat

咱們將nginx日誌 tomcat日誌同時輸出到Redis和本地文件中

[root@elkstack03 conf.d]# vim /etc/filebeat/filebeat.yml filebeat.prospectors: - input_type: log  paths: - /usr/local/nginx/logs/access_json.log #不收集的行  exclude_lines: ["^DBG","^$"] #日誌類型  document_type: ngx_log - input_type: log  paths: - /usr/local/tomcat/logs/tomcat_access_log.*.log #不收集的行  exclude_lines: ["^DBG","^$"] #日誌類型  document_type: tc_log output.redis: #redis 服務器地址，能夠是多個  hosts: ["10.0.0.54:6379"]  key: "tn"  db: 2  timeout: 5  password: zls output.file:  path: "/tmp"  filename: "zls.txt" #工做線程數  worker: 1 #壓縮級別  compression_level: 3 #多個輸出的時候開啓負載  loadbalance: true #重啓Filebeat [root@elkstack03 ~]# /etc/init.d/filebeat stop Stopping filebeat: [肯定] [root@elkstack03 ~]# rm -f /var/lib/filebeat/registry [root@elkstack03 ~]# /etc/init.d/filebeat start 

---

驗證Redis數據和本地文件數據

#登陸redis [root@elkstack04 ~]# redis-cli -a zls #切換2庫 127.0.0.1:6379> SELECT 2 OK #查看全部key 127.0.0.1:6379[2]> KEYS * 1) "tn" #查看key長度 127.0.0.1:6379[2]> LLEN tn (integer) 260 #取出日誌 127.0.0.1:6379[2]> LPOP tn "{\"@timestamp\":\"2019-04-08T18:47:12.133Z\",\"beat\":{\"hostname\":\"elkstack03\",\"name\":\"elkstack03\",\"version\":\"5.3.2\"},\"input_type\":\"log\",\"message\":\"{\\\"@timestamp\\\":\\\"2019-04-08T10:47:41+08:00\\\",\\\"host\\\":\\\"10.0.0.53\\\",\\\"clientip\\\":\\\"10.0.0.1\\\",\\\"size\\\":0,\\\"responsetime\\\":0.000,\\\"upstreamtime\\\":\\\"-\\\",\\\"upstreamhost\\\":\\\"-\\\",\\\"http_host\\\":\\\"10.0.0.53\\\",\\\"url\\\":\\\"/index.html\\\",\\\"domain\\\":\\\"10.0.0.53\\\",\\\"xff\\\":\\\"-\\\",\\\"referer\\\":\\\"-\\\",\\\"status\\\":\\\"304\\\"}\",\"offset\":256,\"source\":\"/usr/local/nginx/logs/access_json.log\",\"type\":\"ngx_log\"}" 



#進入tmp目錄 [root@elkstack03 conf.d]# cd /tmp/ #查看文件是否生成 [root@elkstack03 tmp]# ll 總用量 2284 drwxr-xr-x 2 root root 4096 4月 9 02:20 hsperfdata_root -rw-r--r-- 1 root root 638 4月 9 01:35 keyutil_example.com_5881860801300609526.crt -rw-r--r-- 1 root root 910 4月 9 01:35 keyutil_example.com_672012091508350129.key -rw-r--r-- 1 root root 2010104 4月 9 01:35 libnetty-tcnative-linux-x86_647457201629343237428.so -rw-r--r-- 1 root root 702 3月 30 23:05 message2_2019.03.30.log srwxrwxrwx 1 mysql mysql 0 3月 8 06:28 mysql.sock -rw-r--r-- 1 root root 2523 3月 31 01:39 secure_2019.03.30.log -rw------- 1 root root 148281 4月 9 01:06 zls_filebeat.txt -rw------- 1 root root 154598 4月 9 02:47 zls.txt #查看文件內容 [root@elkstack03 tmp]# cat zls.txt {"@timestamp":"2019-04-08T18:47:12.133Z","beat":{"hostname":"elkstack03","name":"elkstack03","version":"5.3.2"},"input_type":"log","message":"{\"@timestamp\":\"2019-04-08T10:47:41+08:00\",\"host\":\"10.0.0.53\",\"clientip\":\"10.0.0.1\",\"size\":0,\"responsetime\":0.000,\"upstreamtime\":\"-\",\"upstreamhost\":\"-\",\"http_host\":\"10.0.0.53\",\"url\":\"/index.html\",\"domain\":\"10.0.0.53\",\"xff\":\"-\",\"referer\":\"-\",\"status\":\"304\"}","offset":256,"source":"/usr/local/nginx/logs/access_json.log","type":"ngx_log"} {"@timestamp":"2019-04-08T18:47:12.133Z","beat":{"hostname":"elkstack03","name":"elkstack03","version":"5.3.2"},"input_type":"log","message":"{\"@timestamp\":\"2019-04-08T10:47:42+08:00\",\"host\":\"10.0.0.53\",\"clientip\":\"10.0.0.1\",\"size\":0,\"responsetime\":0.000,\"upstreamtime\":\"-\",\"upstreamhost\":\"-\",\"http_host\":\"10.0.0.53\",\"url\":\"/index.html\",\"domain\":\"10.0.0.53\",\"xff\":\"-\",\"referer\":\"-\",\"status\":\"304\"}","offset":512,"source":"/usr/local/nginx/logs/access_json.log","type":"ngx_log"}