ELK實戰之使用filebeat代替logstash收集日誌

1、Filebeat介紹

Filebeat是輕量級單用途的日誌收集工具，用於在沒有安裝java的服務器上專門收集日誌，能夠將日誌轉發到logstash、elasticsearch或redis等場景中進行下一步處理。
官方文檔：https://www.elastic.co/guide/en/beats/filebeat/6.0/index.html

2、Filebeat安裝和配置

一、filebeat安裝

#RPM安裝
[root@linux-node2 ~]# curl -L -O https://artifacts.elastic.co/downloads/beats/filebeat/filebeat-6.0.1-x86_64.rpm
[root@linux-node2 ~]# rpm -vi filebeat-6.0.1-x86_64.rpm

#docker安裝
[root@linux-node2 ~]# docker pull docker.elastic.co/beats/filebeat:6.0.1

二、filebeat配置輸出到文件測試

[root@linux-node2 ~]# grep -v "#" /etc/filebeat/filebeat.yml |grep -v "^$"
filebeat.prospectors:
- type: log
  enabled: true
  paths:
    - /var/log/*.log
    - /var/log/messages      #配置收集的日誌路徑
  exclude_lines: ['^DBG',"^$"]     #排除以DBG開頭和空行
  document_type: filesystem-log-5612    #設置類型
filebeat.config.modules:
  path: ${path.config}/modules.d/*.yml
  reload.enabled: false
setup.template.settings:
  index.number_of_shards: 3
setup.kibana:
output.file:     #輸出到文件
  path: "/tmp"
  filename: "filebeat.txt" 
[root@linux-node2 ~]# systemctl restart filebeat
[root@linux-node2 ~]# echo "666666666666666" >> /var/log/messages
[root@linux-node2 conf.d]# tailf /tmp/filebeat.txt 
{"@timestamp":"2018-01-02T03:55:54.680Z","@metadata":{"beat":"filebeat","type":"doc","version":"6.0.1"},"message":"666666666666666","source":"/var/log/messages","offset":3164930,"prospector":{"type":"log"},"beat":{"name":"linux-node2","hostname":"linux-node2","version":"6.0.1"}}

3、配置filebeat輸出到redis

[root@linux-node2 ~]# vim /etc/filebeat/filebeat.yml 
#修改output
output.redis:
  hosts: "192.168.56.12"
  db: "2"
  port: "6379"
  password: "123456"
  key: "filesystem-log-5612"
[root@linux-node2 ~]# systemctl restart filebeat
[root@linux-node2 ~]# echo "123456" >> /var/log/messages

#查看redis中是否有數據
[root@linux-node2 ~]# redis-cli -h 192.168.56.12 -a 123456
192.168.56.12:6379> select 2
OK
192.168.56.12:6379[2]> KEYS *
1) "filesystem-log-5612"
192.168.56.12:6379[2]> llen filesystem-log-5612
(integer) 3
192.168.56.12:6379[2]> llen filesystem-log-5612
(integer) 3
192.168.56.12:6379[2]> llen filesystem-log-5612
(integer) 3
192.168.56.12:6379[2]> llen filesystem-log-5612
(integer) 3
192.168.56.12:6379[2]> llen filesystem-log-5612
(integer) 4

4、配置linux-node1節點的logstash取redis中的數據

#配置logstash
[root@linux-node1 conf.d]# vim redis-logstash.conf 
input {
    redis {
        data_type => "list"  
        host => "192.168.56.12"
        db => "2"
        port => "6379"
        password => "123456"
        key => "filesystem-log-5612"

    }
}

output {
    elasticsearch {
        hosts => ["192.168.56.11:9200"]
        index => "filesystem-log-5612-%{+YYYY.MM.dd}"
    }
}
#檢測語法
[root@linux-node1 conf.d]# /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/redis-logstash.conf -t
OpenJDK 64-Bit Server VM warning: If the number of processors is expected to increase from one, then you should configure the number of parallel GC threads appropriately using -XX:ParallelGCThreads=N
WARNING: Could not find logstash.yml which is typically located in $LS_HOME/config or /etc/logstash. You can specify the path using --path.settings. Continuing using the defaults
Could not find log4j2 configuration at path /usr/share/logstash/config/log4j2.properties. Using default config which logs errors to the console
Configuration OK
#重啓logstash
[root@linux-node1 conf.d]# systemctl restart logstash
#寫入messages日誌
[root@linux-node1 conf.d]# echo "helloworld" >> /var/log/messages
[root@linux-node1 conf.d]# echo "helloworld" >> /var/log/messages
[root@linux-node1 conf.d]# echo "helloworld" >> /var/log/messages
#查看redis數據
[root@linux-node2 ~]# redis-cli -h 192.168.56.12 -a 123456
192.168.56.12:6379> select 2
OK
192.168.56.12:6379[2]> KEYS *
1) "filesystem-log-5612"
192.168.56.12:6379[2]> llen filesystem-log-5612
(integer) 3
192.168.56.12:6379[2]> llen filesystem-log-5612
(integer) 3
192.168.56.12:6379[2]> llen filesystem-log-5612
(integer) 3
192.168.56.12:6379[2]> llen filesystem-log-5612
(integer) 3
192.168.56.12:6379[2]> llen filesystem-log-5612
(integer) 4
192.168.56.12:6379[2]> llen filesystem-log-5612
(integer) 44
192.168.56.12:6379[2]> llen filesystem-log-5612
(integer) 0

5、head插件查看以及kibana添加索引

6、監控redis長度腳本

爲了監控Redis的隊列長度，能夠寫一個監控腳本對redis進行監控，並增長zabbix報警

[root@linux-node2 ~]# vim redis-test.py 
#!/usr/bin/env python
import redis
def redis_conn():
        pool=redis.ConnectionPool(host="192.168.56.12",port=6379,db=2,password=123456)
        conn = redis.Redis(connection_pool=pool)
        data = conn.llen('filesystem-log-5612')
        print(data)
redis_conn()
[root@linux-node2 ~]# python redis-test.py     #當前redis隊列長度爲0
0