filebeat_elk多機環境入門探測（五）

接着上一篇filebeat_elk多機環境入門探測（四）

在test1和test2上，使用filebeat收集java日誌
[root@test2 ~]# cat /etc/filebeat/filebeat.yml
##################################################### filebeat #######################################################
filebeat:
  prospectors:
    -
      paths:
        - /var/log/messages
      input_type: log
      document_type: messages

    -
      paths:
        - /var/log/secure
      input_type: syslog
      document_type: loginmsg

    -
      paths:
        - /var/log/nginx_access.log 
      input_type: log
      document_type: nginxacclog

    -
      paths:
        - /usr/local/tomcat/logs/catalina.out
      input_type: catalina
      document_type: catalinalog
#      multiline:
#          pattern: '^[[:space:]]'
#          negate: true
#          match: after

  registry_file: /var/lib/filebeat/registry

##################################################### output #######################################################
#output:
#  logstash:
#    hosts: ["192.168.40.83:5044"]
#  file:
#    path: "/tmp/access.log"
output:
  redis:
    host: "192.168.40.103"
    port: 6379
    save_topology: true
    index: "filebeat"
    db: 0
    db_topology: 1
    timeout: 5
    reconnect_interval: 1

##################################################### Logging #######################################################
logging:
  files:
    rotateeverybytes: 10485760 # = 10MB

[root@test1 ~]# cat /etc/filebeat/filebeat.yml
##################################################### filebeat #######################################################
filebeat:
  prospectors:
    -
      paths:
        - /var/log/messages
      input_type: log
      document_type: messages

    -
      paths:
        - /var/log/secure
      input_type: syslog
      document_type: loginmsg

    -
      paths:
        - /usr/local/tomcat/logs/catalina.out
      input_type: catalina
      document_type: catalinalog
#      multiline:
#          pattern: '^[[:space:]]'
#          negate: true
#          match: after

  registry_file: /var/lib/filebeat/registry

##################################################### output #######################################################
#output:
#  logstash:
#    hosts: ["192.168.40.83:5044"]
output:
  redis:
    host: "192.168.40.103"
    port: 6379
    save_topology: true
    index: "filebeat"
    db: 0
    db_topology: 1
    timeout: 5
    reconnect_interval: 1

##################################################### Logging #######################################################
logging:
  files:
    rotateeverybytes: 10485760 # = 10MB

logstash配置爲：
[root@iptables2 ~]# cat ver9.conf
input {
#    beats {
#        port => 5044
#        type => "syslog"
#    }
    redis {
        host => "192.168.40.103"
        data_type => "list"
        type => "redis-input"
        key => "filebeat"
    }
}

filter {
    if [type] == "filebeat" {
        grok {
            match => [ "message", "%{SYSLOGLINE}" ]
            overwrite => [ "message" ]
        }
    }
    date {
        match => [ "timestamp", "MMM dd HH:mm:ss", "MMM  d HH:mm:ss" ]
    }
    if [type] == "nginxacclog" {
        grok {
            match => {
                "message" => "%{IP:client} - (?:%{USERNAME:remote_user}|-) \[%{HTTPDATE:timestamp}\] \"%{WORD:method} %{URIPATHPARAM:request} HTTP/%{NUMBER:http_version}\" \"%{NUMBER:request_time:float}\" %{INT:status} %{NUMBER:bytes} \"(?:%{URI:referer}|-)\" \"(?:%{GREEDYDATA:user_agent}|-)\" (?:%{IP:x_forword_for}|-)"
            }
        }
        date {
            match => [ "timestamp","dd/MMM/YYYY:HH:mm:ss Z" ]
        }
        urldecode {
            all_fields => true
        }
    }
}

output {
    stdout {
        codec => rubydebug
    }
    elasticsearch {
        hosts => "192.168.40.105:9200"
    }
}

 

logstash配置：
[root@iptables2 ~]# cat /usr/local/logstash/patterns/nginx
ELKTIMES %{MONTHDAY}/%{MONTH}/%{YEAR}:%{TIME}
[root@iptables2 ~]# cat define_ver3.conf
input {
    stdin {}
}

filter {
    grok {
        patterns_dir => "/usr/local/logstash/patterns/"
        match => {
            "message" => "%{IP:client} - - \[%{ELKTIMES:log_timestamp} \] \"%{WORD:method} %{URIPATHPARAM:request} HTTP/%{NUMBER:http_version}\" %{INT:status} %{NUMBER:bytes} \"(?:%{URI:referer}|-)\" \"(?:%{GREEDYDATA:user_agent}|-)\""
       }
    }
}

output {
    stdout {
        codec => rubydebug
    }
}
[root@iptables2 ~]# ./logstash-2.3.2/bin/logstash -f define_ver3.conf
Settings: Default pipeline workers: 2
Pipeline main started
183.228.18.94 - - [21/Apr/2017:19:13:35 ] "GET /wp-content/plugins/ueditor/ueditor/third-party/SyntaxHighlighter/shCore.js HTTP/1.1" 200 163160 "http://www.178linux.com/5848" "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:47.0) Gecko/20100101 Firefox/47.0"
{
          "message" => "183.228.18.94 - - [21/Apr/2017:19:13:35 ] \"GET /wp-content/plugins/ueditor/ueditor/third-party/SyntaxHighlighter/shCore.js HTTP/1.1\" 200 163160 \"http://www.178linux.com/5848\" \"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:47.0) Gecko/20100101 Firefox/47.0\"",
         "@version" => "1",
       "@timestamp" => "2017-04-24T06:14:32.649Z",
             "host" => "iptables2",
           "client" => "183.228.18.94",
    "log_timestamp" => "21/Apr/2017:19:13:35",
           "method" => "GET",
          "request" => "/wp-content/plugins/ueditor/ueditor/third-party/SyntaxHighlighter/shCore.js",
     "http_version" => "1.1",
           "status" => "200",
            "bytes" => "163160",
          "referer" => "http://www.178linux.com/5848",
       "user_agent" => "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:47.0) Gecko/20100101 Firefox/47.0"
}

[root@iptables2 ~]# cat define_ver3.conf
input {
    stdin {}
}

filter {
    grok {
        patterns_dir => "/usr/local/logstash/patterns/"
        match => {
            "message" => "%{IP:client} - - \[%{ELKTIMES:log_timestamp} \] \"%{WORD:method} %{URIPATHPARAM:request} HTTP/%{NUMBER:http_version}\" %{INT:status} %{NUMBER:bytes} \"(?:%{URI:referer}|-)\" \"(?:%{GREEDYDATA:user_agent}|-)\""
       }
    }
    date {
        match => [ "log_timestamp","dd/MMM/YYYY:HH:mm:ss" ]
    }
    urldecode {
        all_fields => true
    }
}

output {
    stdout {
        codec => rubydebug
    }
}
[root@iptables2 ~]# ./logstash-2.3.2/bin/logstash -f define_ver3.conf
Settings: Default pipeline workers: 2
Pipeline main started
183.228.18.94 - - [21/Apr/2017:19:13:35 ] "GET /wp-content/plugins/ueditor/ueditor/third-party/SyntaxHighlighter/shCore.js HTTP/1.1" 200 163160 "http://www.178linux.com/5848" "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:47.0) Gecko/20100101 Firefox/47.0"
{
          "message" => "183.228.18.94 - - [21/Apr/2017:19:13:35 ] \"GET /wp-content/plugins/ueditor/ueditor/third-party/SyntaxHighlighter/shCore.js HTTP/1.1\" 200 163160 \"http://www.178linux.com/5848\" \"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:47.0) Gecko/20100101 Firefox/47.0\"",
         "@version" => "1",
       "@timestamp" => "2017-04-21T11:13:35.000Z",
             "host" => "iptables2",
           "client" => "183.228.18.94",
    "log_timestamp" => "21/Apr/2017:19:13:35",
           "method" => "GET",
          "request" => "/wp-content/plugins/ueditor/ueditor/third-party/SyntaxHighlighter/shCore.js",
     "http_version" => "1.1",
           "status" => "200",
            "bytes" => "163160",
          "referer" => "http://www.178linux.com/5848",
       "user_agent" => "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:47.0) Gecko/20100101 Firefox/47.0"
}

logstash添加查IP的功能
wget http://geolite.maxmind.com/download/geoip/database/GeoLiteCity.dat.gz
mkdir logstash-2.3.2/conf
cp GeoLiteCity.dat logstash-2.3.2/conf
[root@iptables2 ~]# cat /usr/local/logstash/patterns/nginx
ELKTIMES %{MONTHDAY}/%{MONTH}/%{YEAR}:%{TIME}
[root@iptables2 ~]# cat ver10.conf
input {
#    beats {
#        port => 5044
#        type => "syslog"
#    }
    redis {
        host => "192.168.40.103"
        data_type => "list"
        type => "redis-input"
        key => "filebeat"
    }
}

filter {
    if [type] == "filebeat" {
        grok {
            match => [ "message", "%{SYSLOGLINE}" ]
            overwrite => [ "message" ]
        }
    }
    date {
        match => [ "timestamp", "MMM dd HH:mm:ss", "MMM  d HH:mm:ss" ]
    }
    if [type] == "nginxacclog" {
        grok {
            match => {
                "message" => "%{IP:client} - (?:%{USERNAME:remote_user}|-) \[%{HTTPDATE:timestamp}\] \"%{WORD:method} %{URIPATHPARAM:request} HTTP/%{NUMBER:http_version}\" \"%{NUMBER:request_time:float}\" %{INT:status} %{NUMBER:bytes} \"(?:%{URI:referer}|-)\" \"(?:%{GREEDYDATA:user_agent}|-)\" (?:%{IP:x_forword_for}|-)"
            }
        }
        date {
            match => [ "timestamp","dd/MMM/YYYY:HH:mm:ss Z" ]
        }
        urldecode {
            all_fields => true
        }
    }
    if [type] == "test1log" {
        grok {
            patterns_dir => "/usr/local/logstash/patterns/"
            match => {
                "message" => "%{IP:client} - - \[%{ELKTIMES:log_timestamp} \] \"%{WORD:method} %{URIPATHPARAM:request} HTTP/%{NUMBER:http_version}\" %{INT:status} %{NUMBER:bytes} \"(?:%{URI:referer}|-)\" \"(?:%{GREEDYDATA:user_agent}|-)\""
            }
        }
        date {
            match => [ "log_timestamp","dd/MMM/YYYY:HH:mm:ss" ]
        }
        geoip {
            source => "client"
            target => "geoip"
            database => "/root/logstash-2.3.2/conf/GeoLiteCity.dat"
            add_field => ["[geoip][coordinates]","%{[geoip][longitude]}"]
            add_field => ["[geoip][coordinates]","%{[geoip][latitude]}"]
        }
        mutate {
            convert => ["[geoip][coordinates]","float", "bytes","integer", "bytes.raw","integer"]
        }
        urldecode {
            all_fields => true
        }
    }
}

output {
    stdout {
        codec => rubydebug
    }
    elasticsearch {
        hosts => "192.168.40.105:9200"
    }
}

[root@test1 ~]# cat /etc/filebeat/filebeat.yml
##################################################### filebeat #######################################################
filebeat:
  prospectors:
    -
      paths:
        - /var/log/messages
      input_type: log
      document_type: messages

    -
      paths:
        - /var/log/secure
      input_type: syslog
      document_type: loginmsg

    -
      paths:
        - /usr/local/tomcat/logs/catalina.out
      input_type: catalina
      document_type: catalinalog

    -
      paths:
        - /var/log/genara.log
      input_type: log
      document_type: test1log
#      multiline:
#          pattern: '^[[:space:]]'
#          negate: true
#          match: after

  registry_file: /var/lib/filebeat/registry

##################################################### output #######################################################
#output:
#  logstash:
#    hosts: ["192.168.40.83:5044"]
output:
  redis:
    host: "192.168.40.103"
    port: 6379
    save_topology: true
    index: "filebeat"
    db: 0
    db_topology: 1
    timeout: 5
    reconnect_interval: 1

##################################################### Logging #######################################################
logging:
  files:
    rotateeverybytes: 10485760 # = 10MB

kibana畫圖能夠本身嘗試