filebeat_elk多機環境入門探測（四）

時間  2019-11-17
標籤 filebeat elk 環境 入門 探測 欄目 系統安全 简体版
原文   原文鏈接

接着上一篇filebeat_elk多機環境入門探測（三） html

在logstash上的filter段處理nginx日誌nginx

ver7.conf
注意：
https://www.elastic.co/guide/en/logstash/current/plugins-filters-grok.html
[root@iptables2 ~]# ./logstash-2.3.2/bin/logstash -e 'input { stdin { } } filter { grok { match => { "message" => "%{IP:client} %{WORD:method} %{URIPATHPARAM:request} %{NUMBER:bytes} %{NUMBER:duration}" } }  } output { stdout { codec => json } }'
Settings: Default pipeline workers: 2
Pipeline main started
55.3.244.1 GET /index.html 15824 0.043
{"message":"55.3.244.1 GET /index.html 15824 0.043","@version":"1","@timestamp":"2017-04-18T11:47:52.053Z","host":"iptables2","client":"55.3.244.1","method":"GET","request":"/index.html","bytes":"15824","duration":"0.043"}
[root@iptables2 ~]# cat define_ver1.conf
input {
    file {
        path => "/var/log/httpd.log"
    }
}正則表達式

filter {
    grok {
        match => { "message" => "%{IP:client} %{WORD:method} %{URIPATHPARAM:request} %{NUMBER:bytes} %{NUMBER:duration}" }
    }
}redis

output {
    stdout {
        codec => rubydebug
    }
}
[root@iptables2 ~]# ./logstash-2.3.2/bin/logstash -f define_ver1.conf   啓動後執行  =================== [root@iptables2 ~]# echo "55.3.244.1 GET /index.html 15824 0.043" >> /var/log/httpd.log
[root@iptables2 ~]# echo "55.3.244.1 GET /index.html 15824 0.043" >> /var/log/httpd.log   =====================
Settings: Default pipeline workers: 2
Pipeline main started
{
       "message" => "55.3.244.1 GET /index.html 15824 0.043",
      "@version" => "1",
    "@timestamp" => "2017-04-18T11:57:02.744Z",
          "path" => "/var/log/httpd.log",
          "host" => "iptables2",
        "client" => "55.3.244.1",
        "method" => "GET",
       "request" => "/index.html",
         "bytes" => "15824",
      "duration" => "0.043"
}
{
       "message" => "55.3.244.1 GET /index.html 15824 0.043",
      "@version" => "1",
    "@timestamp" => "2017-04-18T11:57:07.765Z",
          "path" => "/var/log/httpd.log",
          "host" => "iptables2",
        "client" => "55.3.244.1",
        "method" => "GET",
       "request" => "/index.html",
         "bytes" => "15824",
      "duration" => "0.043"
}json

nginx訪問日誌格式爲：
access '$remote_addr - $remote_user [$time_local] "$request" "$request_time" '
                  '$status $body_bytes_sent "$http_referer" '
                  '"$http_user_agent" $http_x_forwarded_for';
其中一條訪問日誌爲：
10.158.231.117 - - [19/Apr/2017:10:06:57 +0800] "POST /mobile/borrow/investList.html HTTP/1.0" "0.008" 200 74 "-" "YiGangFinance/3.0.3 (iPhone; iOS 10.2.1; Scale/3.00)" 123.158.69.134
[root@iptables2 ~]# cat define_ver2.conf
input {
    file {
        path => "/var/log/httpd.log"
    }
}tomcat

filter {
    grok {
        patterns_dir => "/usr/local/logstash/patterns"
        match => { "message" => "%{IP:client} - (%{USERNAME:remote_user}|-) \[%{HTTPDATE:timestamp}\] \"%{WORD:method} %{URIPATHPARAM:request} HTTP/%{NUMBER:http_version}\" \"%{NUMBER:request_time:float}\" %{INT:status} %{NUMBER:bytes} \"(?:%{GREEDYDATA:http_referrer}|-)\" \"(?:%{GREEDYDATA:user_agent}|-)\" (%{IP:x_forword_for}|-)" }
    }
}ruby

output {
    stdout {
        codec => rubydebug
    }
}
[root@iptables2 ~]# ./logstash-2.3.2/bin/logstash -f define_ver2.conf   啓動時，把# echo '10.158.231.117 - - [19/Apr/2017:10:06:57 +0800] "POST /mobile/borrow/investList.html HTTP/1.0" "0.008" 200 74 "-" "YiGangFinance/3.0.3 (iPhone; iOS 10.2.1; Scale/3.00)" 123.158.69.134' >> /var/log/httpd.log便可
Settings: Default pipeline workers: 2
Pipeline main started
{
          "message" => "10.158.231.117 - - [19/Apr/2017:10:06:57 +0800] \"POST /mobile/borrow/investList.html HTTP/1.0\" \"0.008\" 200 74 \"-\" \"YiGangFinance/3.0.3 (iPhone; iOS 10.2.1; Scale/3.00)\" 123.158.69.134",
         "@version" => "1",
       "@timestamp" => "2017-04-19T03:31:09.996Z",
             "path" => "/var/log/httpd.log",
             "host" => "iptables2",
           "client" => "10.158.231.117",
      "remote_user" => "-",
        "timestamp" => "19/Apr/2017:10:06:57 +0800",
           "method" => "POST",
          "request" => "/mobile/borrow/investList.html",
     "http_version" => "1.0",
     "request_time" => 0.008,
           "status" => "200",
            "bytes" => "74",
    "http_referrer" => "-",
       "user_agent" => "YiGangFinance/3.0.3 (iPhone; iOS 10.2.1; Scale/3.00)",
    "x_forword_for" => "123.158.69.134"
}session

[root@iptables2 ~]# cat ver7.conf
input {
#    beats {
#        port => 5044
#        type => "syslog"
#    }
    redis {
        host => "192.168.40.103"
        data_type => "list"
        type => "redis-input"
        key => "filebeat"
    }
}ssh

filter {
    if [type] == "filebeat" {
        grok {
            match => [ "message", "%{SYSLOGLINE}" ]
            overwrite => [ "message" ]
        }
    }
    date {
        match => [ "timestamp", "MMM dd HH:mm:ss", "MMM  d HH:mm:ss" ]
    }
    if [type] == "nginxacclog" {
        grok {
            match => {
                "message" => "%{IP:client} - (%{USERNAME:remote_user}|-) \[%{HTTPDATE:timestamp}\] \"%{WORD:method} %{URIPATHPARAM:request} HTTP/%{NUMBER:http_version}\" \"%{NUMBER:request_time:float}\" %{INT:status} %{NUMBER:bytes} \"(?:%{GREEDYDATA:http_referrer}|-)\" \"(?:%{GREEDYDATA:user_agent}|-)\" (%{IP:x_forword_for}|-)"
            }
        }
        date {
            match => [ "timestamp","dd/MMM/YYYY:HH:mm:ss Z" ]
        }
        urldecode {
            all_fields => true
        }
    }
}elasticsearch

output {
    stdout {
        codec => rubydebug
    }
    elasticsearch {
        hosts => "192.168.40.105:9200"
    }
}
[root@iptables2 ~]# ./logstash-2.3.2/bin/logstash -f ver7.conf
Settings: Default pipeline workers: 2
Pipeline main started
{
    "@timestamp" => "2017-04-19T05:15:47.522Z",
          "beat" => {
        "hostname" => "test2",
            "name" => "test2"
    },
         "count" => 1,
        "fields" => nil,
    "input_type" => "log",
       "message" => "Apr 19 13:15:41 localhost sshd[13602]: Accepted password for root from 192.168.40.26 port 49216 ssh2",
        "offset" => 7313,
        "source" => "/var/log/secure",
          "type" => "loginmsg",
      "@version" => "1"
}
{
    "@timestamp" => "2017-04-19T05:15:47.522Z",
          "beat" => {
        "hostname" => "test2",
            "name" => "test2"
    },
         "count" => 1,
        "fields" => nil,
    "input_type" => "log",
       "message" => "Apr 19 13:15:41 localhost sshd[13602]: pam_unix(sshd:session): session opened for user root by (uid=0)",
        "offset" => 7414,
        "source" => "/var/log/secure",
          "type" => "loginmsg",
      "@version" => "1"
}
{
       "@timestamp" => "2017-04-19T05:15:53.000Z",
             "beat" => {
        "hostname" => "test2",
            "name" => "test2"
    },
            "count" => 1,
           "fields" => nil,
       "input_type" => "log",
          "message" => "192.168.40.26 - - [19/Apr/2017:13:15:53 +0800] \"GET / HTTP/1.1\" \"0.477\" 200 2340 \"-\" \"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36\" -",
           "offset" => 480648,
           "source" => "/var/log/nginx_access.log",
             "type" => "nginxacclog",
         "@version" => "1",
           "client" => "192.168.40.26",
      "remote_user" => "-",
        "timestamp" => "19/Apr/2017:13:15:53 +0800",
           "method" => "GET",
          "request" => "/",
     "http_version" => "1.1",
     "request_time" => 0.477,
           "status" => "200",
            "bytes" => "2340",
    "http_referrer" => "-",
       "user_agent" => "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36"
}
這裏仍是會存在消息覆蓋的狀況
如：
{
       "@timestamp" => "2017-04-19T05:48:29.000Z",
             "beat" => {
        "hostname" => "test2",
            "name" => "test2"
    },
            "count" => 1,
           "fields" => nil,
       "input_type" => "log",
          "message" => "192.168.50.51 - - [19/Apr/2017:13:48:29 +0800] \"GET /docs/setup.html HTTP/1.1\" \"0.002\" 200 5442 \"http://192.168.40.103/\" \"Mozilla/5.0 (iPhone; CPU iPhone OS 10_3_1 like Mac OS X) AppleWebKit/603.1.30 (KHTML, like Gecko) Version/10.0 Mobile/14E304 Safari/602.1\" -\n192.168.50.51 - - [19/Apr/2017:13:48:29 +0800] \"GET /docs/images/tomcat.gif HTTP/1.1\" \"0.001\" 200 2066 \"http://192.168.40.103/docs/setup.html\" \"Mozilla/5.0 (iPhone; CPU iPhone OS 10_3_1 like Mac OS X) AppleWebKit/603.1.30 (KHTML, like Gecko) Version/10.0 Mobile/14E304 Safari/602.1\" -\n192.168.50.51 - - [19/Apr/2017:13:48:29 +0800] \"GET /docs/images/asf-logo.gif HTTP/1.1\" \"0.001\" 200 7279 \"http://192.168.40.103/docs/setup.html\" \"Mozilla/5.0 (iPhone; CPU iPhone OS 10_3_1 like Mac OS X) AppleWebKit/603.1.30 (KHTML, like Gecko) Version/10.0 Mobile/14E304 Safari/602.1\" -\n192.168.50.51 - - [19/Apr/2017:13:48:29 +0800] \"GET /docs/images/void.gif HTTP/1.1\" \"0.002\" 200 43 \"http://192.168.40.103/docs/setup.html\" \"Mozilla/5.0 (iPhone; CPU iPhone OS 10_3_1 like Mac OS X) AppleWebKit/603.1.30 (KHTML, like Gecko) Version/10.0 Mobile/14E304 Safari/602.1\" -",
           "offset" => 484735,
           "source" => "/var/log/nginx_access.log",
             "type" => "nginxacclog",
         "@version" => "1",
           "client" => "192.168.50.51",
      "remote_user" => "-",
        "timestamp" => "19/Apr/2017:13:48:29 +0800",
           "method" => "GET",
          "request" => "/docs/setup.html",
     "http_version" => "1.1",
     "request_time" => 0.002,
           "status" => "200",
            "bytes" => "5442",
    "http_referrer" => "http://192.168.40.103/\" \"Mozilla/5.0 (iPhone; CPU iPhone OS 10_3_1 like Mac OS X) AppleWebKit/603.1.30 (KHTML, like Gecko) Version/10.0 Mobile/14E304 Safari/602.1\" -\n192.168.50.51 - - [19/Apr/2017:13:48:29 +0800] \"GET /docs/images/tomcat.gif HTTP/1.1\" \"0.001\" 200 2066 \"http://192.168.40.103/docs/setup.html\" \"Mozilla/5.0 (iPhone; CPU iPhone OS 10_3_1 like Mac OS X) AppleWebKit/603.1.30 (KHTML, like Gecko) Version/10.0 Mobile/14E304 Safari/602.1\" -\n192.168.50.51 - - [19/Apr/2017:13:48:29 +0800] \"GET /docs/images/asf-logo.gif HTTP/1.1\" \"0.001\" 200 7279 \"http://192.168.40.103/docs/setup.html\" \"Mozilla/5.0 (iPhone; CPU iPhone OS 10_3_1 like Mac OS X) AppleWebKit/603.1.30 (KHTML, like Gecko) Version/10.0 Mobile/14E304 Safari/602.1\" -\n192.168.50.51 - - [19/Apr/2017:13:48:29 +0800] \"GET /docs/images/void.gif HTTP/1.1\" \"0.002\" 200 43 \"http://192.168.40.103/docs/setup.html",
       "user_agent" => "Mozilla/5.0 (iPhone; CPU iPhone OS 10_3_1 like Mac OS X) AppleWebKit/603.1.30 (KHTML, like Gecko) Version/10.0 Mobile/14E304 Safari/602.1"
}

修改配置以下：
# cat ver7.conf
input {
#    beats {
#        port => 5044
#        type => "syslog"
#    }
    redis {
        host => "192.168.40.103"
        data_type => "list"
        type => "redis-input"
        key => "filebeat"
    }
}

filter {
    if [type] == "filebeat" {
        grok {
            match => [ "message", "%{SYSLOGLINE}" ]
            overwrite => [ "message" ]
        }
    }
    date {
        match => [ "timestamp", "MMM dd HH:mm:ss", "MMM  d HH:mm:ss" ]
    }
    if [type] == "nginxacclog" {
        grok {
            match => {
                "message" => "%{IP:client} - (?:%{USERNAME:remote_user}|-) \[%{HTTPDATE:timestamp}\] \"%{WORD:method} %{URIPATHPARAM:request} HTTP/%{NUMBER:http_version}\" \"%{NUMBER:request_time:float}\" %{INT:status} %{NUMBER:bytes} \"(?:%{URI:referer}|-)\" \"(?:%{GREEDYDATA:user_agent}|-)\" (?:%{IP:x_forword_for}|-)"
            }
        }
        date {
            match => [ "timestamp","dd/MMM/YYYY:HH:mm:ss Z" ]
        }
        urldecode {
            all_fields => true
        }
    }
}

output {
    stdout {
        codec => rubydebug
    }
    elasticsearch {
        hosts => "192.168.40.105:9200"
    }
}


多主機收集日誌配置：
添加一臺test1    192.168.40.101
test1上安裝filebeat
[root@test1 ~]# yum localinstall -y filebeat-1.2.3-x86_64.rpm
[root@test1 ~]# cp /etc/filebeat/filebeat.yml{,.bak_$(date +%F_%H:%M)}
[root@test1 ~]# cat /etc/filebeat/filebeat.yml
##################################################### filebeat #######################################################
filebeat:
  prospectors:
    -
      paths:
        - /var/log/messages
      input_type: log
      document_type: messages

    -
      paths:
        - /var/log/secure
      input_type: syslog
      document_type: loginmsg

      multiline:
          pattern: '^[[:space:]]'
          negate: true
          match: after

  registry_file: /var/lib/filebeat/registry

##################################################### output #######################################################
#output:
#  logstash:
#    hosts: ["192.168.40.83:5044"]
output:
  redis:
    host: "192.168.40.103"
    port: 6379
    save_topology: true
    index: "filebeat"
    db: 0
    db_topology: 1
    timeout: 5
    reconnect_interval: 1

##################################################### Logging #######################################################
logging:
  files:
    rotateeverybytes: 10485760 # = 10MB
操做產生日誌，開關test1回話就能夠了

上面還存在多行日誌都在一個messages中的狀況，尤爲是訪問多的時候，幾乎都在一塊兒好多行都在messages上
修改配置：
定義問題，分析是在filebeat層仍是logstash的問題
[root@test2 ~]# cat /etc/filebeat/filebeat.yml
##################################################### filebeat #######################################################
filebeat:
  prospectors:
    -
      paths:
        - /var/log/messages
      input_type: log
      document_type: messages

    -
      paths:
        - /var/log/secure
      input_type: syslog
      document_type: loginmsg

    -
      paths:
        - /var/log/nginx_access.log 
      input_type: log
      document_type: nginxacclog

#      multiline:
#          pattern: '^[[:space:]]'
#          negate: true
#          match: after

  registry_file: /var/lib/filebeat/registry

##################################################### output #######################################################
#output:
#  logstash:
#    hosts: ["192.168.40.83:5044"]
#  file:
#    path: "/tmp/access.log"
output:
  redis:
    host: "192.168.40.103"
    port: 6379
    save_topology: true
    index: "filebeat"
    db: 0
    db_topology: 1
    timeout: 5
    reconnect_interval: 1

##################################################### Logging #######################################################
logging:
  files:
    rotateeverybytes: 10485760 # = 10MB

http://www.cnblogs.com/toSeek/p/6120778.html   
配置文件位於/etc/filebeat/filebeat.yml，就是filebeat的主配置文件
打開文件，搜索multiline:，默認是註釋的，經常使用的有以下三個配置：
multiline:
    pattern: '^[0-2][0-9]:[0-5][0-9]:[0-5][0-9]'
    negate: true
    match: after
上面配置的意思是：不以時間格式開頭的行都合併到上一行的末尾（正則寫的很差，忽略忽略）
pattern：正則表達式
negate：true 或 false；默認是false，匹配pattern的行合併到上一行；true，不匹配pattern的行合併到上一行
match：after 或 before，合併到上一行的末尾或開頭
還有更多兩個配置，默認也是註釋的，沒特殊要求能夠無論它
max_lines: 500
timeout: 5s
max_lines：合併最大行，默認500
timeout：一次合併事件的超時時間，默認5s，防止合併消耗太多時間甚至卡死

在filebeat上輸出到文件中，logstash能夠直接輸出到標準輸出，輸入時不作任何處理
[root@iptables2 ~]# cat ver8.conf
input {
    beats {
        port => 5044
        type => "syslog"
    }
#    redis {
#        host => "192.168.40.103"
#        data_type => "list"
#        type => "redis-input"
#        key => "filebeat"
#    }
}

#filter {
#    if [type] == "filebeat" {
#        grok {
#            match => [ "message", "%{SYSLOGLINE}" ]
#            overwrite => [ "message" ]
#        }
#    }
#    date {
#        match => [ "timestamp", "MMM dd HH:mm:ss", "MMM  d HH:mm:ss" ]
#    }
#    if [type] == "nginxacclog" {
#        grok {
#            match => {
#                "message" => "%{IP:client} - (?:%{USERNAME:remote_user}|-) \[%{HTTPDATE:timestamp}\] \"%{WORD:method} %{URIPATHPARAM:request} HTTP/%{NUMBER:http_version}\" \"%{NUMBER:request_time:float}\" %{INT:status} %{NUMBER:bytes} \"(?:%{URI:referer}|-)\" \"(?:%{GREEDYDATA:user_agent}|-)\" (?:%{IP:x_forword_for}|-)"
#            }
#        }
#        date {
#            match => [ "timestamp","dd/MMM/YYYY:HH:mm:ss Z" ]
#        }
#        urldecode {
#            all_fields => true
#        }
#    }
#}

output {     stdout {         codec => rubydebug     } #    elasticsearch { #        hosts => "192.168.40.105:9200" #    } } 修改配置後如今正常了，不會出現多行在messages中了

相關文章
相關標籤/搜索
每日一句
    每一个你不满意的现在，都有一个你没有努力的曾经。
本站公眾號
   歡迎關注本站公眾號,獲取更多信息
聯繫我們 最近搜索 最新文章 沪ICP备13005482号-10 MyBatis教程 SQL 教程 MySQL教程 Java 教程 Thymeleaf 教程 Hibernate教程 Spring教程 Redis教程