filebeat_elk多機環境入門探測（二）

時間 2019-11-17

標籤 filebeat elk 環境入門探測欄目系統安全简体版

原文原文鏈接

接着上篇http://www.javashuo.com/article/p-eoudmvck-ec.html（filebeat_elk多機環境入門探測（一））繼續：java

4.使用logstash和filebeat 【主機有test2和iptables2】nginx

filebeat用來收集日誌；bootstrap

logstash中的input段經過filebeat發送過來的數據；vim

logstash中的filter段使用grok插件和date插件及if條件語句；ruby

logstash中的output段使用stdout插件；session

test2中filebeat經過rpm包方式安裝app

[root@iptables2 ~]# cat ver4.conf
input {
    beats {
        port => 5044
        type => "syslog"
    }
}less

filter {
    if [type] == "syslog" {
        grok {
            match => [ "message", "%{SYSLOGLINE}" ]
            overwrite => [ "message" ]
        }
    }
    date {
        match => [ "timestamp", "MMM dd HH:mm:ss", "MMM d HH:mm:ss" ]
    }
}ssh

output {
    stdout {
        codec => rubydebug
    }
}
[root@test2 ~]# cat /etc/filebeat/filebeat.yml
##################################################### filebeat #######################################################
filebeat:
prospectors:
    -
      paths:
        - /var/log/messages
      input_type: log
      document_type: messagesjvm

    -
      paths:
        - /var/log/secure
      input_type: syslog
      document_type: loginmsg

    -
      paths:
        - /var/log/nginx_access.log
      input_type: log
      document_type: nginxacclog

      multiline:    # 注意：這一段存在文件，後面會有演示
          pattern: '^[[:space:]]'
          negate: true
          match: after

registry_file: /var/lib/filebeat/registry

##################################################### output #######################################################
output:
logstash:
hosts: ["192.168.40.83:5044"]

##################################################### Logging #######################################################
logging:
files:
rotateeverybytes: 10485760 # = 10MB
[root@test2 ~]# service filebeat start
Starting filebeat: [ OK ]

[root@iptables2 ~]# ./logstash-2.3.2/bin/logstash -f ver4.conf    #比filebeat先啓動，filebeat啓動後就能夠看到輸出了
Settings: Default pipeline workers: 2
Pipeline main started
{
       "message" => "Apr 10 14:59:08 localhost sshd[1127]: Received disconnect from 192.168.40.26: 0: ",
      "@version" => "1",
    "@timestamp" => "2017-04-11T10:15:35.612Z",
    "input_type" => "log",
         "count" => 1,
          "beat" => {
        "hostname" => "test2",
            "name" => "test2"
    },
        "source" => "/var/log/secure",
        "offset" => 0,
          "type" => "loginmsg",
        "fields" => nil,
          "host" => "test2",
          "tags" => [
        [0] "beats_input_codec_plain_applied"
    ]
}
{
       "message" => "Apr 10 14:59:08 localhost sshd[1127]: pam_unix(sshd:session): session closed for user root",
      "@version" => "1",
    "@timestamp" => "2017-04-11T10:15:35.612Z",
          "type" => "loginmsg",
    "input_type" => "log",
          "beat" => {
        "hostname" => "test2",
            "name" => "test2"
    },
        "source" => "/var/log/secure",
         "count" => 1,
        "fields" => nil,
        "offset" => 82,
          "host" => "test2",
          "tags" => [
        [0] "beats_input_codec_plain_applied"
    ]
}
{
       "message" => "Apr 10 15:02:40 localhost sshd[3425]: Accepted password for root from 192.168.40.102 port 49597 ssh2",
      "@version" => "1",
    "@timestamp" => "2017-04-11T10:15:35.612Z",
        "source" => "/var/log/secure",
        "offset" => 173,
    "input_type" => "log",
        "fields" => nil,
          "type" => "loginmsg",
         "count" => 1,
          "beat" => {
        "hostname" => "test2",
            "name" => "test2"
    },
          "host" => "test2",
          "tags" => [
        [0] "beats_input_codec_plain_applied"
    ]
}

5.使用Elasticsearch和logstash和filebeat 【使用的主機test5（安裝Elasticsearch）、iptables二、test2】

filebeat用來收集日誌

logstash中input來自filebeat，filter段處理後，output到Elasticsearch中；

test5上Elasticsearch經過rpm方式安裝

Elasticsearch上安裝插件
使用ES自帶的命令plugin
# head
/usr/share/elasticsearch/bin/plugin install mobz/elasticsearch-head
# kopf
/usr/share/elasticsearch/bin/plugin install lmenezes/elasticsearch-kopf
# bigdesk
/usr/share/elasticsearch/bin/plugin install hlstudio/bigdesk

那如何訪問安裝好的插件呢？
http://ES_server_ip:port/_plugin/plugin_name
Example:
http://127.0.0.1:9200/_plugin/head/
http://127.0.0.1:9200/_plugin/kopf/
編輯配置文件
[root@iptables2 ~]# vim /etc/elasticsearch/elasticsearch.yml
# ---------------------------------- Network -----------------------------------
#
# Set the bind address to a specific IP (IPv4 or IPv6):
#
network.host: 0.0.0.0
#
# Set a custom port for HTTP:
#
http.port: 9200
[root@test5 ~]# service elasticsearch start

[root@iptables2 ~]# cat ver5.conf
input {
    beats {
        port => 5044
        type => "syslog"
    }
}

filter {
    if [type] == "syslog" {
        grok {
            match => [ "message", "%{SYSLOGLINE}" ]
            overwrite => [ "message" ]
        }
    }
    date {
        match => [ "timestamp", "MMM dd HH:mm:ss", "MMM d HH:mm:ss" ]
    }
}

output {
    stdout {
        codec => rubydebug
    }
    elasticsearch {
        hosts => [ "192.168.40.105:9200" ]
    }
}
[root@test2 ~]# cat /etc/filebeat/filebeat.yml
##################################################### filebeat #######################################################
filebeat:
prospectors:
    -
      paths:
        - /var/log/messages
      input_type: log
      document_type: messages

    -
      paths:
        - /var/log/secure
      input_type: syslog
      document_type: loginmsg

    -
      paths:
        - /var/log/nginx_access.log
      input_type: log
      document_type: nginxacclog

      multiline:
          pattern: '^[[:space:]]'
          negate: true
          match: after

registry_file: /var/lib/filebeat/registry

##################################################### output #######################################################
output:
logstash:
hosts: ["192.168.40.83:5044"]

##################################################### Logging #######################################################
logging:
files:
rotateeverybytes: 10485760 # = 10MB

[root@iptables2 ~]# ./logstash-2.3.2/bin/logstash -f ver5.conf    #filebeat啓動後就可看到輸出了
Settings: Default pipeline workers: 2
Pipeline main started
{
       "message" => "Apr 17 13:09:00 localhost kernel: hpet1: lost 1 rtc interrupts",
      "@version" => "1",
    "@timestamp" => "2017-04-17T05:09:06.724Z",
        "offset" => 780,
    "input_type" => "log",
         "count" => 1,
          "type" => "messages",
        "fields" => nil,
          "beat" => {
        "hostname" => "test2",
            "name" => "test2"
    },
        "source" => "/var/log/messages",
          "host" => "test2",
          "tags" => [
        [0] "beats_input_codec_plain_applied"
    ]
}
{
       "message" => "Apr 17 13:09:00 localhost kernel: hpet1: lost 1 rtc interrupts",
      "@version" => "1",
    "@timestamp" => "2017-04-17T05:09:06.724Z",
        "source" => "/var/log/messages",
        "offset" => 843,
          "type" => "messages",
         "count" => 1,
        "fields" => nil,
          "beat" => {
        "hostname" => "test2",
            "name" => "test2"
    },
    "input_type" => "log",
          "host" => "test2",
          "tags" => [
        [0] "beats_input_codec_plain_applied"
    ]
}
{
       "message" => "Apr 17 13:33:06 localhost sshd[5618]: Accepted password for root from 192.168.40.26 port 58504 ssh2",
      "@version" => "1",
    "@timestamp" => "2017-04-17T05:33:07.111Z",
        "offset" => 2475,
    "input_type" => "log",
         "count" => 1,
        "fields" => nil,
        "source" => "/var/log/secure",
          "type" => "loginmsg",
          "beat" => {
        "hostname" => "test2",
            "name" => "test2"
    },
          "host" => "test2",
          "tags" => [
        [0] "beats_input_codec_plain_applied"
    ]
}
{
       "message" => "Apr 17 13:33:06 localhost sshd[5618]: pam_unix(sshd:session): session opened for user root by (uid=0)",
      "@version" => "1",
    "@timestamp" => "2017-04-17T05:33:07.111Z",
        "source" => "/var/log/secure",
    "input_type" => "log",
         "count" => 1,
        "fields" => nil,
        "offset" => 2575,
          "type" => "loginmsg",
          "beat" => {
        "hostname" => "test2",
            "name" => "test2"
    },
          "host" => "test2",
          "tags" => [
        [0] "beats_input_codec_plain_applied"
    ]
}
[root@test2 ~]# service filebeat start
Starting filebeat:

所有啓動後就能夠收集數據了
http://192.168.40.105:9200/_search?pretty { "took" : 2, "timed_out" : false, "_shards" : { "total" : 5, "successful" : 5, "failed" : 0 }, "hits" : { "total" : 4, "max_score" : 1.0, "hits" : [ { "_index" : "logstash-2017.04.17", "_type" : "loginmsg", "_id" : "AVt71af-9wChB39UHyzu", "_score" : 1.0, "_source" : { "message" : "Apr 17 20:13:56 localhost sshd[6912]: Accepted password for root from 192.168.40.26 port 60148 ssh2", "@version" : "1", "@timestamp" : "2017-04-17T12:14:03.336Z", "offset" : 173, "type" : "loginmsg", "input_type" : "log", "count" : 1, "beat" : { "hostname" : "test2", "name" : "test2" }, "source" : "/var/log/secure", "fields" : null, "host" : "test2", "tags" : [ "beats_input_codec_plain_applied" ] } }, { "_index" : "logstash-2017.04.17", "_type" : "loginmsg", "_id" : "AVt71pQW9wChB39UHyzw", "_score" : 1.0, "_source" : { "message" : "Apr 17 20:15:03 localhost sshd[6912]: Received disconnect from 192.168.40.26: 0: ", "@version" : "1", "@timestamp" : "2017-04-17T12:15:08.341Z", "input_type" : "log", "count" : 1, "fields" : null, "beat" : { "hostname" : "test2", "name" : "test2" }, "source" : "/var/log/secure", "offset" : 375, "type" : "loginmsg", "host" : "test2", "tags" : [ "beats_input_codec_plain_applied" ] } },。。。。默認啓動參數爲： /usr/bin/java -Xms256m -Xmx1g -Djava.awt.headless=true -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:CMSInitiatingOccupancyFraction=75 -XX:+UseCMSInitiatingOccupancyOnly -XX:+HeapDumpOnOutOfMemoryError -XX:+DisableExplicitGC -Dfile.encoding=UTF-8 -Djna.nosys=true -Des.path.home=/usr/share/elasticsearch -cp /usr/share/elasticsearch/lib/elasticsearch-2.3.4.jar:/usr/share/elasticsearch/lib/* org.elasticsearch.bootstrap.Elasticsearch start -p /var/run/elasticsearch/elasticsearch.pid -d -Des.default.path.home=/usr/share/elasticsearch -Des.default.path.logs=/var/log/elasticsearch -Des.default.path.data=/var/lib/elasticsearch -Des.default.path.conf=/etc/elasticsearch 修改jvm參數修改配置文件/etc/sysconfig/elasticsearch ES_HEAP_SIZE=2G ES_MAX_MEM=2G 重啓後以下： /usr/bin/java -Xms2g -Xmx2g -Djava.awt.headless=true -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:CMSInitiatingOccupancyFraction=75 -XX:+UseCMSInitiatingOccupancyOnly -XX:+HeapDumpOnOutOfMemoryError -XX:+DisableExplicitGC -Dfile.encoding=UTF-8 -Djna.nosys=true -Des.path.home=/usr/share/elasticsearch -cp /usr/share/elasticsearch/lib/elasticsearch-2.3.4.jar:/usr/share/elasticsearch/lib/* org.elasticsearch.bootstrap.Elasticsearch start -p /var/run/elasticsearch/elasticsearch.pid -d -Des.default.path.home=/usr/share/elasticsearch -Des.default.path.logs=/var/log/elasticsearch -Des.default.path.data=/var/lib/elasticsearch -Des.default.path.conf=/etc/elasticsearch 修改爲功了

相關標籤/搜索

每日一句

每一个你不满意的现在，都有一个你没有努力的曾经。