Centos7.2 安裝 ELK環境

1，rpm下載：https://www.elastic.co/downloads

    wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-5.3.0.rpm

    wget https://artifacts.elastic.co/downloads/kibana/kibana-5.3.0-x86_64.rpm

    wget https://artifacts.elastic.co/downloads/logstash/logstash-5.3.0.rpm

2，安裝elasticsearch

    rpm -ivh elasticsearch-5.3.0.rpm

    vim /etc/elasticsearch/elasticsearch.yml

    cluster.name: elasticsearch
    node.name: node0
    path.data: /var/lib/elasticsearch
    path.logs: /var/log/elasticsearch
    #path.work: /var/elasticsearch/work
    #transport.tcp.port: 9300
    http.port: 9200
    http.host: 192.168.180.94

    systemctl start elasticsearch.service

    systemctl enable elasticsearch.service

使用：

    /etc/init.d/elasticsearch status 查看狀態

    whereis elasticsearch  查看所在目錄

3，安裝Kibana

    rpm -ivh kibana-5.3.0-x86_64.rpm

    vim /etc/kibana/kibana.yml

    server.port: 5601
    server.host: "192.168.180.94"
    elasticsearch.url: "http://192.168.180.94:9200"

4，安裝logstash

    rpm -ivh logstash-5.3.0.rpm 

    vim /etc/logstash/conf.d/first-pipeline.conf 

input {
    beats {
        #default
        codec => "plain"
        client_inactivity_timeout => 60
        host => "0.0.0.0"

        id => "input_beats"
        port => "5043"
    }
}

filter {
    if "nginx_access_log" in [tags] {
        grok {
            #match => { "message" => "%{IPORHOST:client_ip} - %{NOTSPACE:remote_user} \[%{HTTPDATE:log_date}\] \"(?:%{WORD:method} %{NOTSPACE:request}(?: HTTP/%{NUMBER:version})?|%{DATA:rawrequest})\" %{NUMBER:response} (?:%{NUMBER:sent_bytes}|-) %{QS:referer} %{QS:user_agent} \"%{IPV4:x_forwarded_for}\""}
            match => { "message" => "%{IPORHOST:client_ip} - %{NOTSPACE:remote_user} \[%{HTTPDATE:log_date}\] \"(?:%{WORD:method} %{NOTSPACE:request}(?: HTTP/%{NUMBER:version})?|%{DATA:rawrequest})\" %{NUMBER:response} (?:%{NUMBER:sent_bytes}|-) %{QS:referer} %{QS:user_agent}"}
        }
        geoip {
            source => "client_ip"
        }
    }
 
    if "nginx_error_log" in [tags] {
        grok {
            match => { "message" => "%{DATESTAMP:log_date} \[%{LOGLEVEL:log_level}\] (?:%{GREEDYDATA:err_str})"}
        }
    }

    date {
        match => [ "log_date", "dd/MMM/yyyy:HH:mm:ss Z", "yy/MM/dd HH:mm:ss"]
    }
}
output {
    stdout { codec => rubydebug }

    if "nginx_access_log" in [tags] {
        elasticsearch {
            hosts => [ "192.168.180.94:9200" ]
            index => "logstash-nginx-access-%{+YYYY.MM.dd}"
        }
    }

    if "nginx_error_log" in [tags] {
        elasticsearch {
            hosts => [ "192.168.180.94:9200" ]
            index => "logstash-nginx-error-%{+YYYY.MM.dd}"
        }
    }
}
 

    logstash 用service啓動致使不能自動建立索引，因此用nohup的方式啓動：

    nohup /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/first-pipeline.conf &

    從新加載文件的操做：

    systemctl stop filebeat.service

    mv /var/lib/filebeat/registry /var/lib/filebeat/registry.bak

    kill -9 $logstash_pid

    curl -XDELETE 'http://192.168.180.94:9200/logstash-*'

    systemctl start filebeat.service

    nohup /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/first-pipeline.conf &

5，部署方案：

方案1、（Nginx to syslog + logstash(udp)）-> es集羣+kibana

方案2、(Nginx to file + featbeat(file))-> logstash + es集羣+kibana

方案3、（Nginx to redis）-> logstash + es集羣+kibana

    建議選中方案一，nginx須要配置：

        error_log  syslog:server=192.168.180.94:555;

        access_log syslog:server=192.168.180.94:555;

    logstash須要配置：

        udp {           port => "555"         }