elk+redis+filebeat

#安裝elasticsearch-5.3.1.tar.gz(單節點安裝)
tar xf elasticsearch-5.3.1.tar.gz
cd elasticsearch-5.3.1

vim config/elasticsearch.yml #修改配置文件
path.data: /data/elas/data #數據目錄
path.logs: /data/elas/logs
network.host: 0.0.0.0 #容許哪一個IP訪問
http.port: 9200

修改系統參數以確保系統有足夠資源啓動ES
設置內核參數
vim /etc/sysctl.conf
# 增長如下參數
vm.max_map_count=655360
執行如下命令，確保生效配置生效：
sysctl -p

設置資源參數
vim /etc/security/limits.conf
# 修改
* soft nofile 65536
* hard nofile 131072
* soft nproc 65536
* hard nproc 131072

設置用戶資源參數
vim /etc/security/limits.d/20-nproc.conf
elk soft nproc 65536 # 設置elk用戶參數

添加啓動用戶，設置權限
啓動ElasticSearch5版本要非root用戶，須要新建一個用戶來啓動ElasticSearch
useradd elk
mkdir -pv /data/elas/{data,logs} # 建立數據和日誌目錄
chown -R elk:elk /data/elas/ # 修改文件全部者
chown -R elk:elk /usr/local/elasticsearch-5.3.1/
su elk
bin/elasticsearch -d #-d 爲保持後臺運行
curl localhost:9200 來檢查ES是否啓動成功

#安裝kibana-5.3.1-linux-x86_64.tar.gz
tar xf kibana-5.3.1-linux-x86_64.tar.gz

配置kibana
vi config/kibana.yml
server.port: 5601 #開啓默認端口5601
server.host: 「192.168.182.100」 #站點地址,必須是IP
elasticsearch.url: http://192.168.182.100:9200 #指向>elasticsearch服務的ip地址,必須是IP
kibana.index: 「.kibana」
bin/kibana #啓動kibana
curl http://192.168.182.100:5601

#redis5.0.2安裝
下載：wget http://download.redis.io/redis-stable.tar.gz
yum -y install gcc
tar xf redis-5.0.2.tar.gz
mv redis-5.0.2 /usr/local/redis
cd /usr/local/redis
make
make install

vim redis.conf
bind 127.0.0.1 #表示只能夠本機訪問，要是遠程訪問須要註釋掉（前面加#號）
protected-mode yes #改成no 能夠不用輸入密碼登錄
port 6379 #修改端口
daemonize no #改成 yes 後臺運行

#啓動和關閉redis
./src/redis-server ./redis.conf
./src/redis-cli -p 6379 shutdown

#登陸
./src/redis-cli -h 127.0.0.1 -p 6379

#開機啓動
chmod u+x /etc/rc.d/rc.local
echo "/usr/local/redis/src/redis-server /usr/local/redis/redis.conf" >> /etc/rc.d/rc.local

#問題
#會發現warning警告，WARNING: The TCP backlog setting of 511 cannot be enforced because /proc/sys/net/core/somaxconn is set to the lower value of 128.
net.core.somaxconn= 1024 #sysctl.conf
vm.overcommit_memory=1 #sysctl.conf
protected-mode no #關閉保護模式

#安裝filebeat-6.6.1-linux-x86_64.tar.gz
tar xf filebeat-6.6.1-linux-x86_64.tar.gz
vim filebeat.yml
filebeat.inputs:
- type: log
enabled: true
paths:
- /usr/local/nginx/logs/*.log
output.redis:
hosts: ["192.168.182.100:6379"]
key: "nginx-log"
db: 0
timeout: 5
./filebeat -e -c ./filebeat.yml #啓動

#安裝

input {
redis {
data_type => "list"
key => "nginx-log"
host => "192.168.182.100"
port => 6379
}
}
filter {
grok {
match => { "message" => "%{COMBINEDAPACHELOG} %{QS:x_forwarded_for}"}
}
date {
match => [ "timestamp" , "dd/MMM/YYYY:HH:mm:ss Z" ]
}
geoip {
source => "clientip"
}
}
output {
elasticsearch {
hosts => "192.168.182.100:9200"
}
stdout { codec => rubydebug }
}

/usr/local/logstash-5.3.1/bin/logstash -f /usr/local/logstash-5.3.1/config/logstash.conf