1、logstash結合redis收集系統日誌
架構圖：
環境準備：
A主機：elasticsearch主機 IP地址：192.168.7.100java
B主機：logstash主機 IP地址：192.168.7.102linux
C主機：redis主機 IP地址：192.168.7.103nginx
D主機：logstash主機/nginx主機 IP地址：192.168.7.101redis
一、安裝並配置redis
一、安裝並配置redis服務，並啓動redis服務數據庫
[root@redis ~]# yum install redis -y [root@redis ~]# vim /etc/redis.conf bind 0.0.0.0 # 監聽本地的全部地址 requirepass 123456 #爲了redis安全，設置一個密碼 [root@redis ~]# systemctl restart redis # 啓動redis服務
二、在logstash-D主機安裝logstash服務
一、先安裝JDK、並建立軟連接json
[root@logstash-1 ~]# cd /usr/local/src [root@logstash-1 src]# ls jdk1.8.0_212 jdk-8u212-linux-x64.tar.gz sonarqube-6.7.7 sonarqube-6.7.7.zip [root@logstash-1 src]# tar xvf jdk-8u212-linux-x64.tar.gz [root@logstash-1 src]# ln -s /usr/local/src/jdk-8u212-linux-x64.tar.gz /usr/local/jdk [root@logstash-1 src]# ln -s /usr/local/jdk/bin/java /usr/bin/
二、配置JDK的環境變量，並使其生效vim
[root@logstash-1 src]# vim /etc/profile.d/jdk.sh # 設置JDK環境變量 export HISTTIMEFORMAT="%F %T `whoami`" export export LANG="en_US.utf-8" export JAVA_HOME=/usr/local/jdk export CLASSPATH=.:$JAVA_HOME/jre/lib/rt.jar:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar export PATH=$PATH:$JAVA_HOME/bin [root@logstash-1 src]# . /etc/profile.d/jdk.sh # 使JDK環境變量生效
三、安裝logstash緩存
[root@logstash-1 src]# yum install logstash-6.8.1.rpm -y [root@logstash-1 ~]# vim /etc/profile.d/logstash.sh # 定義logstash環境變量 export PATH=$PATH:/usr/share/logstash/bin/ [root@logstash-1 ~]# . /etc/profile.d/logstash.sh # 使環境變量生效
四、在/etc/logstash/conf.d目錄下建立一個寫入到redis日誌的文件：redis-es.conf安全
input {
file {
path => "/var/log/messages" # 收集檔期那logstash日誌文件
type => "message-101" # 日誌類型
start_position => "beginning"
stat_interval => "2" # 間隔2s
#codec => "json"
}
}
output {
if [type] == "message-101" {
redis {
host => "192.168.7.103" # 將日誌傳到103的redis主機
port => "6379" # redis的監聽端口號
password => "123456" # redisa的登錄密碼
db => "1" # redis的數據庫類型，默認是0
key => "linux-7-101-key" #自定義的key
data_type => "list" # 數據類型改成list
}
}
}
五、若是logstash服務是以logstash用戶啓動，將logstash系統日誌的權限改成644，不然logstash系統日誌沒法訪問。bash
[root@logstash-1 conf.d]# vim /etc/systemd/system/logstash.service [Unit] Description=logstash [Service] Type=simple User=root #以root方式啓動logstash服務，生產中最好以logstash服務啓動 Group=root # Load env vars from /etc/default/ and /etc/sysconfig/ if they exist. # Prefixing the path with '-' makes it try to load, but if the file doesn't # exist, it continues onward. EnvironmentFile=-/etc/default/logstash EnvironmentFile=-/etc/sysconfig/logstash ExecStart=/usr/share/logstash/bin/logstash "--path.settings" "/etc/logstash" Restart=always WorkingDirectory=/ Nice=19 LimitNOFILE=16384 [Install] WantedBy=multi-user.target [root@logstash-1 conf.d]# chmod 644 /var/log/messages # 將系統日誌權限進行修改。
六、啓動logstash服務
# systemctl start logstash
三、開始測試logstash服務
一、在logstash服務上對/var/log/messages系統日誌進行輸入信息
[root@logstash-1 src]# echo 11 >> /var/log/messages
二、在redis服務器上查登錄redis客戶端查看此時顯示的KEYS值
[root@redis ~]# redis-cli -h 192.168.7.103 192.168.7.103:6379> auth 123456 OK 192.168.7.103:6379> SELECT 1 OK 192.168.7.103:6379[1]> KEYS * 1) "linux-7-101-key" # 能夠看到此時的logstash服務將logstash服務器的系統日誌已經傳遞到redis服務器上
此時在第二臺logstash主機上能夠將系統日誌傳到redis主機上。
四、在logstash-B主機上配置
一、在/etc/logstash/conf.d目錄下建立一個提取redis緩存日誌文件
input {
redis {
host => "192.168.7.103" # redis主機IP地址
port => "6379"
password => "123456"
db => "1"
key => "linux-7-101-key" # 取出對應的KEY值
data_type => "list"
}
}
output {
if [type] == "message-101" { # 提取與第二臺logstash主機的log類型一致
elasticsearch {
hosts => ["192.168.7.100:9200"] # elasticsearch主機的IP地址
index => "message-7-101-%{+YYYY.MM.dd}"
}
}
}
二、重啓B主機的logstash服務
# systemctl restart logstash
三、此時在redis服務器上查看數據已經被logstash服務器採集到並傳到了elasticsearch服務器上。
192.168.7.103:6379[1]> KEYS * (empty list or set) # 此時的redis服務器數據爲空
五、在kibana控制檯建立索引
一、建立收集到redis數據的索引
二、在discover選項中查看收集到的信息
2、logstash結合redis收集nginx訪問日誌
一、在D主機安裝nginx服務並將log日誌配置爲json格式
一、安裝nginx服務，最好是源碼編譯，方便後期升級nginx版本
[root@logstash-1 ~]# cd /usr/local/src [root@logstash-1 src]# wget http://nginx.org/download/nginx-1.14.2.tar.gz [root@logstash-1 src]# tar xvf nginx-1.14.2.tar.gz [root@logstash-1 nginx-1.14.2]# ./configure --prefix=/apps/nginx # 安裝nginx，並制定安裝目錄 [root@logstash-1 nginx-1.14.2]# make -j 2 && make install # 編譯安裝nginx
二、修改nginx配置文件，並將log日誌改成json格式/apps/nginx/conf/nginx.conf
log_format access_json '{"@timestamp":"$time_iso8601",'
'"host":"$server_addr",'
'"clientip":"$remote_addr",'
'"size":$body_bytes_sent,'
'"responsetime":$request_time,'
'"upstreamtime":"$upstream_response_time",'
'"upstreamhost":"$upstream_addr",'
'"http_host":"$host",'
'"url":"$uri",'
'"domain":"$host",'
'"http_user_agent":"$http_user_agent",'
'"xff":"$http_x_forwarded_for",'
'"referer":"$http_referer",'
'"status":"$status"}';
access_log /var/log/nginx/access.log access_json; # 定義json格式的日誌，並指定存放在/var/log/nginx目錄下
[root@logstash-1 nginx-1.14.2]# mkdir /var/log/nginx # 建立一個存放log日誌的目錄
三、啓動nginx服務，並查看啓動的80端口
[root@logstash-1 nginx-1.14.2]# /apps/nginx/sbin/nginx [root@logstash-1 nginx-1.14.2]# ss -nlt State Recv-Q Send-Q Local Address:Port Peer Address:Port LISTEN 0 100 127.0.0.1:25 *:* LISTEN 0 511 *:80 *:* LISTEN 0 128 *:22 *:* LISTEN 0 100 [::1]:25 [::]:* LISTEN 0 50 [::ffff:127.0.0.1]:9600 [::]:* LISTEN 0 128 [::]:22 [::]:*
二、修改第logstash-D主機的配置文件
一、在第二臺logstash主機的/etc/logstash/conf.d目錄下建立配置文件
input {
file {
path => "/var/log/messages"
type => "message-7-101"
start_position => "beginning"
stat_interval => "2"
#codec => "json"
}
file {
path => "/var/log/nginx/access.log"
type => "nginx-accesslog-7-101"
start_position => "beginning"
stat_interval => "2"
codec => "json"
}
}
output {
if [type] == "message-7-101" {
redis {
host => "192.168.7.103"
port => "6379"
password => "123456"
db => "1"
key => "linux-7-101-key"
data_type => "list"
}}
if [type] == "nginx-accesslog-7-101" {
redis {
host => "192.168.7.103"
port => "6379"
password => "123456"
db => "1"
key => "linux-nginxlog-7-101-key"
data_type => "list"
}
}
}
二、重啓logstash服務，將第一臺的logstash主機服務停掉。
[root@logstash-1 conf.d]# systemctl restart logstash
三、在redis主機上查看logstash主機是否已經將數據傳到redis上
192.168.7.103:6379[1]> KEYS * 1) "linux-7-101-key" # 系統日誌 2) "linux-nginxlog-7-101-key" # nginx日誌
三、在logstash-A主機上修改配置文件
一、在/etc/logstash/conf.d目錄下建立一個提取redis數據的配置文件
input {
redis {
host => "192.168.7.103"
port => "6379"
password => "123456"
db => "1"
key => "linux-7-101-key" # 與第二臺的logstash服務器key對應
data_type => "list"
}
redis {
host => "192.168.7.103"
port => "6379"
password => "123456"
db => "1"
key => "linux-nginxlog-7-101-key" # 與第二臺logstash服務器對應
data_type => "list"
}
}
output {
if [type] == "message-7-101" { # 與第二臺logstash服務器對應
elasticsearch {
hosts => ["192.168.7.100:9200"]
index => "message-7-101-%{+YYYY.MM.dd}"
}
}
if [type] == "nginx-accesslog-7-101" { # 與第二臺logstash服務器對應
elasticsearch {
hosts => ["192.168.7.100:9200"]
index => "nginx-accesslog-7-101-%{+YYYY.MM.dd}"
}
}
}
二、重啓logstash服務
# systemctl restart logstash
三、查看reids主機的數據，此時數據已經爲空，被此臺logstash服務器已經取走
192.168.7.103:6379[1]> KEYS * (empty list or set) 192.168.7.103:6379[1]> KEYS * (empty list or set)
四、在kibaba網頁上建立索引
一、在kibana網頁上建立索引
二、查看discover選項添加的索引信息