1、logstash結合redis收集系統日誌
架構圖:
環境準備:
A主機:elasticsearch主機 IP地址:192.168.7.100java
B主機:logstash主機 IP地址:192.168.7.102linux
C主機:redis主機 IP地址:192.168.7.103nginx
D主機:logstash主機/nginx主機 IP地址:192.168.7.101redis
一、安裝並配置redis
一、安裝並配置redis服務,並啓動redis服務數據庫
[root@redis ~]# yum install redis -y [root@redis ~]# vim /etc/redis.conf bind 0.0.0.0 # 監聽本地的全部地址 requirepass 123456 #爲了redis安全,設置一個密碼 [root@redis ~]# systemctl restart redis # 啓動redis服務
二、在logstash-D主機安裝logstash服務
一、先安裝JDK、並建立軟連接json
[root@logstash-1 ~]# cd /usr/local/src [root@logstash-1 src]# ls jdk1.8.0_212 jdk-8u212-linux-x64.tar.gz sonarqube-6.7.7 sonarqube-6.7.7.zip [root@logstash-1 src]# tar xvf jdk-8u212-linux-x64.tar.gz [root@logstash-1 src]# ln -s /usr/local/src/jdk-8u212-linux-x64.tar.gz /usr/local/jdk [root@logstash-1 src]# ln -s /usr/local/jdk/bin/java /usr/bin/
二、配置JDK的環境變量,並使其生效vim
[root@logstash-1 src]# vim /etc/profile.d/jdk.sh # 設置JDK環境變量 export HISTTIMEFORMAT="%F %T `whoami`" export export LANG="en_US.utf-8" export JAVA_HOME=/usr/local/jdk export CLASSPATH=.:$JAVA_HOME/jre/lib/rt.jar:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar export PATH=$PATH:$JAVA_HOME/bin [root@logstash-1 src]# . /etc/profile.d/jdk.sh # 使JDK環境變量生效
三、安裝logstash緩存
[root@logstash-1 src]# yum install logstash-6.8.1.rpm -y [root@logstash-1 ~]# vim /etc/profile.d/logstash.sh # 定義logstash環境變量 export PATH=$PATH:/usr/share/logstash/bin/ [root@logstash-1 ~]# . /etc/profile.d/logstash.sh # 使環境變量生效
四、在/etc/logstash/conf.d目錄下建立一個寫入到redis日誌的文件:redis-es.conf安全
input { file { path => "/var/log/messages" # 收集檔期那logstash日誌文件 type => "message-101" # 日誌類型 start_position => "beginning" stat_interval => "2" # 間隔2s #codec => "json" } } output { if [type] == "message-101" { redis { host => "192.168.7.103" # 將日誌傳到103的redis主機 port => "6379" # redis的監聽端口號 password => "123456" # redisa的登錄密碼 db => "1" # redis的數據庫類型,默認是0 key => "linux-7-101-key" #自定義的key data_type => "list" # 數據類型改成list } } }
五、若是logstash服務是以logstash用戶啓動,將logstash系統日誌的權限改成644,不然logstash系統日誌沒法訪問。bash
[root@logstash-1 conf.d]# vim /etc/systemd/system/logstash.service [Unit] Description=logstash [Service] Type=simple User=root #以root方式啓動logstash服務,生產中最好以logstash服務啓動 Group=root # Load env vars from /etc/default/ and /etc/sysconfig/ if they exist. # Prefixing the path with '-' makes it try to load, but if the file doesn't # exist, it continues onward. EnvironmentFile=-/etc/default/logstash EnvironmentFile=-/etc/sysconfig/logstash ExecStart=/usr/share/logstash/bin/logstash "--path.settings" "/etc/logstash" Restart=always WorkingDirectory=/ Nice=19 LimitNOFILE=16384 [Install] WantedBy=multi-user.target [root@logstash-1 conf.d]# chmod 644 /var/log/messages # 將系統日誌權限進行修改。
六、啓動logstash服務
# systemctl start logstash
三、開始測試logstash服務
一、在logstash服務上對/var/log/messages系統日誌進行輸入信息
[root@logstash-1 src]# echo 11 >> /var/log/messages
二、在redis服務器上查登錄redis客戶端查看此時顯示的KEYS值
[root@redis ~]# redis-cli -h 192.168.7.103 192.168.7.103:6379> auth 123456 OK 192.168.7.103:6379> SELECT 1 OK 192.168.7.103:6379[1]> KEYS * 1) "linux-7-101-key" # 能夠看到此時的logstash服務將logstash服務器的系統日誌已經傳遞到redis服務器上
此時在第二臺logstash主機上能夠將系統日誌傳到redis主機上。
四、在logstash-B主機上配置
一、在/etc/logstash/conf.d目錄下建立一個提取redis緩存日誌文件
input { redis { host => "192.168.7.103" # redis主機IP地址 port => "6379" password => "123456" db => "1" key => "linux-7-101-key" # 取出對應的KEY值 data_type => "list" } } output { if [type] == "message-101" { # 提取與第二臺logstash主機的log類型一致 elasticsearch { hosts => ["192.168.7.100:9200"] # elasticsearch主機的IP地址 index => "message-7-101-%{+YYYY.MM.dd}" } } }
二、重啓B主機的logstash服務
# systemctl restart logstash
三、此時在redis服務器上查看數據已經被logstash服務器採集到並傳到了elasticsearch服務器上。
192.168.7.103:6379[1]> KEYS * (empty list or set) # 此時的redis服務器數據爲空
五、在kibana控制檯建立索引
一、建立收集到redis數據的索引
二、在discover選項中查看收集到的信息
2、logstash結合redis收集nginx訪問日誌
一、在D主機安裝nginx服務並將log日誌配置爲json格式
一、安裝nginx服務,最好是源碼編譯,方便後期升級nginx版本
[root@logstash-1 ~]# cd /usr/local/src [root@logstash-1 src]# wget http://nginx.org/download/nginx-1.14.2.tar.gz [root@logstash-1 src]# tar xvf nginx-1.14.2.tar.gz [root@logstash-1 nginx-1.14.2]# ./configure --prefix=/apps/nginx # 安裝nginx,並制定安裝目錄 [root@logstash-1 nginx-1.14.2]# make -j 2 && make install # 編譯安裝nginx
二、修改nginx配置文件,並將log日誌改成json格式/apps/nginx/conf/nginx.conf
log_format access_json '{"@timestamp":"$time_iso8601",' '"host":"$server_addr",' '"clientip":"$remote_addr",' '"size":$body_bytes_sent,' '"responsetime":$request_time,' '"upstreamtime":"$upstream_response_time",' '"upstreamhost":"$upstream_addr",' '"http_host":"$host",' '"url":"$uri",' '"domain":"$host",' '"http_user_agent":"$http_user_agent",' '"xff":"$http_x_forwarded_for",' '"referer":"$http_referer",' '"status":"$status"}'; access_log /var/log/nginx/access.log access_json; # 定義json格式的日誌,並指定存放在/var/log/nginx目錄下 [root@logstash-1 nginx-1.14.2]# mkdir /var/log/nginx # 建立一個存放log日誌的目錄
三、啓動nginx服務,並查看啓動的80端口
[root@logstash-1 nginx-1.14.2]# /apps/nginx/sbin/nginx [root@logstash-1 nginx-1.14.2]# ss -nlt State Recv-Q Send-Q Local Address:Port Peer Address:Port LISTEN 0 100 127.0.0.1:25 *:* LISTEN 0 511 *:80 *:* LISTEN 0 128 *:22 *:* LISTEN 0 100 [::1]:25 [::]:* LISTEN 0 50 [::ffff:127.0.0.1]:9600 [::]:* LISTEN 0 128 [::]:22 [::]:*
二、修改第logstash-D主機的配置文件
一、在第二臺logstash主機的/etc/logstash/conf.d目錄下建立配置文件
input { file { path => "/var/log/messages" type => "message-7-101" start_position => "beginning" stat_interval => "2" #codec => "json" } file { path => "/var/log/nginx/access.log" type => "nginx-accesslog-7-101" start_position => "beginning" stat_interval => "2" codec => "json" } } output { if [type] == "message-7-101" { redis { host => "192.168.7.103" port => "6379" password => "123456" db => "1" key => "linux-7-101-key" data_type => "list" }} if [type] == "nginx-accesslog-7-101" { redis { host => "192.168.7.103" port => "6379" password => "123456" db => "1" key => "linux-nginxlog-7-101-key" data_type => "list" } } }
二、重啓logstash服務,將第一臺的logstash主機服務停掉。
[root@logstash-1 conf.d]# systemctl restart logstash
三、在redis主機上查看logstash主機是否已經將數據傳到redis上
192.168.7.103:6379[1]> KEYS * 1) "linux-7-101-key" # 系統日誌 2) "linux-nginxlog-7-101-key" # nginx日誌
三、在logstash-A主機上修改配置文件
一、在/etc/logstash/conf.d目錄下建立一個提取redis數據的配置文件
input { redis { host => "192.168.7.103" port => "6379" password => "123456" db => "1" key => "linux-7-101-key" # 與第二臺的logstash服務器key對應 data_type => "list" } redis { host => "192.168.7.103" port => "6379" password => "123456" db => "1" key => "linux-nginxlog-7-101-key" # 與第二臺logstash服務器對應 data_type => "list" } } output { if [type] == "message-7-101" { # 與第二臺logstash服務器對應 elasticsearch { hosts => ["192.168.7.100:9200"] index => "message-7-101-%{+YYYY.MM.dd}" } } if [type] == "nginx-accesslog-7-101" { # 與第二臺logstash服務器對應 elasticsearch { hosts => ["192.168.7.100:9200"] index => "nginx-accesslog-7-101-%{+YYYY.MM.dd}" } } }
二、重啓logstash服務
# systemctl restart logstash
三、查看reids主機的數據,此時數據已經爲空,被此臺logstash服務器已經取走
192.168.7.103:6379[1]> KEYS * (empty list or set) 192.168.7.103:6379[1]> KEYS * (empty list or set)
四、在kibaba網頁上建立索引
一、在kibana網頁上建立索引
二、查看discover選項添加的索引信息