log.io+inotify+rsync 作日誌監控

1、rsync+inotify部署

#log-prepublish爲slave   elastic爲master  elastic上的日誌經過logstash從服務器抓取
[root@log-prepublish ~]# yum install rsync -y
[root@log-prepublish ~]# useradd rsync -s /sbin/nologin  -M #添加rsync用戶
[root@log-prepublish ~]# grep rsync /etc/passwd
rsync:x:2004:2004::/home/rsync:/sbin/nologin
[root@log-prepublish ~]# mkdir /backup   #建立rsync daemon工做模式的模塊目錄
[root@log-prepublish ~]# chown rsync.rsync /backup/   #更改模塊目錄的用戶組
[root@log-prepublish ~]# cat /etc/rsyncd.conf
uid = rsync
gid = rsync
use chroot = no
max connections = 200
timeout = 300
pid file = /var/run/rsyncd.pid
lock file = /var/run/rsync.lock
log file = /var/log/rsyncd.log
[backup]
path = /backup/
ignore errors
read only = false
list = false
hosts allow = 192.168.1.0/24
hosts deny = 0.0.0.0/32
auth users = rsync_backup
secrets file = /etc/rsync.password

[root@log-prepublish ~]# echo "rsync_backup:123456" >/etc/rsync.password
[root@log-prepublish ~]# chmod 600 /etc/rsync.password 
[root@log-prepublish ~]# rsync --daemon

#master服務器上

[root@elastic ~]# wget http://github.com/downloads/rvoicilas/inotify-tools/inotify-tools-3.14.tar.gz  #下載inotify源碼包
[root@elastic ~]# tar zxf inotify-tools-3.14.tar.gz
[root@elastic ~]# cd inotify-tools-3.14
[root@elastic inotify-tools-3.14]#./configure --prefix=/usr/local/inotify-3.14 
[root@ elastic  inotify-tools-3.14]# make && make install

[root@elastic ~]# cd /usr/local/inotify-3.14/
[root@elastic inotify-3.14]# cat inotify.sh
#!/bin/bash
#para
host01=192.168.1.92  #inotify-slave的ip地址
src=/tzgData/logstash/prepublish/       #本地監控的目錄
dst=backup         #inotify-slave的rsync服務的模塊名
user=rsync_backup      #inotify-slave的rsync服務的虛擬用戶
rsync_passfile=/etc/rsync.password   #本地調用rsync服務的密碼文件
inotify_home=/usr/local/inotify-3.14    #inotify的安裝目錄
#judge
if [ ! -e "$src" ] \
|| [ ! -e "${rsync_passfile}" ] \
|| [ ! -e "${inotify_home}/bin/inotifywait" ] \
|| [ ! -e "/usr/bin/rsync" ];
then
echo "Check File and Folder"
exit 9
fi
${inotify_home}/bin/inotifywait -mrq --timefmt '%d/%m/%y %H:%M' --format '%T %w%f' -e close_write,delete,create,attrib $src \
| while read file
do
#  rsync -avzP --delete --timeout=100 --password-file=${rsync_passfile} $src $user@$host01::$dst >/dev/null 2>&1
cd $src && rsync -aruz -R --delete ./  --timeout=100 $user@$host01::$dst --password-file=${rsync_passfile} >/dev/null 2>&1
done
exit 0

[root@elastic inotify-3.14] sh inotify.sh &  #將腳本加入後臺執行

2、部署log.io

[root@log-prepublish ~]# yum install gcc gcc-c++ openssl-devel  pkgconfig -y #安裝依賴包
[root@log-prepublish ~]# yum install epel-release -y
[root@log-prepublish ~]# yum install nodejs -y
[root@log-prepublish ~]# yum install npm -y
[root@log-prepublish ~]# node -v
v6.9.4
[root@log-prepublish ~]# npm -v
3.10.10
[root@log-prepublish ~]# npm install -g log.io --user "root"  #安裝log.io
[root@log-prepublish ~]# cat /root/.log.io/harvester.conf
exports.config = {
  nodeName: "prepublish",
  logStreams: {
    rest: [
      "/backup/rest.prepublishtomcat-catalina-out",
    ],
    web: [
      "/backup/pay.prepublishtomcat-catalina-out",
    ]
  },
  server: {
    host: '192.168.1.92',
    port: 28777
  }
}

[root@log-prepublish ~]# log.io-server start   #啓動服務端
[root@log-prepublish ~]# log.io-harvester start  #啓動客戶端 （客戶端服務器在同一臺服務器上）


# 測試發現使用 nohup  & 也沒法在後臺運行不中斷服務，因此使用screen開啓服務


[root@log-prepublish prepublish]# screen -ls
There are screens on:
    2200.harvester  (Detached)
    2173.logioserver    (Detached)
2 Sockets in /var/run/screen/S-root.