巧用rsyslog收集多套日誌並作單套日誌的過濾分離

  日誌是supervisor打出來的python日誌，且把不一樣格式的日誌打印到了同一批文件裏，需求是把帶post和ERROR關鍵字的日誌分離，並進入兩個不一樣kafka的topic隊列，目前的狀況是rsyslog已經收集了nginx的訪問日誌，不能相互影響，就是說不能直接用if判斷作分離，由於可能會日誌混掉。要收集的日誌格式以下：

ERROR:root:requeue {"withRefresh": false, "localPath": "/data1/ms/cache/file_store_location/n.fdaimg.cn/translate/20170219/oobE-fyarref6029227.jpg?43", "remotePath": "translate/20170219/oobE-fyarref6029227.jpg?43"}
INFO:root:2017-02-22T11:53:11.395165, {"withRefresh": false, "localPath": "/data1/ms/cache/file_store_location/n.adfaimg.cn/w/20170222/aue--fyarref6523250.jpeg", "remotePath": "w/20170222/aue--fyarref6523250.jpeg"}
INFO:root:post /data1/ms/cache/file_store_location/n.fsdaimg.cn/w/20170222/aue--fyarref6523250.jpeg to w/20170222/aue--fyarref6523250.jpeg took 112.954854965 ms...

  操做作以前配置的rsyslog的規則以下：

module(load="imfile")
module(load="omkafka")
$PreserveFQDN on
main_queue(
  queue.workerthreads="10"      # threads to work on the queue
  queue.dequeueBatchSize="1000" # max number of messages to process at once
  queue.size="50000"            # max queue size
)
######################### nginx access #####################
$template nginxlog,"xd172\.16\.11\.44`%msg%"
ruleset(name="nginxlog") {
    action(
        broker=["10.13.88.190:9092","10.13.88.191:9092","10.13.88.192:9092","10.13.88.193:9092"]
        type="omkafka"
        topic="cms-nimg-s3"
        template="nginxlog"
        partitions.auto="on"
     )
  }
input(type="imfile"
      File="/data1/ms/comos/logs/access_s3.log"
      Tag=""
      ruleset="nginxlog"
      freshStartTail="on"
      reopenOnTruncate="on"
     )

   當時想直接用if判斷作分離，後來發現全部的日誌都會進if判斷，徹底可能把日誌混淆，後來測試發現,ruleset里居然能夠嵌套if判斷，神奇的rsyslog，解決了一個大問題，配置以下：

module(load="imfile")
module(load="omkafka")
$PreserveFQDN on
main_queue(
  queue.workerthreads="10"      # threads to work on the queue
  queue.dequeueBatchSize="1000" # max number of messages to process at once
  queue.size="50000"            # max queue size
)
######################### nginx access #####################
$template nginxlog,"xd172\.16\.11\.44`%msg%"
ruleset(name="nginxlog") {
    action(
        broker=["10.13.88.190:9092","10.13.88.191:9092","10.13.88.192:9092","10.13.88.193:9092"]
        type="omkafka"
        topic="cms-nimg-s3"
        template="nginxlog"
        partitions.auto="on"
     )
  }
input(type="imfile"
      File="/data1/ms/comos/logs/access_s3.log"
      Tag=""
      ruleset="nginxlog"
      freshStartTail="on"
      reopenOnTruncate="on"
     )
####################### python s3 post  error################################
$template s3post,"xd172\.16\.11\.43 %msg%"
ruleset(name="s3post") {
    if  ( $msg contains "post" ) then {
        action(
              broker=["10.13.88.190:9092","10.13.88.191:9092","10.13.88.192:9092","10.13.88.193:9092"]
              type="omkafka"
              topic="cms-nimg-s3-post"
              template="s3post"
              partitions.auto="on"
              )
       }
    if  ( $msg contains "ERROR" ) then {
        action(
              broker=["10.13.88.190:9092","10.13.88.191:9092","10.13.88.192:9092","10.13.88.193:9092"]
              type="omkafka"
              topic="cms-nimg-s3-post-error"
              template="s3post"
              partitions.auto="on"
              )
       }
}
input(type="imfile"
      File="/data1/ms/comos/logs/s3q_daemon_0.err"
      Tag=""
      ruleset="s3post"
      freshStartTail="on"
      reopenOnTruncate="on"
     )
input(type="imfile"
      File="/data1/ms/comos/logs/s3q_daemon_1.err"
      Tag=""
      ruleset="s3post"
      freshStartTail="on"
      reopenOnTruncate="on"
     )
input(type="imfile"
      File="/data1/ms/comos/logs/s3q_daemon_2.err"
      Tag=""
      ruleset="s3post"
      freshStartTail="on"
      reopenOnTruncate="on"
     )

自建我的原創站運維網咖社(www.net-add.com)，新的博文會在網咖社更新，歡迎瀏覽。