Fluentd日誌收集

時間 2019-12-09

標籤 fluentd 日誌收集简体版

原文原文鏈接

高度擴展性的開源日誌收集系統，同時提供td-agent至雲端Treasure Data的大數據服務php

架構

六大插件類型

Input
Parser
Filter
Output
Formatter
Buffer

工做流：Events -> Inputs -> Filters -> Buffer -> Matches and Labelsnginx

Event對象Json結構

一個event對象經過source標籤的input plugin轉換爲如下結構web

tag：事件來源，用於消息路由
time：事發時間(經過source標籤中的time_key、time_format來解析)
record：json格式的日誌內容（每一個format匹配的正則捕獲鍵名做爲一個json字段）

安裝配置

準備工做

網絡時間同步ntp
提升文件句柄上線ulimit -n
高負載環境下優化內核參數

net.core.somaxconn = 1024
	net.core.netdev_max_backlog = 5000
	net.core.rmem_max = 16777216
	net.core.wmem_max = 16777216
	net.ipv4.tcp_wmem = 4096 12582912 16777216
	net.ipv4.tcp_rmem = 4096 12582912 16777216
	net.ipv4.tcp_max_syn_backlog = 8096
	net.ipv4.tcp_slow_start_after_idle = 0
	net.ipv4.tcp_tw_reuse = 1
	net.ipv4.ip_local_port_range = 10240 65535

主配置示例

## 配置路徑`/etc/fluentd/fluentd.conf`
## 新配置須要重啓：fluentd -c alert-email.conf

<source>
  @type tail
  format nginx
  path /var/log/nginx/access.log #須要確認Fluentd daemon有權讀取
  pos_file /var/log/fluentd/tmp/access.log.pos  #上次讀取位置的備忘文件
  tag nginx.access
  @label @mainstream
</source>

<source>
  @type http
  format json
  port 8888
  bind 0.0.0.0
  body_size_limit 32m
  keepalive_timeout 10s
  #http模式下，tag取決於http請求的endpoint路徑
</source>

#nginx單行錯誤日誌的解析
<source>
    @type tail
    format /^(?<time>\d{4}/\d{2}/\d{2} \d{2}:\d{2}:\d{2}) \[(?<log_level>\w+)\] (?<pid>\d+).(?<tid>\d+): (?<message>.*)$/
    tag nginx.error
    path /var/log/nginx/error.log
</source>
#nginx多行錯誤日誌的解析
<source>
    @type tail
    tag nginx.error
    path /var/log/nginx/error.log

    format multiline
    format_firstline /^\d{4}/\d{2}/\d{2} \d{2}:\d{2}:\d{2} \[\w+\] (?<pid>\d+).(?<tid>\d+): /
    format1 /^(?<time>\d{4}/\d{2}/\d{2} \d{2}:\d{2}:\d{2}) \[(?<log_level>\w+)\] (?<pid>\d+).(?<tid>\d+): (?<message>.*)/
    multiline_flush_interval 3s
</source>


<filter info.**>
  @type blackhole_plugin  #簡單丟棄
</filter >

<filter **.**>
  @type grep
  input_key code
  exclude ^200$  #雙行exclude排除語法
  tag filtered.log  #重置tag
</filter>

<filter filtered.log>
    @type record_modifier  #修改器插件(比record_transformer輕量級且高效)
    server_host "#{Socket.gethostname}"  #增長的server_host字段（執行rule指令得到："#{指令}"）
</filter>

<filter filtered.log>
  @type record_transformer
  <record>
    hostname "#{Socket.gethostname}"  #增長的hostname字段（執行rule指令得到："#{指令}"）
  </record>
</filter>

<match filtered.log>
  @type grep
  regexp1 code ^4\d\d$  #單行regex匹配語法
  exclude1 action logout  #單行exclude排除語法
  add_tag_prefix my
</match>

<match my.filtered.log>
  @type stdout
</match>


# label可用於分組不一樣處理流程到指定source
<label @mainstream>
  <match docker.**>
    @type file
    @id   output_docker1
    path         /fluentd/log/docker.*.log
    symlink_path /fluentd/log/docker.log
    append       true
    time_slice_format %Y%m%d
    time_slice_wait   1m
    time_format       %Y%m%dT%H%M%S%z
  </match>
  <match **>
    @type file
    @id   output1
    path         /fluentd/log/data.*.log
    symlink_path /fluentd/log/data.log
    append       true
    time_slice_format %Y%m%d
    time_slice_wait   10m
    time_format       %Y%m%dT%H%M%S%z
  </match>
</label>

PHP平臺應用日誌接入

Fluentd安裝fluent-plugin-elasticsearch插件gem install fluent-plugin-elasticsearch --no-document
Fluentd配置增長Forward輸入（Fluentd最高效的接收方式）

<source>
  @type forward
  port 24224
  bind 0.0.0.0
  #forward模式下，tag取決於http請求的endpoint路徑
</source>

<match **.**>
  @type elasticsearch
  logstash_format true
  host <hostname> #(optional; default="localhost")
  port <port> #(optional; default=9200)
  index_name <index name> #(optional; default=fluentd)
  type_name <type name> #(optional; default=fluentd)
  buffer_type file
  buffer_path /var/log/fluentd/buffer
  flush_interval 10s # for testing
</match>

Laravel應用安裝客戶端Library庫（日誌投遞擴展性最高） composer require fluent/logger=v1.0.0 --profile --prefer-dist --optimize-autoloader
Laravel應用投遞日誌

use Fluent\Logger\FluentLogger;
$logger = new FluentLogger("localhost","24224");
$logger->post("debug.test",array("hello"=>"world"));

PHP平臺cli報錯日誌接入

<source>
      @type tail
      tag php.error
      path /var/log/php-fpm/www-error.log
      pos_file /tmp/fluentd/pos/php-fpm/php_errors.log.pos
      format multiline
      multiline_flush_interval 2s
      format_firstline /^\[[^\]]*?\]/
      format1 /^\[(?<time>[^\]]*?)\] PHP (?<level>.*):  (?<message>.*)$/
      format2 /^Stack trace:$/
      format3 /^(?<trace>.*)/
      time_format %d-%b-%Y %H:%M:%S %Z
    </source>

    <match php.error>
      @type elasticsearch
      host "#{ENV['FLUENT_ELASTICSEARCH_HOST']}"
      port "#{ENV['FLUENT_ELASTICSEARCH_PORT']}"
      scheme "#{ENV['FLUENT_ELASTICSEARCH_SCHEME'] || 'http'}"
      user "elastic"
      password "$ELASTICSEARCH_PASSWORD"
      logstash_format true
      logstash_prefix php.error
      logstash_dateformat %Y.%m.%d
      type_name log
      utc_index false
      reconnect_on_error true
      num_threads 2

      <buffer>
        flush_interval 30s
      </buffer>
    </match>

Nginx訪問日誌接入

<source>
      #use default nginx main log_format
      @type tail
      tag nginx.access
      path /var/log/nginx/access.log
      pos_file /tmp/fluentd/pos/nginx/access.log.pos
      format /^(?<remote_ip>[^ ]*) - (?<remote_user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<uri>[^\"]*) +\S*)?" (?<status_code>[^ ]*) (?<response_size>[^ ]*) "(?<referer>[^\"]*)" "(?<user_agent>[^\"]*)" "(?<forwarded_for>[^\"]*)"$/
      time_format %d/%b/%Y:%H:%M:%S %z
      types status_code:integer,response_size:integer
    </source>

    #### filters ####
    <filter nginx.access>
      @type record_transformer
      enable_ruby
      <record>
        path ${URI(URI.encode(uri.strip)).path}
      </record>
    </filter>

    <match nginx.access>
      @type elasticsearch
      host "#{ENV['FLUENT_ELASTICSEARCH_HOST']}"
      port "#{ENV['FLUENT_ELASTICSEARCH_PORT']}"
      scheme "#{ENV['FLUENT_ELASTICSEARCH_SCHEME'] || 'http'}"
      user "elastic"
      password "$ELASTICSEARCH_PASSWORD"
      logstash_format true
      logstash_prefix nginx.access
      logstash_dateformat %Y.%m.%d
      type_name log
      utc_index false
      reconnect_on_error true
      num_threads 2

      <buffer>
        flush_interval 30s
      </buffer>
    </match>

郵件告警配置

安裝插件

gem install fluent-plugin-grepcounter --no-document
gem install fluent-plugin-mail --no-document

Fluentd配置

#測試日誌中500錯誤並觸發新的count event
<match apache.access>
  @type grepcounter
  count_interval 3  #計算週期
  input_key code    #正則測試的字段
  regexp ^5\d\d$
  threshold 1       #觸發閾值
  add_tag_prefix error_5xx  #新event增長tag前綴
</match>

#處理新發的count event
<match error_5xx.apache.access>
  @type copy
  <store>
    @type stdout  #打印到標準輸出（調試）
  </store>
  <store>
    @type mail
    host smtp.gmail.com
    port 587
    user USERNAME
    password PASSWORD
    enable_starttls_auto true
    from example@gmail.com
    to alert@example.com
    subject 'HTTP SERVER ERROR'
    message Total 5xx error count: %s\n\nPlease check your Nginx webserver
    message_out_keys filed1     #message的參數字段
  </store>
</match>

調試

手動發送日誌logger -t test foobar
增長filter_stdout以記錄日誌流到/etc/td-agent/td-agent.log

<filter **.**>
  @type stdout
</filter>

相關標籤/搜索

每日一句

每一个你不满意的现在，都有一个你没有努力的曾经。