spring boot下使用logback或log4j生成符合Logstash標準的JSON格式 spring boot下使用logback或log4j生成符合Logstash標準的JSON格式

時間 2019-12-04

標籤 spring boot 使用 logback log4j log 生成符合 logstash 標準 json 格式欄目 Spring 简体版

原文原文鏈接

spring boot下使用logback或log4j生成符合Logstash標準的JSON格式

1、依賴

因爲配置中使用了json格式的日誌輸出，因此須要引入以下依賴css

"net.logstash.logback:logstash-logback-encoder:4.11",

<dependency>
<groupId>net.logstash.logback</groupId>
<artifactId>logstash-logback-encoder</artifactId>
<version>4.11</version>
</dependency>html

2、配置說明

1.日誌的輸出路徑nginx

<property name="LOG_PATH" value="phantom-log" />

2.讀取spring容器中的屬性，這裏是獲取項目名稱和運行的服務器IPweb

<springProperty scope="context" name="appName" source="spring.application.name" /> <springProperty scope="context" name="ip" source="spring.cloud.client.ipAddress" />

3.設置日誌的格式spring

<property name="CONSOLE_LOG_PATTERN" value="[%d{yyyy-MM-dd HH:mm:ss.SSS} ${ip} ${appName} %highlight(%-5level) %yellow(%X{X-B3-TraceId}),%green(%X{X-B3-SpanId}),%blue(%X{X-B3-ParentSpanId}) %yellow(%thread) %green(%logger) %msg%n"/>

4.添加一個輸出器，並滾動輸出sql

<appender name="FILEERROR" class="ch.qos.logback.core.rolling.RollingFileAppender">

5.指定輸出的文件位置apache

<file>../${LOG_PATH}/${appName}/${appName}-error.log</file>

6.指定滾動輸出的策略，按天數進行切分，或者文件大小超過2M進行切分json

<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy"> <fileNamePattern>../${LOG_PATH}/${appName}/${appName}-error-%d{yyyy-MM-dd}.%i.log</fileNamePattern> <timeBasedFileNamingAndTriggeringPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP"> <maxFileSize>2MB</maxFileSize> </timeBasedFileNamingAndTriggeringPolicy> </rollingPolicy>

1api

7.下面的文件中一共有四個appender, FILEERROR, FILEEWARN, FILEINFO, logstash。tomcat

其中FILEERROR, FILEEWARN, FILEINFO三個是相相似的，只是打印不一樣級別的日誌信息。
logstash是用來生成json格式的日誌文件，方便與ELK日誌系統進行集成。

3、完整配置

<?xml version="1.0" encoding="UTF-8"?>
<configuration>
  <contextName>${HOSTNAME}</contextName>
  <property name="LOG_PATH" value="phantom-log" />
  <springProperty scope="context" name="appName" source="spring.application.name" />
  <springProperty scope="context" name="ip" source="spring.cloud.client.ipAddress" />
  <property name="CONSOLE_LOG_PATTERN"
            value="[%d{yyyy-MM-dd HH:mm:ss.SSS} ${ip} ${appName} %highlight(%-5level) %yellow(%X{X-B3-TraceId}),%green(%X{X-B3-SpanId}),%blue(%X{X-B3-ParentSpanId}) %yellow(%thread) %green(%logger) %msg%n"/>

  <appender name="FILEERROR" class="ch.qos.logback.core.rolling.RollingFileAppender">
    <file>../${LOG_PATH}/${appName}/${appName}-error.log</file>
    <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
      <fileNamePattern>../${LOG_PATH}/${appName}/${appName}-error-%d{yyyy-MM-dd}.%i.log</fileNamePattern>
      <timeBasedFileNamingAndTriggeringPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP">
        <maxFileSize>2MB</maxFileSize>
      </timeBasedFileNamingAndTriggeringPolicy>
    </rollingPolicy>
    <append>true</append>
    <encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
      <pattern>${CONSOLE_LOG_PATTERN}</pattern>
      <charset>utf-8</charset>
    </encoder>
    <filter class="ch.qos.logback.classic.filter.LevelFilter">
      <level>error</level>
      <onMatch>ACCEPT</onMatch>
      <onMismatch>DENY</onMismatch>
    </filter>
  </appender>

  <appender name="FILEWARN" class="ch.qos.logback.core.rolling.RollingFileAppender">
    <file>../${LOG_PATH}/${appName}/${appName}-warn.log</file>
    <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
      <fileNamePattern>../${LOG_PATH}/${appName}/${appName}-warn-%d{yyyy-MM-dd}.%i.log</fileNamePattern>
      <timeBasedFileNamingAndTriggeringPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP">
        <maxFileSize>2MB</maxFileSize>
      </timeBasedFileNamingAndTriggeringPolicy>
    </rollingPolicy>
    <append>true</append>
    <encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
      <pattern>${CONSOLE_LOG_PATTERN}</pattern>
      <charset>utf-8</charset>
    </encoder>
    <filter class="ch.qos.logback.classic.filter.LevelFilter">
      <level>warn</level>
      <onMatch>ACCEPT</onMatch>
      <onMismatch>DENY</onMismatch>
    </filter>
  </appender>

  <appender name="FILEINFO" class="ch.qos.logback.core.rolling.RollingFileAppender">
    <file>../${LOG_PATH}/${appName}/${appName}-info.log</file>
    <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
      <fileNamePattern>../${LOG_PATH}/${appName}/${appName}-info-%d{yyyy-MM-dd}.%i.log</fileNamePattern>
      <timeBasedFileNamingAndTriggeringPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP">
        <maxFileSize>2MB</maxFileSize>
      </timeBasedFileNamingAndTriggeringPolicy>
    </rollingPolicy>
    <append>true</append>
    <encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
      <pattern>${CONSOLE_LOG_PATTERN}</pattern>
      <charset>utf-8</charset>
    </encoder>

    <filter class="ch.qos.logback.classic.filter.LevelFilter">
      <level>info</level>
      <onMatch>ACCEPT</onMatch>
      <onMismatch>DENY</onMismatch>
    </filter>
  </appender>

  <appender name="logstash" class="ch.qos.logback.core.rolling.RollingFileAppender">
    <file>../${LOG_PATH}/${appName}/${appName}.json</file>
    <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
      <fileNamePattern>../${LOG_PATH}/${appName}/${appName}-%d{yyyy-MM-dd}.json</fileNamePattern>
      <maxHistory>7</maxHistory>
    </rollingPolicy>
    <encoder class="net.logstash.logback.encoder.LoggingEventCompositeJsonEncoder">
      <providers>
        <timestamp>
          <timeZone>UTC</timeZone>
        </timestamp>
        <pattern>
          <pattern>
            {
            "ip": "${ip}",
            "app": "${appName}",
            "level": "%level",
            "trace": "%X{X-B3-TraceId:-}",
            "span": "%X{X-B3-SpanId:-}",
            "parent": "%X{X-B3-ParentSpanId:-}",
            "thread": "%thread",
            "class": "%logger{40}",
            "message": "%message",
            "stack_trace": "%exception{10}"
            }
          </pattern>
        </pattern>
      </providers>
    </encoder>
  </appender>

  <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
    <encoder>
      <pattern>${CONSOLE_LOG_PATTERN}</pattern>
      <charset>utf-8</charset>
    </encoder>
    <filter class="ch.qos.logback.classic.filter.ThresholdFilter">
      <level>debug</level>
    </filter>
  </appender>

  <logger name="org.springframework" level="INFO" />
  <logger name="org.hibernate" level="INFO" />
  <logger name="com.kingboy.repository" level="DEBUG" />

  <root level="INFO">
    <appender-ref ref="FILEERROR" />
    <appender-ref ref="FILEWARN" />
    <appender-ref ref="FILEINFO" />
    <appender-ref ref="logstash" />
    <appender-ref ref="STDOUT" />
  </root>
</configuration>

生成的日誌文件以下：

{"@timestamp":"2019-03-14T07:02:15.318+00:00","ip":"ip_IS_UNDEFINED","app":"appName_IS_UNDEFINED","level":"INFO","trace":"","span":"","parent":"","thread":"main","class":"o.apache.coyote.http11.Http11NioProtocol","message":"Starting ProtocolHandler [\"https-jsse-nio-8443\"]","stack_trace":""}
{"@timestamp":"2019-03-14T07:02:15.621+00:00","ip":"ip_IS_UNDEFINED","app":"appName_IS_UNDEFINED","level":"INFO","trace":"","span":"","parent":"","thread":"main","class":"o.apache.tomcat.util.net.NioSelectorPool","message":"Using a shared selector for servlet write/read","stack_trace":""}
{"@timestamp":"2019-03-14T07:02:15.633+00:00","ip":"ip_IS_UNDEFINED","app":"appName_IS_UNDEFINED","level":"INFO","trace":"","span":"","parent":"","thread":"main","class":"o.apache.coyote.http11.Http11NioProtocol","message":"Starting ProtocolHandler [\"http-nio-80\"]","stack_trace":""}
{"@timestamp":"2019-03-14T07:02:15.642+00:00","ip":"ip_IS_UNDEFINED","app":"appName_IS_UNDEFINED","level":"INFO","trace":"","span":"","parent":"","thread":"main","class":"o.s.b.w.embedded.tomcat.TomcatWebServer","message":"Tomcat started on port(s): 8443 (https) 80 (http) with context path ''","stack_trace":""}
{"@timestamp":"2019-03-14T07:02:15.645+00:00","ip":"ip_IS_UNDEFINED","app":"appName_IS_UNDEFINED","level":"INFO","trace":"","span":"","parent":"","thread":"main","class":"org.data.collector.Application","message":"Started Application in 5.971 seconds (JVM running for 19.807)","stack_trace":""}

filebeat.yml

#=========================== Filebeat prospectors =============================

filebeat.prospectors:

- input_type: log

enabled: true #半天沒有日誌上傳，就是這裏的緣由！！！


 # Paths that should be crawled and fetched. Glob based paths.
 paths:
   - /var/log/nginx/*access*.log
 json.keys_under_root: true #json的配置要放在這裏！！！ input裏面的prospector
 json.overwrite_keys: true

#-------------------------- Elasticsearch output ------------------------------
output.elasticsearch:
 # Array of hosts to connect to.
 hosts: ["ip:port","ip:port"]
 index: "filebeat_server_nginx_%{+YYYY-MM}"

這裏面須要注意的是
json.keys_under_root：默認這個值是FALSE的，也就是咱們的json日誌解析後會被放在json鍵上。設爲TRUE，全部的keys就會被放到根節點
json.overwrite_keys: 是否要覆蓋原有的key，這是關鍵配置，將keys_under_root設爲TRUE後，再將overwrite_keys也設爲TRUE，就能把filebeat默認的key值給覆蓋了

還有其餘的配置
json.add_error_key：添加json_error key鍵記錄json解析失敗錯誤
json.message_key：指定json日誌解析後放到哪一個key上，默認是json，你也能夠指定爲log等。

logback-spring.xml

<?xml version="1.0" encoding="UTF-8"?>

<configuration debug="false" scan="true" scanPeriod="600000">
    <!--定義日誌文件的存儲地址 勿在 LogBack 的配置中使用相對路徑-->  
    <property name="LOG_HOME" value="/var/log" />  
    <contextName>${HOSTNAME}</contextName>
    <springProperty scope="context" name="appName" source="spring.application.name" />
      <springProperty scope="context" name="ip" source="spring.cloud.client.ipAddress" />
      
      <!--格式化輸出：%d表示日期，%thread表示線程名，%-5level：級別從左顯示5個字符寬度%msg：日誌消息，%n是換行符 -->
  <property name="CONSOLE_LOG_PATTERN"
            value="[%d{yyyy-MM-dd HH:mm:ss.SSS} ${ip} ${appName} %highlight(%-5level) %yellow(%X{X-B3-TraceId}),%green(%X{X-B3-SpanId}),%blue(%X{X-B3-ParentSpanId}) %yellow(%thread) %green(%logger) %msg%n"/>
    
    
    <!-- <logger name="org.springframework.web" level="DEBUG" /> -->


    <!-- show parameters for hibernate sql 專爲 Hibernate 定製 -->
    <!--<logger name="org.hibernate.type.descriptor.sql.BasicBinder"  level="TRACE" />-->
    <!--<logger name="org.hibernate.type.descriptor.sql.BasicExtractor"  level="DEBUG" />-->
    <!--<logger name="org.hibernate.engine.QueryParameters" level="DEBUG" />-->
    <!--<logger name="org.hibernate.engine.query.HQLQueryPlan" level="DEBUG" />-->
    
    <!-- <logger name="org.hibernate.SQL" level="DEBUG" />  -->
    <logger name="logging.level.com.italktv.platform" level="info" />

    <!-- 控制檯輸出 -->
    <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
        <encoder>
            <pattern>${CONSOLE_LOG_PATTERN}</pattern>
            <charset>utf-8</charset>
        </encoder>
        <filter class="ch.qos.logback.classic.filter.ThresholdFilter">
            <level>debug</level>
        </filter>
    </appender>
    
    <!-- 按照天天生成日誌文件 -->   
    <appender name="FILE"  class="ch.qos.logback.core.rolling.RollingFileAppender">  
             <!-- 正在記錄的日誌文件的路徑及文件名 -->
        <file>${LOG_HOME}/bigdata/data-api.log</file> 
        <rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
            <!--日誌文件輸出的文件名-->
            <FileNamePattern>${LOG_HOME}/bigdata/data-api.%d{yyyy-MM-dd}.%i.log</FileNamePattern> 
            <!--日誌文件保留天數-->
            <MaxHistory>30</MaxHistory>
            <maxFileSize>1MB</maxFileSize> 
            <totalSizeCap>10MB</totalSizeCap> 
        </rollingPolicy>   
    <encoder class="net.logstash.logback.encoder.LoggingEventCompositeJsonEncoder">
      <providers>
        <timestamp>
          <timeZone>UTC</timeZone>
        </timestamp>
        <pattern>
          <pattern>
            {
            "ip": "${ip}",
            "app": "${appName}",
            "level": "%level",
            "trace": "%X{X-B3-TraceId:-}",
            "span": "%X{X-B3-SpanId:-}",
            "parent": "%X{X-B3-ParentSpanId:-}",
            "thread": "%thread",
            "class": "%logger{40}",
            "message": "%message",
            "stack_trace": "%exception{10}"
            }
          </pattern>
        </pattern>
      </providers>
    </encoder> 
        <!--日誌文件最大的大小
       <triggeringPolicy class="ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy">
         <MaxFileSize>10KB</MaxFileSize>
       </triggeringPolicy>
       -->
    </appender> 
      
    <!-- 日誌輸出級別 -->
    <root level="INFO">
        <!-- 生產上不輸出stdout log -->
        <!--appender-ref ref="STDOUT" /-->
        
        <appender-ref ref="FILE" />
    </root> 

</configuration>