logstash 添加nginx日誌

選擇需求分類廢話少說直接上圖

 

第一張圖：

　　

 

2.此圖搭配的日誌格式是：

　　　

　　

log_format main '$remote_addr - $remote_user [$time_local] $http_host $request_method "$uri" "$query_string" '
'$status $body_bytes_sent "$http_referer" $upstream_status $upstream_addr $request_time $upstream_response_time '
'"$http_user_agent" "$http_cdn_src_ip" "$http_x_forwarded_for"' ;

 

3.寫正則放在那裏呢？

放在logstash 解壓目錄下，建立一個 patterns/nginx文件

URIPARM1 [A-Za-z0-9$.+!*'|(){},~@#%&/=:;_?\-\[\]]*
URIPATH1 (?:/[A-Za-z0-9$.+!*'(){},~:;=@#%&_\- ]*)+
URI1 (%{URIPROTO}://)?(?:%{USER}(?::[^@]*)?@)?(?:%{URIHOST})?(?:%{URIPATHPARAM})?
HOSTPORT %{IPORHOST}:%{POSINT}

NGINXACCESS %{IPORHOST:remote_ip} - (%{USERNAME:user}|-) \[%{HTTPDATE:log_timestamp}\] %{HOSTNAME:http_host} %{WORD:request_method} \"%{URIPATH1:uri}\" \"%{URIPARM1:param}\" %{BASE10NUM:http_status} (?:%{BASE10NUM:body_bytes_sent}|-) \"(?:%{URI1:http_referrer}|-)\" (%{BASE10NUM:upstream_status}|-) (?:%{HOSTPORT:upstream_addr}|-) (%{BASE16FLOAT:upstream_response_time}|-) (%{BASE16FLOAT:request_time}|-) (?:%{QUOTEDSTRING:user_agent}|-) \"(%{IPV4:client_ip}|-)\" \"(%{WORD:x_forword_for}|-)\"

 

 

 

 

 

 

 

 

 

 

 

 

第二類 nginx默認log格式：

 

1.日誌格式

 

(1)

log_format main '$remote_addr - $remote_user [$time_local] "$request" '
'$status $body_bytes_sent "$http_referer" '
'"$http_user_agent" "$http_x_forwarded_for"';

(2)

log_format logstash '$remote_addr - $remote_user [$time_local] "$request" '
'$status $body_bytes_sent $request_time "$http_referer" '
'"$http_user_agent" "$http_x_forwarded_for"';

　　

2.patterns/nginx

(1)
WZ ([^ ]*)
NGINXACCESS %{IP:remote_ip} \- \- \[%{HTTPDATE:timestamp}\] "%{WORD:method} %{WZ:request} HTTP/%{NUMBER:httpversion}" %{NUMBER:status} %{NUMBER:bytes} %{QS:referer} %{QS:agent} %{QS:xforward}

 (2)

NGINXACCESS %{IP:remote_ip} \- \- \[%{HTTPDATE:timestamp}\] "%{WORD:method} %{WZ:request} HTTP/%{NUMBER:httpversion}" %{NUMBER:status} %{NUMBER:bytes} %{BASE16FLOAT:request_time} %{QS:referer} %{QS:agent} \"%{IP:xforward}\"

(2)tomcat

 

 

 3.一段小代碼

input {
file {
path => [ "/var/log/nginx/access.log" ]
start_position => "beginning"
ignore_older => 0
}
}

filter {
grok {
match => { "message" => "%{NGINXACCESS}" }

}
geoip {
source => "remote_ip"
target => "geoip"
database => "/etc/logstash/GeoLiteCity.dat"
add_field => [ "[geoip][coordinates]", "%{[geoip][longitude]}" ]
add_field => [ "[geoip][coordinates]", "%{[geoip][latitude]}" ]
}

mutate {
convert => [ "[geoip][coordinates]", "float" ]
convert => [ "response","integer" ]
convert => [ "bytes","integer" ]
replace => { "type" => "nginx_access" }
remove_field => "message"
}

date {
match => [ "timestamp","dd/MMM/yyyy:HH:mm:ss Z"]

}
mutate {
remove_field => "timestamp"

}

}
output {
elasticsearch {
hosts => ["elk01:9200","elk02:9200","elk03:9200"]
index => "logstash-nginx-access-%{+YYYY.MM.dd}"
}
stdout {codec => rubydebug} #調試的時候用的
}

 

 

以上兩種都已經測試了

 

 

 

使用json添加nginxlogstash

 

log_format logstash_json '{ "@timestamp": "$time_iso8601",'
'"@version":"1",'
'"client": "$http_x_forwarded_for",'
'"request_method": "$request_method",'
'"request_uri": "$request_uri",'
'"status": $status,'
'"host": "$server_addr",'
'"size": $body_bytes_sent,'
'"responsetime": $request_time'
'}';

 

 

input { file { path => "/hwdata/logs/nginx/access_json.log" type => "appstore-accesslog" codec => "json" }}filter { mutate { gsub => [ "client", ",*", "", "client", "-", "本機IP" ] } geoip { source => "client" add_tag => [ "geoip" ] database => "/etc/logstash/GeoLiteCity.dat" fields => ["country_name", "country_code2","region_name", "city_name", "real_region_name", "latitude", "longitude"] remove_field => [ "[geoip][longitude]", "[geoip][latitude]" ] add_field => [ "[geoip][coordinates]", "%{[geoip][longitude]}" ] add_field => [ "[geoip][coordinates]", "%{[geoip][latitude]}" ] } mutate { convert => [ "[geoip][coordinates]", "float"] }}output { redis { host => "redis_IP" port => "6379" db => "6" data_type => "list" key => "appstore-accesslog" }}