搭建部署分佈式ELK平臺（二）

時間 2019-11-07

標籤搭建部署分佈式 elk 平臺欄目系統架構简体版

原文原文鏈接

logstashhtml

• logstash 是什麼java

– logstash是一個數據採集、加工處理以及傳輸的工具git

• logstash 特色:github

– 全部類型的數據集中處理web

– 不一樣模式和格式數據的正常化正則表達式

– 自定義日誌格式的迅速擴展json

– 爲自定義數據源輕鬆添加插件vim

• logstash 安裝數組

– Logstash 依賴 java 環境,須要安裝 java-1.8.0-openjdkruby

– Logstash 沒有默認的配置文件,須要手動配置

– logstash 安裝在 /opt/logstash 目錄下

# rpm -ivh logstash-2.3.4-1.noarch.rpm

# rpm -qc logstash

/etc/init.d/logstash

/etc/logrotate.d/logstash

/etc/sysconfig/logstash

// 查看logstash 模塊插件列表

# /opt/logstash/bin/logstash-plugin list

logstash-codec-collectd

logstash-codec-json

.. ..

logstash-filter-anonymize

logstash-filter-checksum

.. ..

logstash-input-beats

logstash-input-exec

.. ..

logstash-output-cloudwatch

logstash-output-csv

.. ..

logstash-patterns-core

第一列表示是 logstash 的模塊

第二列表示在那個區域段執行 codec 屬於編解碼是字符編碼類型的在所有的區域段能夠運行

• logstash 工做結構

– { 數據源 } ==>

– input { } ==> //收集日誌

– filter { } ==> //日誌處理整理格式

– output { } ==> //日誌輸出

– { ES }

• logstash 裏面的類型

– 布爾值類型: ssl_enable => true

– 字節類型:bytes => "1MiB"

– 字符串類型: name => "xkops"

– 數值類型: port => 22

– 數組: match => ["datetime","UNIX"]

– 哈希: options => {k => "v",k2 => "v2"}

– 編碼解碼: codec => "json"

– 路徑: file_path => "/tmp/filename"

– 註釋: #

• logstash 條件判斷

– 等於: ==

– 不等於: !=

– 小於: <

– 大於: >

– 小於等於: <=

– 大於等於: >=

– 匹配正則: =~

– 不匹配正則: !~

logstash 條件判斷

– 包含: in

– 不包含: not in

– 與: and

– 或: or

– 非與: nand

– 非或: xor

– 複合表達式: ()

– 取反符合: !()

配置 logastash

# cd /etc/logstash/

// logstash 默認沒有配置文件

# vim logstash.conf

input{

stdin{} //標準輸入

}

filter{}

output{

stdout{} // 標準輸出

}

# /opt/logstash/bin/logstash -f logstash.conf // 至關cat

Settings: Default pipeline workers: 1

Pipeline main started

Hello word!!!

2018-01-26T13:21:22.031Z 0.0.0.0 Hello word!!!

– 上頁的配置文件使用了 logstash-input-stdin 和logstash-output-stdout 兩個插件

查看插件具體使用方法的方式是 https://github.com/logstash-plugins

練習1

# vim logstash.conf

input{

stdin{ codec => "json" }

}

filter{}

output{

stdout{}

}

# /opt/logstash/bin/logstash -f logstash.conf

Settings: Default pipeline workers: 1

Pipeline main started

12345 // 123456 不是 json 模式全部報錯

A plugin had an unrecoverable error. Will restart this plugin.

Plugin: <LogStash::Inputs::Stdin codec=><LogStash::Codecs::JSON charset=>"UTF-8">>

Error: can't convert String into Integer {:level=>:error}

'abc'

2018-01-26T13:43:17.840Z 0.0.0.0 'abc'

'{"a":1,"b":2}'

2018-01-26T13:43:46.889Z 0.0.0.0 '{"a":1,"b":2}'

練習2

# vim logstash.conf

input{

stdin{ codec => "json" }

}

filter{}

output{

stdout{ codec => "rubydebug"} //調試數據的模式

}

# /opt/logstash/bin/logstash -f logstash.conf

Settings: Default pipeline workers: 1

Pipeline main started

'"aaa"'

{

"message" => "'\"aaa\"'",

"tags" => [

[0] "_jsonparsefailure"

"@version" => "1",

"@timestamp" => "2018-01-26T13:52:45.307Z",

"host" => "0.0.0.0"

}

{"aa":1,"bb":2}

{

"aa" => 1,

"bb" => 2,

"@version" => "1",

"@timestamp" => "2018-01-26T13:53:00.452Z",

"host" => "0.0.0.0"

}

練習3

# touch /tmp/a.log /tmp/b.log

# vim logstash.conf

input{

file { //監控文件

path => ["/tmp/a.log","/tmp/b.log"] //監控文件路徑

type => "testlog" //聲明文件類型

}

filter{}

output{

stdout{ codec => "rubydebug"}

}

# /opt/logstash/bin/logstash -f logstash.conf

Settings: Default pipeline workers: 1

Pipeline main started

... //開始監控日誌文件

//切換到另外一個終端測試

# cd /tmp/

// 爲日誌文件添加內容

# echo 12345 > a.log

// 這時在會看見終端輸出監控信息

{

"message" => "12345",

"@version" => "1",

"@timestamp" => "2018-01-27T00:45:15.470Z",

"path" => "/tmp/a.log",

"host" => "0.0.0.0",

"type" => "testlog"

}

# echo b123456 > b.log

{

"message" => "b123456",

"@version" => "1",

"@timestamp" => "2018-01-27T00:45:30.487Z",

"path" => "/tmp/b.log",

"host" => "0.0.0.0",

"type" => "testlog"

}

# echo c123456 >> b.log

{

"message" => "c123456",

"@version" => "1",

"@timestamp" => "2018-01-27T00:45:45.501Z",

"path" => "/tmp/b.log",

"host" => "0.0.0.0",

"type" => "testlog"

}

// 默認記錄讀取位置的文件在管理員家目錄 .sincedb_ 後面是一串哈希數

# cat /root/.sincedb_ab3977c541d1144f701eedeb3af4956a

3190503 0 64768 6

3190504 0 64768 16

# du -b /tmp/a.log du -b 查看文件類型

6/tmp/a.log

# du -b /tmp/b.log

16/tmp/b.log

// 進行優化

# vim logstash.conf

input{

file {

start_position => "beginning" //設置當記錄位置的庫文件不存在時從文件開始讀取

sincedb_path => "/var/lib/logstash/sincedb-access" //記錄位置的庫文件默認放在每一個用戶下的因此將記錄位置的庫文件固定位置

path => ["/tmp/a.log","/tmp/b.log"]

type => "testlog"

}

filter{}

output{

stdout{ codec => "rubydebug"}

}

# rm -rf /root/.sincedb_ab3977c541d1144f701eedeb3af4956a

# /opt/logstash/bin/logstash -f logstash.conf

Settings: Default pipeline workers: 1

Pipeline main started

{

"message" => "12345",

"@version" => "1",

"@timestamp" => "2018-01-27T10:44:54.890Z",

"path" => "/tmp/a.log",

"host" => "0.0.0.0",

"type" => "testlog"

}

{

"message" => "b123456",

"@version" => "1",

"@timestamp" => "2018-01-27T10:44:55.242Z",

"path" => "/tmp/b.log",

"host" => "0.0.0.0",

"type" => "testlog"

}

{

"message" => "c123456",

"@version" => "1",

"@timestamp" => "2018-01-27T10:44:55.242Z",

"path" => "/tmp/b.log",

"host" => "0.0.0.0",

"type" => "testlog"

}

練習 tcp udp 插件

# vim logstash.conf

input{

tcp {

host => "0.0.0.0"

port => 8888

type => "tcplog"

}

udp {

host => "0.0.0.0"

port => 9999

type => "udplog"

}

filter{}

output{

stdout{ codec => "rubydebug"}

}

# /opt/logstash/bin/logstash -f logstash.conf

Settings: Default pipeline workers: 1

Pipeline main started

...

//在另外一個終端查看監聽端口

# netstat -pantu | grep -E "(8888|9999)"

tcp6 0 0 :::8888 :::* LISTEN 3098/java

udp6 0 0 :::9999 :::* 3098/java

模擬客戶端發送數據報文

// 發送tcp 數據報文 exec 改變當前文件描述符標準輸入標準輸出從新設置

# exec 9<>/dev/tcp/192.168.4.10/8888 //打開創建鏈接

# echo "hello world" >&9 //發送 hello world 字符串給鏈接

# exec 9<&- // 關閉鏈接

// 接收到鏈接

{

"message" => "hello world",

"@version" => "1",

"@timestamp" => "2018-01-27T11:01:35.356Z",

"host" => "192.168.4.11",

"port" => 48654,

"type" => "tcplog"

}

// 發送udp 數據報文

# exec 7<>/dev/udp/192.168.4.10/9999

# echo "is udp log" >&7

# exec 7<&-

// 接收到鏈接

{

"message" => "is udp log\n",

"@version" => "1",

"@timestamp" => "2018-01-27T11:05:18.850Z",

"type" => "udplog",

"host" => "192.168.4.11"

}

// 發送文件

# exec 8<>/dev/udp/192.168.4.10/9999

# cat /etc/hosts >&8

# exec 9<&-

// 接受到信息

{

"message" => "127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4\n::1 localhost localhost.localdomain localhost6 localhost6.localdomain6\n192.168.4.11\tes1\n192.168.4.12\tes2\n192.168.4.13\tes3\n192.168.4.14\tes4\n192.168.4.15\tes5\n",

"@version" => "1",

"@timestamp" => "2018-01-27T11:10:31.099Z",

"type" => "udplog",

"host" => "192.168.4.11"

}

syslog 插件練習

# vim logstash.conf

input{

syslog{

host => "192.168.4.10"

port => 514 //系統日誌默認端口

type => "syslog"

}

filter{}

output{

stdout{ codec => "rubydebug"}

}

# /opt/logstash/bin/logstash -f logstash.conf

Settings: Default pipeline workers: 1

Pipeline main started

. . .

# netstat -pantu | grep 514

tcp6 0 0 192.168.4.10:514 :::* LISTEN 3545/java

udp6 0 0 192.168.4.10:514 :::* 3545/java

//在客戶端主機上自定義日誌文件發給logstash 主機

# vim /etc/rsyslog.conf

# sed -n '74p' /etc/rsyslog.conf

local0.info@@192.168.4.10:514

# systemctl restart rsyslog.service

# logger -p local0.info -t testlog "hello world" // 發送一條測試日誌

// logstash 主機收到日誌

{

"message" => "hello world\n",

"@version" => "1",

"@timestamp" => "2018-01-27T11:29:03.000Z",

"type" => "syslog",

"host" => "192.168.4.11",

"priority" => 134,

"timestamp" => "Jan 27 06:29:03",

"logsource" => "es1",

"program" => "testlog",

"severity" => 6,

"facility" => 16,

"facility_label" => "local0",

"severity_label" => "Informational"

}

擴展若是想要把登陸日誌發給 logstash

# vim /etc/rsyslog.conf

# sed -n '75p' /etc/rsyslog.conf

authpriv.*@@192.168.4.10:514

# systemctl restart rsyslog.service

//測試登陸登出

# exit

登出

Connection to 192.168.4.11 closed.

# ssh -X root@192.168.4.11

root@192.168.4.11's password:

Last login: Sat Jan 27 05:27:21 2018 from 192.168.4.254

//實時接受的日誌

{

"message" => "Accepted password for root from 192.168.4.254 port 50820 ssh2\n",

"@version" => "1",

"@timestamp" => "2018-01-27T11:32:07.000Z",

"type" => "syslog",

"host" => "192.168.4.11",

"priority" => 86,

"timestamp" => "Jan 27 06:32:07",

"logsource" => "es1",

"program" => "sshd",

"pid" => "3734",

"severity" => 6,

"facility" => 10,

"facility_label" => "security/authorization",

"severity_label" => "Informational"

}

{

"message" => "pam_unix(sshd:session): session opened for user root by (uid=0)\n",

"@version" => "1",

"@timestamp" => "2018-01-27T11:32:07.000Z",

"type" => "syslog",

"host" => "192.168.4.11",

"priority" => 86,

"timestamp" => "Jan 27 06:32:07",

"logsource" => "es1",

"program" => "sshd",

"pid" => "3734",

"severity" => 6,

"facility" => 10,

"facility_label" => "security/authorization",

"severity_label" => "Informational"

}

• filter grok插件

– 解析各類非結構化的日誌數據插件

– grok 使用正則表達式把飛結構化的數據結構化

– 在分組匹配,正則表達式須要根據具體數據結構編寫

– 雖然編寫困難,但適用性極廣

– 幾乎能夠應用於各種數據

• grok 正則分組匹配

– 匹配 ip 時間戳和請求方法

"(?<ip>(\d+\.){3}\d+) \S+ \S+

(?<time>.*\])\s+\"(?<method>[A-Z]+)"]

– 使用正則宏

%{IPORHOST:clientip} %{HTTPDUSER:ident} %{USER:auth}

\[%{HTTPDATE:timestamp}\] \"%{WORD:verb}

– 最終版本

%{COMMONAPACHELOG} \"(?<referer>[^\"]+)\"

\"(?<UA>[^\"]+)\"

練習匹配Apache 日誌

# cd /opt/logstash/vendor/bundle/jruby/1.9/gems/logstash-patterns-core-2.0.5/patterns/

# vim grok-patterns //日誌宏定義倉庫

....

COMMONAPACHELOG %{IPORHOST:clientip} %{HTTPDUSER:ident} %{USER:auth} \[%{HTTPDATE:timestamp}\] "(?:%{WORD:verb} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest})" %{NUMBER:response} (?:%{NUMBER:bytes}|-)

....

# vim /tmp/test.log //在測試文件中添加一條日誌

220.181.108.115 - - [11/Jul/2017:03:07:16 +0800] "GET /%B8%DF%BC%B6%D7%E2%C1%DE%D6%F7%C8%CE/QYQiu_j.html HTTP/1.1" 200 20756 "-" "Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)" "-" zxzs.buildhr.com 558742

# vim logstash.conf

input{

file{

start_position => "beginning"

sincedb_path => "/dev/null" //爲了調試方便

path => [ "/tmp/test.log" ]

type => 'filelog'

}

filter{

grok{

match => ["message","%{COMMONAPACHELOG}"]

}

output{

stdout{ codec => "rubydebug"}

}

# /opt/logstash/bin/logstash -f logstash.conf Settings: Default pipeline workers: 1

Pipeline main started

{

"message" => "220.181.108.115 - - [11/Jul/2017:03:07:16 +0800] \"GET /%B8%DF%BC%B6%D7%E2%C1%DE%D6%F7%C8%CE/QYQiu_j.html HTTP/1.1\" 200 20756 \"-\" \"Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)\" \"-\" zxzs.buildhr.com 558742",

"@version" => "1",

"@timestamp" => "2018-01-27T12:03:10.363Z",

"path" => "/tmp/test.log",

"host" => "0.0.0.0",

"type" => "filelog",

"clientip" => "220.181.108.115",

"ident" => "-",

"auth" => "-",

"timestamp" => "11/Jul/2017:03:07:16 +0800",

"verb" => "GET",

"request" => "/%B8%DF%BC%B6%D7%E2%C1%DE%D6%F7%C8%CE/QYQiu_j.html",

"httpversion" => "1.1",

"response" => "200",

"bytes" => "20756"

}

{

"message" => "",

"@version" => "1",

"@timestamp" => "2018-01-27T12:03:10.584Z",

"path" => "/tmp/test.log",

"host" => "0.0.0.0",

"type" => "filelog",

"tags" => [

[0] "_grokparsefailure"

]

}

編寫分析日誌正則時先區宏庫中尋找找不到能夠去百度儘可能不要本身手動寫會很累

練習同時解析不一樣日誌

# vim logstash.conf

input{

file{

start_position => "beginning"

sincedb_path => "/dev/null"

path => [ "/tmp/test.log" ]

type => 'filelog'

}

file{

start_position => "beginning"

sincedb_path => "/dev/null"

path => [ "/tmp/test.json" ]

type => 'jsonlog'

codec => 'json'

}

filter{

if [type] == "filelog"{

grok{

match => ["message","%{COMMONAPACHELOG}"]

}

output{

stdout{ codec => "rubydebug"}

}

[root@klog logstash]# /opt/logstash/bin/logstash -f logstash.conf

Settings: Default pipeline workers: 1

Pipeline main started

{

"@version" => "1",

"@timestamp" => "2018-01-27T12:22:06.481Z",

"path" => "/tmp/test.log",

"host" => "0.0.0.0",

"type" => "filelog",

"clientip" => "220.181.108.115",

"ident" => "-",

"auth" => "-",

"timestamp" => "11/Jul/2017:03:07:16 +0800",

"verb" => "GET",

"request" => "/%B8%DF%BC%B6%D7%E2%C1%DE%D6%F7%C8%CE/QYQiu_j.html",

"httpversion" => "1.1",

"response" => "200",

"bytes" => "20756"

}

{

"message" => "",

"@version" => "1",

"@timestamp" => "2018-01-27T12:22:06.834Z",

"path" => "/tmp/test.log",

"host" => "0.0.0.0",

"type" => "filelog",

"tags" => [

[0] "_grokparsefailure"

]

}

{

"@timestamp" => "2015-05-18T12:28:25.013Z",

"ip" => "79.1.14.87",

"extension" => "gif",

"response" => "200",

"geo" => {

"coordinates" => {

"lat" => 35.16531472,

"lon" => -107.9006142

"src" => "GN",

"dest" => "US",

"srcdest" => "GN:US"

"@tags" => [

[0] "success",

[1] "info"

"utc_time" => "2015-05-18T12:28:25.013Z",

"referer" => "http://www.slate.com/warning/b-alvin-drew",

"agent" => "Mozilla/5.0 (X11; Linux i686) AppleWebKit/534.24 (KHTML, like Gecko) Chrome/11.0.696.50 Safari/534.24",

"clientip" => "79.1.14.87",

"bytes" => 774,

"host" => "motion-media.theacademyofperformingartsandscience.org",

"request" => "/canhaz/james-mcdivitt.gif",

"url" => "https://motion-media.theacademyofperformingartsandscience.org/canhaz/james-mcdivitt.gif",

"@message" => "79.1.14.87 - - [2015-05-18T12:28:25.013Z] \"GET /canhaz/james-mcdivitt.gif HTTP/1.1\" 200 774 \"-\" \"Mozilla/5.0 (X11; Linux i686) AppleWebKit/534.24 (KHTML, like Gecko) Chrome/11.0.696.50 Safari/534.24\"",

"spaces" => "this is a thing with lots of spaces wwwwoooooo",

"xss" => "<script>console.log(\"xss\")</script>",

"headings" => [

[0] "<h3>charles-bolden</h5>",

[1] "http://www.slate.com/success/barry-wilmore"

"links" => [

[0] "george-nelson@twitter.com",

[1] "http://facebook.com/info/anatoly-solovyev",

[2] "www.www.slate.com"

"relatedContent" => [],

"machine" => {

"os" => "osx",