elk es
能把修改過的行顯示出來
[root@es2 ~]# grep -Pv "^(#|$)" /etc/elasticsearch/elasticsearch.yml
cluster.name: nsd1808
node.name: es2
network.host: 0.0.0.0
discovery.zen.ping.unicast.hosts: ["es1", "es2", "es3"]php
用命令行訪問
[root@es1 ~]# curl 118.144.89.240/info.php
<pre>
[ REQUEST_METHOD] ==> GET
[ REMOTE_ADDR] ==> 14.20.38.115
[HTTP_USER_AGENT] ==> curl/7.29.0 顯示客戶端的設備信息
[ HTTP_REFERER] ==> html
--- --- GET options is --- ---
Array
(
)java
谷歌瀏覽器訪問
[ REQUEST_METHOD] ==> GET
[ REMOTE_ADDR] ==> 14.20.38.115
[HTTP_USER_AGENT] ==> Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.113 Safari/537.36
[ HTTP_REFERER] ==> node
--- --- GET options is --- ---
Array
(
)python
[root@es1 ~]# curl -A "test" http://118.144.89.240/info.php 用-A能夠修改顯示的客戶端設備信息
<pre>
[ REQUEST_METHOD] ==> GET
[ REMOTE_ADDR] ==> 14.20.38.115
[HTTP_USER_AGENT] ==> test
[ HTTP_REFERER] ==> linux
--- --- GET options is --- ---
Array
(
)ios
[root@es1 ~]# curl -X "GET" http://118.144.89.240/info.php
<pre>
[ REQUEST_METHOD] ==> GET
[ REMOTE_ADDR] ==> 14.20.38.115
[HTTP_USER_AGENT] ==> curl/7.29.0
[ HTTP_REFERER] ==> web
--- --- GET options is --- ---
Array
(
)正則表達式
[root@es1 ~]# curl -X "DELETE" http://118.144.89.240/info.php
<pre>
[ REQUEST_METHOD] ==> DELETE
[ REMOTE_ADDR] ==> 14.20.38.115
[HTTP_USER_AGENT] ==> curl/7.29.0
[ HTTP_REFERER] ==> redis
--- --- DELETE options is --- ---
Array
(
)
分片數要小於等於集羣的臺數，副本數要少於分片數
[root@es5 ~]# cd bigdesk
[root@es5 bin]# ./plugin install file:///root/elk軟件包/elasticsearch-head-master.zip
[root@es5 bin]# ./plugin install file:///root/elk軟件包/elasticsearch-kopf-master.zip
[root@es5 bin]# ./plugin install file:///root/elk軟件包/bigdesk-master.zip
[root@es5 bin]# ./plugin list
火狐瀏覽器訪問
http://192.168.3.115:9200/_plugin/head/
http://192.168.3.115:9200/_plugin/kopf/
http://192.168.3.115:9200/_plugin/bigdesk/
[root@es5 ~]# curl http://192.168.3.112:9200/_cat
=^.^=
/_cat/allocation
/_cat/shards
/_cat/shards/{index}
/_cat/master
/_cat/nodes
/_cat/indices
/_cat/indices/{index}
/_cat/segments
/_cat/segments/{index}
/_cat/count
/_cat/count/{index}
/_cat/recovery
/_cat/recovery/{index}
/_cat/health
/_cat/pending_tasks
/_cat/aliases
/_cat/aliases/{alias}
/_cat/thread_pool
/_cat/plugins
/_cat/fielddata
/_cat/fielddata/{fields}
/_cat/nodeattrs
/_cat/repositories
/_cat/snapshots/{repository}
[root@es5 ~]# curl http://192.168.3.112:9200/_cat/master
-f2FWBalRg-MC0lq-QlNUQ 192.168.3.113 192.168.3.113 es3
[root@es5 ~]# curl http://192.168.3.112:9200/_cat/master?v
id host ip node
-f2FWBalRg-MC0lq-QlNUQ 192.168.3.113 192.168.3.113 es3
[root@es5 ~]# curl http://192.168.3.112:9200/_cat/master?help
id | | node id
host | h | host name
ip | | ip address
node | n | node name
[root@es5 ~]# curl http://192.168.3.112:9200/_cat/nodes
192.168.3.113 192.168.3.113 3 55 0.00 d * es3 有星號*表明master
192.168.3.114 192.168.3.114 9 53 0.00 d m es4
192.168.3.115 192.168.3.115 12 67 0.00 d m es5
192.168.3.112 192.168.3.112 10 55 0.00 d m es2
192.168.3.111 192.168.3.111 10 64 0.01 d m es1
[root@es5 ~]# curl http://192.168.3.112:9200/_cat/indices?v
health status index pri rep docs.count docs.deleted store.size pri.store.size
green open abc 5 1 0 0 1.5kb 795b
[root@es5 ~]# curl http://192.168.3.112:9200/_cat/shards?v
index shard prirep state docs store ip node
abc 1 p STARTED 0 159b 192.168.3.111 es1 這裏p表明主，master
abc 1 r STARTED 0 159b 192.168.3.112 es2
abc 4 r STARTED 0 159b 192.168.3.113 es3
abc 4 p STARTED 0 159b 192.168.3.115 es5
abc 2 p STARTED 0 159b 192.168.3.112 es2
abc 2 r STARTED 0 159b 192.168.3.114 es4
abc 3 p STARTED 0 159b 192.168.3.114 es4
abc 3 r STARTED 0 159b 192.168.3.115 es5
abc 0 p STARTED 0 159b 192.168.3.113 es3
abc 0 r STARTED 0 159b 192.168.3.111 es1
#############################################################################################
任意一臺eslastsearch虛擬機。
這裏在虛擬機es5
在文本里寫好如下內容後，再粘貼到命令行
curl -XPUT http://192.168.3.112:9200/nsd1808 -d '
{
"settings": {
"index": {
"number_of_shards": 5 ,
"number_of_replicas": 1
}
}
}'
這是在命令行粘貼後的結果
[root@es5 ~]# curl -XPUT http://192.168.3.112:9200/nsd1808 -d '
> {
> "settings": {
> "index": {
> "number_of_shards": 5 ,
> "number_of_replicas": 1
> }
> }
> }'
在文本里寫好如下內容後，再粘貼到命令行
curl -XPUT http://192.168.3.112:9200/nsd1808/xs/1 -d '
{
"姓名": "張三",
"愛好": "吃",
"階段": 1,
"年齡": 20
}'
這是在命令行粘貼後的結果
[root@es5 ~]# curl -XPUT http://192.168.3.112:9200/nsd1808/xs/1 -d '
> {
> "姓名": "張三",
> "愛好": "吃",
> "階段": 1,
> "年齡": 20
> }'
顯示以下：
{"_index":"nsd1808","_type":"xs","_id":"1","_version":1,"_shards":{"total":2,"successful":2,"failed":0},"created":true}[root@es5 ~]#
在文本里寫好如下內容後，再粘貼到命令行
curl -XPOST http://192.168.3.112:9200/nsd1807/xs/1/_update -d '
{
"doc":{
"愛好": "吃喝睡"
}
}'
這是在命令行粘貼後的結果
[root@es5 ~]# curl -XPOST http://192.168.3.112:9200/nsd1807/xs/1/_update -d '
> {
> "doc":{
> "愛好": "吃喝睡"
> }
> }'
顯示以下：
{"_index":"nsd1807","_type":"xs","_id":"1","_version":2,"_shards":{"total":2,"successful":2,"failed":0}}[root@es5 ~]#
[root@es5 ~]# curl -XDELETE http://192.168.3.112:9200/nsd1808
顯示以下：
{"acknowledged":true}
[root@es5 ~]# curl -XDELETE http://192.168.3.112:9200/*
顯示以下：
{"acknowledged":true}
#############################################################################################
虛擬機kibana
[root@kibana json]# ls
accounts.json.gz logs.jsonl.gz shakespeare.json.gz
[root@kibana json]# for i in * ; do gzip -d $i ;done 解包
[root@kibana json]# ls
accounts.json logs.jsonl shakespeare.json
[root@kibana json]# curl -XPOST http://192.168.3.111:9200/_bulk --data-binary @shakespeare.json
[root@kibana json]# curl -XPOST http://192.168.3.111:9200/_bulk --data-binary @logs.jsonl
索引是aa，類型是bb，必須導入索引和類型，沒有索引，要加上
[root@kibana json]# curl -XPOST http://192.168.3.111:9200/aa/bb/_bulk --data-binary @accounts.json
#############################################################################################
虛擬機kibana
在文本里寫好如下內容後，再粘貼到命令行
curl -XGET http://192.168.3.111:9200/_mget?pretty -d '
{
"docs" : [
{
"_index" : "aa",
"_type" : "bb",
"_id" : "190"
},
{
"_index" : "shakespeare",
"_type" : "line",
"_id" : "24"
},
{
"_index" : "logstash-2015.05.18",
"_type" : "log",
"_id" : "AWfoHycao9bttkMidMF9"
}
]
}'
這是在命令行粘貼後的結果
[root@kibana json]# curl -XGET http://192.168.3.111:9200/_mget?pretty -d '
> {
> "docs" : [
> {
> "_index" : "aa",
> "_type" : "bb",
> "_id" : "190"
> },
> {
> "_index" : "shakespeare",
> "_type" : "line",
> "_id" : "24"
> },
> {
> "_index" : "logstash-2015.05.18",
> "_type" : "log",
> "_id" : "AWfoHycao9bttkMidMF9"
> }
> ]
> }'
顯示以下：
{
"docs" : [ {
"_index" : "aa",
"_type" : "bb",
"_id" : "190",
"_version" : 1,
"found" : true,
"_source" : {
"account_number" : 190,
"balance" : 3150,
"firstname" : "Blake",
"lastname" : "Davidson",
"age" : 30,
"gender" : "F",
"address" : "636 Diamond Street",
"employer" : "Quantasis",
"email" : "blakedavidson@quantasis.com",
"city" : "Crumpler",
"state" : "KY"
}
}, {
"_index" : "shakespeare",
"_type" : "line",
"_id" : "24",
"_version" : 1,
"found" : true,
"_source" : {
"line_id" : 25,
"play_name" : "Henry IV",
"speech_number" : 1,
"line_number" : "1.1.22",
"speaker" : "KING HENRY IV",
"text_entry" : "Forthwith a power of English shall we levy;"
}
}, {
"_index" : "logstash-2015.05.18",
"_type" : "log",
"_id" : "AWfoHycao9bttkMidMF9",
"_version" : 1,
"found" : true,
"_source" : {
"@timestamp" : "2015-05-18T19:25:45.051Z",
"ip" : "210.155.113.62",
"extension" : "jpg",
"response" : "200",
"geo" : {
"coordinates" : {
"lat" : 48.47088889,
"lon" : -122.4208611
},
"src" : "CO",
"dest" : "IN",
"srcdest" : "CO:IN"
},
"@tags" : [ "success", "info" ],
"utc_time" : "2015-05-18T19:25:45.051Z",
"referer" : "http://www.slate.com/success/duane-g-carey",
"agent" : "Mozilla/5.0 (X11; Linux i686) AppleWebKit/534.24 (KHTML, like Gecko) Chrome/11.0.696.50 Safari/534.24",
"clientip" : "210.155.113.62",
"bytes" : 5630,
"host" : "media-for-the-masses.theacademyofperformingartsandscience.org",
"request" : "/uploads/owen-garriott.jpg",
"url" : "https://media-for-the-masses.theacademyofperformingartsandscience.org/uploads/owen-garriott.jpg",
"@message" : "210.155.113.62 - - [2015-05-18T19:25:45.051Z] \"GET /uploads/owen-garriott.jpg HTTP/1.1\" 200 5630 \"-\" \"Mozilla/5.0 (X11; Linux i686) AppleWebKit/534.24 (KHTML, like Gecko) Chrome/11.0.696.50 Safari/534.24\"",
"spaces" : "this is a thing with lots of spaces wwwwoooooo",
"xss" : "<script>console.log(\"xss\")</script>",
"headings" : [ "<h3>lawrence-j-delucas</h5>", "http://www.slate.com/success/robert-s-kimbrough" ],
"links" : [ "charles-camarda@www.slate.com", "http://nytimes.com/info/mark-brown", "www.twitter.com" ],
"relatedContent" : [ {
"url" : "http://www.laweekly.com/news/tim-biskup-mural-on-la-brea-2368038",
"og:type" : "article",
"og:title" : "Tim Biskup Mural on La Brea",
"og:description" : "Yeah, it's actually an ad for Helio... Kind of. There's a small Helio logo off to the left side. But for the most part it's just a cool huge original pa...",
"og:url" : "http://www.laweekly.com/news/tim-biskup-mural-on-la-brea-2368038",
"article:published_time" : "2007-12-20T07:16:16-08:00",
"article:modified_time" : "2014-10-28T14:59:54-07:00",
"article:section" : "News",
"og:image" : "http://IMAGES1.laweekly.com/imager/tim-biskup-mural-on-la-brea/u/original/2430352/img_1746.jpg",
"og:image:height" : "360",
"og:image:width" : "480",
"og:site_name" : "LA Weekly",
"twitter:title" : "Tim Biskup Mural on La Brea",
"twitter:description" : "Yeah, it's actually an ad for Helio... Kind of. There's a small Helio logo off to the left side. But for the most part it's just a cool huge original pa...",
"twitter:card" : "summary",
"twitter:image" : "http://IMAGES1.laweekly.com/imager/tim-biskup-mural-on-la-brea/u/original/2430352/img_1746.jpg",
"twitter:site" : "@laweekly"
}, {
"url" : "http://www.laweekly.com/music/friday-video-ozark-jubilee-2400136",
"og:type" : "article",
"og:title" : "Friday Video: Ozark Jubilee",
"og:description" : "The Ozark Jubilee: Li'l Liza Jane and more clips from the 1950s TV show",
"og:url" : "http://www.laweekly.com/music/friday-video-ozark-jubilee-2400136",
"article:published_time" : "2007-07-20T12:40:45-07:00",
"article:modified_time" : "2014-11-27T07:07:25-08:00",
"article:section" : "Music",
"og:site_name" : "LA Weekly",
"twitter:title" : "Friday Video: Ozark Jubilee",
"twitter:description" : "The Ozark Jubilee: Li'l Liza Jane and more clips from the 1950s TV show",
"twitter:card" : "summary",
"twitter:site" : "@laweekly"
}, {
"url" : "http://www.laweekly.com/arts/rock-with-me-2374144",
"og:type" : "article",
"og:title" : "Rock With Me",
"og:description" : ""How many boulders can we possibly have to climb over?" I chuckled to my friend Julie, when she warned me that her boyfriend David's friend Ben had aske...",
"og:url" : "http://www.laweekly.com/arts/rock-with-me-2374144",
"article:published_time" : "2005-11-14T20:11:46-08:00",
"article:modified_time" : "2014-11-25T18:43:01-08:00",
"article:section" : "Arts",
"article:tag" : "Malibu Creek State",
"og:image" : "http://images1.laweekly.com/imager/rock-with-me/u/original/2444410/shimmy1.jpg",
"og:image:height" : "666",
"og:image:width" : "500",
"og:site_name" : "LA Weekly",
"twitter:title" : "Rock With Me",
"twitter:description" : ""How many boulders can we possibly have to climb over?" I chuckled to my friend Julie, when she warned me that her boyfriend David's friend Ben had aske...",
"twitter:card" : "summary",
"twitter:image" : "http://images1.laweekly.com/imager/rock-with-me/u/original/2444410/shimmy1.jpg",
"twitter:site" : "@laweekly"
} ],
"machine" : {
"os" : "win 7",
"ram" : 13958643712
},
"@version" : "1"
}
} ]
}
#############################################################################################
虛擬機logstash
[root@logstash ~]# cd /etc/logstash/
[root@logstash logstash]# ls
conf.d
[root@logstash logstash]# vim logstash.conf
input{
stdin{}
}
filter{}
output{
stdout{}
}
[root@logstash logstash]# cd /opt/logstash/bin/
[root@logstash bin]# ls
logstash logstash.bat logstash.lib.sh logstash-plugin logstash-plugin.bat plugin plugin.bat rspec rspec.bat setup.bat
[root@logstash bin]# alias logstash=/opt/logstash/bin/logstash
[root@logstash bin]# cd /etc/logstash/
[root@logstash logstash]# ls
conf.d logstash.conf
[root@logstash logstash]# logstash -f logstash.conf 用-f來指定文件路徑
Settings: Default pipeline workers: 2
Pipeline main started
aaaaaaaaaaaaaaaaaaaaa
2018-12-26T06:26:10.830Z logstash aaaaaaaaaaaaaaaaaaaaa
bbbbbbbbbbbb
2018-12-26T06:26:13.853Z logstash bbbbbbbbbbbb
cccccccccccc
2018-12-26T06:26:16.189Z logstash cccccccccccc
ssssssssssss
2018-12-26T06:26:18.741Z logstash ssssssssssss
ddddddddddddddddd
2018-12-26T06:26:21.570Z logstash ddddddddddddddddd
kkkkkkk
2018-12-26T06:26:24.609Z logstash kkkkkkk
^CSIGINT received. Shutting down the agent. {:level=>:warn}
stopping pipeline {:id=>"main"}
Received shutdown signal, but pipeline is still waiting for in-flight events
to be processed. Sending another ^C will force quit Logstash, but this may cause
data loss. {:level=>:warn}
^CSIGINT received. Terminating immediately.. {:level=>:fatal}
########################################################################################
參考：
[root@logstash logstash]# logstash --help
Usage:
/bin/logstash agent [OPTIONS]
Options:
-f, --config CONFIG_PATH Load the logstash config from a specific file
or directory. If a directory is given, all
files in that directory will be concatenated
in lexicographical order and then parsed as a
single config file. You can also specify
wildcards (globs) and any matched files will
be loaded in the order described above.
-e CONFIG_STRING Use the given string as the configuration
data. Same syntax as the config file. If no
input is specified, then the following is
used as the default input:
"input { stdin { type => stdin } }"
and if no output is specified, then the
following is used as the default output:
"output { stdout { codec => rubydebug } }"
If you wish to use both defaults, please use
the empty string for the '-e' flag.
(default: "")
-w, --pipeline-workers COUNT Sets the number of pipeline workers to run.
(default: 2)
-b, --pipeline-batch-size SIZE Size of batches the pipeline is to work in.
(default: 125)
-u, --pipeline-batch-delay DELAY_IN_MS When creating pipeline batches, how long to wait while polling
for the next event.
(default: 5)
--filterworkers COUNT DEPRECATED. Now an alias for --pipeline-workers and -w
-l, --log FILE Write logstash internal logs to the given
file. Without this flag, logstash will emit
logs to standard output.
-v Increase verbosity of logstash internal logs.
Specifying once will show 'informational'
logs. Specifying twice will show 'debug'
logs. This flag is deprecated. You should use
--verbose or --debug instead.
--quiet Quieter logstash logging. This causes only
errors to be emitted.
--verbose More verbose logging. This causes 'info'
level logs to be emitted.
--debug Most verbose logging. This causes 'debug'
level logs to be emitted.
--debug-config Print the compiled config ruby code out as a debug log (you must also have --debug enabled).
WARNING: This will include any 'password' options passed to plugin configs as plaintext, and may result
in plaintext passwords appearing in your logs!
(default: false)
-V, --version Emit the version of logstash and its friends,
then exit.
-p, --pluginpath PATH A path of where to find plugins. This flag
can be given multiple times to include
multiple paths. Plugins are expected to be
in a specific directory hierarchy:
'PATH/logstash/TYPE/NAME.rb' where TYPE is
'inputs' 'filters', 'outputs' or 'codecs'
and NAME is the name of the plugin.
-t, --configtest Check configuration for valid syntax and then exit.
--[no-]allow-unsafe-shutdown Force logstash to exit during shutdown even
if there are still inflight events in memory.
By default, logstash will refuse to quit until all
received events have been pushed to the outputs.
(default: false)
-r, --[no-]auto-reload Monitor configuration changes and reload
whenever it is changed.
NOTE: use SIGHUP to manually reload the config
(default: false)
--reload-interval RELOAD_INTERVAL How frequently to poll the configuration location
for changes, in seconds.
(default: 3)
--allow-env EXPERIMENTAL. Enables templating of environment variable
values. Instances of "${VAR}" in strings will be replaced
with the respective environment variable value named "VAR".
(default: false)
-h, --help print help
########################################################################################
參考：
[root@logstash bin]# cd /opt/logstash/bin/
[root@logstash bin]# ls
logstash logstash.bat logstash.lib.sh logstash-plugin logstash-plugin.bat plugin plugin.bat rspec rspec.bat setup.bat
[root@logstash bin]# ./logstash-plugin list
Ignoring ffi-1.9.13 because its extensions are not built. Try: gem pristine ffi --version 1.9.13
logstash-codec-collectd
logstash-codec-dots
logstash-codec-edn
logstash-codec-edn_lines
logstash-codec-es_bulk
logstash-codec-fluent
logstash-codec-graphite
logstash-codec-json
logstash-codec-json_lines
logstash-codec-line
logstash-codec-msgpack
logstash-codec-multiline
logstash-codec-netflow
logstash-codec-oldlogstashjson
logstash-codec-plain
logstash-codec-rubydebug
logstash-filter-anonymize
logstash-filter-checksum
logstash-filter-clone
logstash-filter-csv
logstash-filter-date
logstash-filter-dns
logstash-filter-drop
logstash-filter-fingerprint
logstash-filter-geoip
logstash-filter-grok
logstash-filter-json
logstash-filter-kv
logstash-filter-metrics
logstash-filter-multiline
logstash-filter-mutate
logstash-filter-ruby
logstash-filter-sleep
logstash-filter-split
logstash-filter-syslog_pri
logstash-filter-throttle
logstash-filter-urldecode
logstash-filter-useragent
logstash-filter-uuid
logstash-filter-xml
logstash-input-beats
logstash-input-couchdb_changes
logstash-input-elasticsearch
logstash-input-eventlog
logstash-input-exec
logstash-input-file
logstash-input-ganglia
logstash-input-gelf
logstash-input-generator
logstash-input-graphite
logstash-input-heartbeat
logstash-input-http
logstash-input-http_poller
logstash-input-imap
logstash-input-irc
logstash-input-jdbc
logstash-input-kafka
logstash-input-log4j
logstash-input-lumberjack
logstash-input-pipe
logstash-input-rabbitmq
logstash-input-redis
logstash-input-s3
logstash-input-snmptrap
logstash-input-sqs
logstash-input-stdin
logstash-input-syslog
logstash-input-tcp
logstash-input-twitter
logstash-input-udp
logstash-input-unix
logstash-input-xmpp
logstash-input-zeromq
logstash-output-cloudwatch
logstash-output-csv
logstash-output-elasticsearch
logstash-output-email
logstash-output-exec
logstash-output-file
logstash-output-ganglia
logstash-output-gelf
logstash-output-graphite
logstash-output-hipchat
logstash-output-http
logstash-output-irc
logstash-output-juggernaut
logstash-output-kafka
logstash-output-lumberjack
logstash-output-nagios
logstash-output-nagios_nsca
logstash-output-null
logstash-output-opentsdb
logstash-output-pagerduty
logstash-output-pipe
logstash-output-rabbitmq
logstash-output-redis
logstash-output-s3
logstash-output-sns
logstash-output-sqs
logstash-output-statsd
logstash-output-stdout
logstash-output-tcp
logstash-output-udp
logstash-output-xmpp
logstash-output-zeromq
logstash-patterns-core
#############################################################################################
網頁訪問插件文檔
https://www.elastic.co/guide/en/logstash/current/index.html
在Input plugins 那裏點擊左邊的+，右鍵file，選擇在新標籤中打開
#############################################################################################
虛擬機logstash
[root@logstash logstash]# cd /etc/logstash
[root@logstash logstash]# vim logstash.conf
input{
stdin{ codec => "json" }
}
filter{}
output{
stdout{ codec => "rubydebug" }
}
[root@logstash logstash]# logstash -f logstash.conf
Settings: Default pipeline workers: 2
Pipeline main started
aaaaaaaaaaa
{
"message" => "aaaaaaaaaaa",
"tags" => [
[0] "_jsonparsefailure"
],
"@version" => "1",
"@timestamp" => "2018-12-26T06:49:08.292Z",
"host" => "logstash"
}
{"a":1,"b":2,"c":3}
{
"a" => 1,
"b" => 2,
"c" => 3,
"@version" => "1",
"@timestamp" => "2018-12-26T06:49:36.806Z",
"host" => "logstash"
}
#############################################################################################
虛擬機logstash
[root@logstash logstash]# vim logstash.conf
input{
file {
path => ["/tmp/a.log","/var/tmp/b.log"]
}
}
filter{}
output{
stdout{ codec => "rubydebug" }
}
[root@logstash logstash]# touch /tmp/a.log
[root@logstash logstash]# touch /var/tmp/b.log
[root@logstash logstash]# logstash -f logstash.conf
Settings: Default pipeline workers: 2
Pipeline main started
另開一個終端，給2個文件輸入東西
[root@logstash ~]# echo A_${RANDOM}
A_19459
[root@logstash ~]# echo A_${RANDOM} >> /tmp/a.log
[root@logstash ~]# echo B_${RANDOM} >> /var/tmp/b.log
在原來的終端會發現跳出來，咱們在另外一個終端裏，往文件裏寫入的信息
[root@logstash logstash]# logstash -f logstash.conf
Settings: Default pipeline workers: 2
Pipeline main started
會跳出來以下信息：
{
"message" => "A_26708",
"@version" => "1",
"@timestamp" => "2018-12-26T07:31:15.675Z",
"path" => "/tmp/a.log",
"host" => "logstash"
}
{
"message" => "B_21",
"@version" => "1",
"@timestamp" => "2018-12-26T07:31:30.707Z",
"path" => "/var/tmp/b.log",
"host" => "logstash"
}
兩次ctrl+c退出
而後再往文件寫入內容
[root@logstash logstash]# echo aaa >>/tmp/a.log
[root@logstash logstash]# echo bbb>>/var/tmp/b.log
再次啓動服務，還能讀出上次沒有讀完的新內容
[root@logstash logstash]# logstash -f logstash.conf
Settings: Default pipeline workers: 2
Pipeline main started
{
"message" => "aaa",
"@version" => "1",
"@timestamp" => "2018-12-26T07:40:19.504Z",
"path" => "/tmp/a.log",
"host" => "logstash"
}
{
"message" => "bbb",
"@version" => "1",
"@timestamp" => "2018-12-26T07:40:19.605Z",
"path" => "/var/tmp/b.log",
"host" => "logstash"
}
[root@logstash logstash]# cat /etc/logstash/logstash.conf
input{
file {
path => ["/tmp/a.log","/var/tmp/b.log"]
sincedb_path => "/var/lib/logstash/sincedb" 增長這行，指定新的讀取文件的位置
}
}
filter{}
output{
stdout{ codec => "rubydebug" }
}
[root@logstash logstash]# ls -A /root/
.sincedb_e9a1772295a869da80134b5c4e75816e ... ... 這是默認的指定讀取文件位置的文件
[root@logstash logstash]# cat /root/.sincedb_e9a1772295a869da80134b5c4e75816e
467997 0 64769 12
3984678 0 64769 5
[root@logstash logstash]# rm -rf /root/.sincedb_e9a1772295a869da80134b5c4e75816e
[root@logstash ~]# echo 11111 >> /tmp/a.log
[root@logstash logstash]# logstash -f logstash.conf
Settings: Default pipeline workers: 2
Pipeline main started
無輸出
另開終端
[root@logstash ~]# echo 22222 >> /var/tmp/b.log
原終端有輸出了
[root@logstash logstash]# logstash -f logstash.conf
Settings: Default pipeline workers: 2
Pipeline main started
{
"message" => "22222",
"@version" => "1",
"@timestamp" => "2018-12-26T08:02:48.169Z",
"path" => "/var/tmp/b.log",
"host" => "logstash"
}
[root@logstash logstash]# ls /var/lib/logstash/ 此時已經生成了指定讀取文件位置的文件，裏面有偏移量
sincedb
[root@logstash logstash]# cat /etc/logstash/logstash.conf
input{
file {
path => ["/tmp/a.log","/var/tmp/b.log"]
sincedb_path => "/var/lib/logstash/sincedb"
start_position => "beginning" 指定從開始的偏移量讀
}
}
filter{}
output{
stdout{ codec => "rubydebug" }
}
[root@logstash logstash]# logstash -f logstash.conf
Settings: Default pipeline workers: 2
Pipeline main started
這裏會從頭開始顯示2個文件裏的全部內容。
[root@logstash logstash]# cat /etc/logstash/logstash.conf
input{
file {
path => ["/tmp/a.log","/var/tmp/b.log"]
sincedb_path => "/var/lib/logstash/sincedb"
start_position => "beginning" 指定從開始的偏移量讀
type => "test log" 指定類型
}
}
filter{}
output{
stdout{ codec => "rubydebug" }
}
[root@logstash ~]# echo test >> /tmp/a.log 寫入新內容
[root@logstash logstash]# logstash -f logstash.conf
Settings: Default pipeline workers: 2
Pipeline main started
{
"message" => "test",
"@version" => "1",
"@timestamp" => "2018-12-26T08:07:12.887Z",
"path" => "/tmp/a.log",
"host" => "logstash",
"type" => "test log" 這裏有顯示類型
}
##############################################################################################
[root@logstash logstash]# cat logstash.conf
input{
file {
path => ["/tmp/a.log","/var/tmp/b.log"]
sincedb_path => "/var/lib/logstash/sincedb"
start_position => "beginning"
type => "test log"
}
tcp {
mode => "server"
host=> "0.0.0.0"
port => 8888
type => "tcp log"
}
udp {
port => 8888
type => "udp log"
}
}
filter{}
output{
stdout{ codec => "rubydebug" }
}
[root@logstash logstash]# logstash -f /etc/logstash/logstash.conf
Settings: Default pipeline workers: 2
Pipeline main started
新開一個終端
[root@logstash fd]# ss -tunlp | grep 8888
udp UNCONN 0 0 :::8888 :::* users:(("java",pid=12416,fd=37))
tcp LISTEN 0 50 :::8888 :::* users:(("java",pid=12416,fd=7))
#####################################################################################################
虛擬機kibana
[root@kibana ~]# echo 0000000 >/dev/udp/192.168.3.200/8888
######################################################################################################
虛擬機logstash
會跳出新的信息
[root@logstash logstash]# logstash -f /etc/logstash/logstash.conf
Settings: Default pipeline workers: 2
Pipeline main started
{
"message" => "0000000\n", 這是客戶端輸入的信息
"@version" => "1",
"@timestamp" => "2018-12-26T09:06:52.222Z",
"type" => "udp log", 這是udp協議鏈接的
"host" => "192.168.3.116" 這是客戶端的IP
}
#####################################################################################################
虛擬機kibana
[root@kibana ~]# echo 9999999 >/dev/tcp/192.168.3.200/8888
######################################################################################################
虛擬機logstash
會跳出新的信息
[root@logstash logstash]# logstash -f /etc/logstash/logstash.conf
... ...
{
"message" => "9999999",
"@version" => "1",
"@timestamp" => "2018-12-26T09:08:53.830Z",
"host" => "192.168.3.116",
"port" => 54206,
"type" => "tcp log"
}
######################################################################################################
虛擬機logstash
[root@logstash fd]# cd /proc/self/fd
[root@logstash fd]# ls
0 1 2 255
[root@logstash fd]# ll
總用量 0
lrwx------ 1 root root 64 12月 26 15:30 0 -> /dev/pts/1
lrwx------ 1 root root 64 12月 26 15:30 1 -> /dev/pts/1
lrwx------ 1 root root 64 12月 26 15:30 2 -> /dev/pts/1
lrwx------ 1 root root 64 12月 26 16:45 255 -> /dev/pts/1
[root@logstash fd]# echo haha >./1
haha
[root@logstash fd]# exec 8<>/dev/tcp/192.168.3.200/8888
[root@logstash fd]# ll
總用量 0
lrwx------ 1 root root 64 12月 26 15:30 0 -> /dev/pts/1
lrwx------ 1 root root 64 12月 26 15:30 1 -> /dev/pts/1
lrwx------ 1 root root 64 12月 26 15:30 2 -> /dev/pts/1
lrwx------ 1 root root 64 12月 26 16:45 255 -> /dev/pts/1
lrwx------ 1 root root 64 12月 26 17:14 8 -> socket:[31877]
[root@logstash fd]# echo 88888888888 >&8
[root@logstash fd]# ls
0 1 2 255 8
######################################################################################################
虛擬機logstash
會跳出新的信息
[root@logstash logstash]# logstash -f /etc/logstash/logstash.conf
... ...
{
"message" => "88888888888",
"@version" => "1",
"@timestamp" => "2018-12-26T09:15:14.316Z",
"host" => "192.168.3.200",
"port" => 59430,
"type" => "tcp log"
}
######################################################################################################
虛擬機logstash
[root@logstash fd]# ll
總用量 0
... ...
lrwx------ 1 root root 64 12月 26 17:14 8 -> socket:[31877] 這裏有8
[root@logstash fd]# ls
0 1 2 255 8 這裏有8
[root@logstash fd]# exec 8<&- 刪去8
[root@logstash fd]# ll
已經沒有8
[root@logstash fd]# ls
0 1 2 255 已經沒有8了
.每個進程都有本身的文件描述符集合.
.當建立進程時,一般有3個打開文件描述符(0,1,2),0表明標準輸入,1表明標準輸出,2表明標準錯誤,它們統稱爲標準IO.
子目錄/proc/self自己就是當前運行進程ID的符號連接.
用ls -ld查看/proc/self目錄的符號連接,發現每次都不同,說明咱們每次用ls命令時的進程ID都是不一樣的.
ls -ld /proc/self
lrwxrwxrwx 1 root root 64 2010-10-10 06:25 /proc/self -> 30525
咱們查看/proc/self/fd目錄下的文件描述符,以下:
ls -l /proc/self/fd
total 0
lrwx------ 1 root root 64 2010-10-10 12:16 0 -> /dev/pts/1
lrwx------ 1 root root 64 2010-10-10 12:16 1 -> /dev/pts/1
lrwx------ 1 root root 64 2010-10-10 12:16 2 -> /dev/pts/1
lr-x------ 1 root root 64 2010-10-10 12:16 3 -> /proc/30578/fd
咱們看到了3個標準的IO描述符,它們都被軟錠接到了/dev/pts/1,/dev/pts/1是咱們經過ssh打開第2個終端,若是是第1個終端,那將是/dev/pts/0.
若是咱們經過ipmi的串口登陸,這裏應該是/dev/ttySx,而若是是本地登陸那應該是/dev/ttyx,若是是單用戶登陸那將是/dev/console.
/dev/pts/x是虛擬終端
/dev/ttySx是串行控制端
/dev/ttyx是控制檯
/dev/console是單用戶控制檯
#####################################################################################################
虛擬機kibana
定義一個函數send()
[root@kibana ~]# function send() {
> exec 9<>/dev/tcp/192.168.3.200/8888
> echo #1 >&9
> exec 9<&-
> }
[root@kibana ~]# send hello
######################################################################################################
虛擬機logstash
以前一直開着的程序，會跳出新的信息
[root@logstash logstash]# logstash -f /etc/logstash/logstash.conf
Settings: Default pipeline workers: 2
Pipeline main started
跳出如下信息：
{
"message" => "\e]0;root@kibana:~\a\e]0;root@kibana:~\a",
"@version" => "1",
"@timestamp" => "2018-12-26T11:00:14.992Z",
"host" => "192.168.3.116",
"port" => 46972,
"type" => "tcp log"
}
######################################################################################################
虛擬機logstash
[root@logstash 11729]# man logger
在最後面的幾行裏找到
EXAMPLES
logger System rebooted
logger -p local0.notice -t HOSTIDM -f /dev/idmc 複製這行
logger -n loghost.example.com System rebooted
粘貼到命令行，而後修改
[root@logstash 11729]# logger -p local0.info -t testlog aaaaaaaaaaaa
等級 標題 內容
[root@logstash 11729]# tail -1 /var/log/messages
Dec 26 17:49:30 logstash testlog: aaaaaaaaaaaa
[root@logstash 11729]# vim /etc/rsyslog.conf
74 local0.info /var/log/info.log 增長這行
[root@logstash 11729]# systemctl restart rsyslog
[root@logstash 11729]# logger -p local0.info -t testlog bbbbb
[root@logstash 11729]# tail -1 /var/log/info.log
Dec 26 17:56:41 logstash testlog: bbbbb
######################################################################################################
虛擬機kibana
[root@kibana ~]# vim /etc/rsyslog.conf
74 local0.info @@192.168.3.200:514 增長74行
90 #*.* @@remote-host:514 第90行有註釋說明，把信息寫入遠程的哪一個主機的哪一個端口
[root@kibana ~]# systemctl restart rsyslog
######################################################################################################
虛擬機logstash
[root@logstash logstash]# vim logstash.conf
input{
file {
path => ["/tmp/a.log","/var/tmp/b.log"]
sincedb_path => "/var/lib/logstash/sincedb"
start_position => "beginning"
type => "test log"
}
tcp {
mode => "server"
host=> "0.0.0.0"
port => 8888
type => "tcp log"
}
udp {
port => 8888
type => "udp log"
}
syslog {
port => 514
}
}
filter{}
output{
stdout{ codec => "rubydebug" }
}
打開程序，一直開着
[root@logstash logstash]# logstash -f /etc/logstash/logstash.conf
Settings: Default pipeline workers: 2
Pipeline main started
######################################################################################################
新開終端虛擬機logstash
能夠查看到已經啓動了端口514
[root@logstash 11729]# ss -tunlp | grep 514
udp UNCONN 0 0 :::514 :::* users:(("java",pid=12577,fd=28))
tcp LISTEN 0 50 :::514 :::* users:(("java",pid=12577,fd=40))
######################################################################################################
虛擬機kibana
[root@kibana ~]# logger -p local0.info -t testlog kkkkk
######################################################################################################
虛擬機logstash
以前一直開着的程序，會跳出新的信息
[root@logstash logstash]# logstash -f /etc/logstash/logstash.conf
Settings: Default pipeline workers: 2
Pipeline main started
跳出如下信息：
{
"message" => "kkkkk\n", 內容
"@version" => "1",
"@timestamp" => "2018-12-26T10:09:43.000Z",
"host" => "192.168.3.116", 哪一個IP的主機寫的
"priority" => 134,
"timestamp" => "Dec 26 18:09:43",
"logsource" => "kibana", 主機kibana的日誌
"program" => "testlog",
"severity" => 6,
"facility" => 16,
"facility_label" => "local0",
"severity_label" => "Informational" 等級
}
######################################################################################################
虛擬機kibana
工做中的用途，如把有關登錄的日誌信息發送給遠程的某個主機
[root@logstash logstash]# vim logstash.conf
58 authpriv.* @@192.168.3.200:514 增長58行
[root@kibana ~]# systemctl restart rsyslog
[root@kibana ~]# exit
登出
Connection to 192.168.3.116 closed.
[root@room9pc01 qemu]#
######################################################################################################
虛擬機logstash
以前一直開着的程序，會跳出新的信息
[root@logstash logstash]# logstash -f /etc/logstash/logstash.conf
Settings: Default pipeline workers: 2
Pipeline main started
跳出如下信息：
{
"message" => "Unregistered Authentication Agent for unix-process:1289:8507839 (system bus name :1.65, object path /org/freedesktop/PolicyKit1/AuthenticationAgent, locale zh_CN.UTF-8) (disconnected from bus)\n",
"@version" => "1",
"@timestamp" => "2018-12-26T10:15:24.000Z",
"host" => "192.168.3.116",
"priority" => 85,
"timestamp" => "Dec 26 18:15:24",
"logsource" => "kibana",
"program" => "polkitd",
"pid" => "476",
"severity" => 5,
"facility" => 10,
"facility_label" => "security/authorization",
"severity_label" => "Notice"
}
{
"message" => "Received disconnect from 192.168.3.254 port 34262:11: disconnected by user\n",
"@version" => "1",
"@timestamp" => "2018-12-26T10:15:37.000Z",
"host" => "192.168.3.116",
"priority" => 86,
"timestamp" => "Dec 26 18:15:37",
"logsource" => "kibana",
"program" => "sshd",
"pid" => "1061",
"severity" => 6,
"facility" => 10,
"facility_label" => "security/authorization",
"severity_label" => "Informational"
}
{
"message" => "Disconnected from 192.168.3.254 port 34262\n",
"@version" => "1",
"@timestamp" => "2018-12-26T10:15:37.000Z",
"host" => "192.168.3.116",
"priority" => 86,
"timestamp" => "Dec 26 18:15:37",
"logsource" => "kibana",
"program" => "sshd",
"pid" => "1061",
"severity" => 6,
"facility" => 10,
"facility_label" => "security/authorization",
"severity_label" => "Informational"
}
{
"message" => "pam_unix(sshd:session): session closed for user root\n",
"@version" => "1",
"@timestamp" => "2018-12-26T10:15:37.000Z",
"host" => "192.168.3.116",
"priority" => 86,
"timestamp" => "Dec 26 18:15:37",
"logsource" => "kibana",
"program" => "sshd",
"pid" => "1061",
"severity" => 6,
"facility" => 10,
"facility_label" => "security/authorization",
"severity_label" => "Informational"
}
#####################################################################################
真機鏈接kibana，故意輸錯密碼
[root@room9pc01 qemu]# ssh 192.168.3.116
root@192.168.3.116's password:
Permission denied, please try again.
root@192.168.3.116's password:
######################################################################################################
虛擬機logstash
以前一直開着的程序，會跳出新的信息,
[root@logstash logstash]# logstash -f /etc/logstash/logstash.conf
Settings: Default pipeline workers: 2
Pipeline main started
跳出如下信息：
{
"message" => "pam_unix(sshd:auth): authentication failure; logname= uid=0 euid=0 tty=ssh ruser= rhost=192.168.3.254 user=root\n",
"@version" => "1",
"@timestamp" => "2018-12-26T10:17:02.000Z",
"host" => "192.168.3.116",
"priority" => 85,
"timestamp" => "Dec 26 18:17:02",
"logsource" => "kibana",
"program" => "sshd",
"pid" => "1302",
"severity" => 5,
"facility" => 10,
"facility_label" => "security/authorization",
"severity_label" => "Notice"
}
{
"message" => "pam_succeed_if(sshd:auth): requirement \"uid >= 1000\" not met by user \"root\"\n",
"@version" => "1",
"@timestamp" => "2018-12-26T10:17:02.000Z",
"host" => "192.168.3.116",
"priority" => 86,
"timestamp" => "Dec 26 18:17:02",
"logsource" => "kibana",
"program" => "sshd",
"pid" => "1302",
"severity" => 6,
"facility" => 10,
"facility_label" => "security/authorization",
"severity_label" => "Informational"
}
{
"message" => "Failed password for root from 192.168.3.254 port 35864 ssh2\n", 從主機3.254遠程鏈接kibana失敗
"@version" => "1",
"@timestamp" => "2018-12-26T10:17:04.000Z",
"host" => "192.168.3.116",
"priority" => 86,
"timestamp" => "Dec 26 18:17:04",
"logsource" => "kibana",
"program" => "sshd",
"pid" => "1302",
"severity" => 6,
"facility" => 10,
"facility_label" => "security/authorization",
"severity_label" => "Informational"
}
#####################################################################################
真機成功鏈接kibana
[root@room9pc01 qemu]# ssh 192.168.3.116
root@192.168.3.116's password:
Last failed login: Wed Dec 26 18:17:04 CST 2018 from 192.168.3.254 on ssh:notty
There was 1 failed login attempt since the last successful login.
Last login: Wed Dec 26 09:01:00 2018 from 192.168.3.254
[root@kibana ~]#
######################################################################################################
虛擬機logstash
以前一直開着的程序，會跳出新的信息
[root@logstash logstash]# logstash -f /etc/logstash/logstash.conf
Settings: Default pipeline workers: 2
Pipeline main started
跳出如下信息：
{
"message" => "Accepted password for root from 192.168.3.254 port 35868 ssh2\n", 從主機3.254成功遠程鏈接kibana
"@version" => "1",
"@timestamp" => "2018-12-26T10:19:08.000Z",
"host" => "192.168.3.116",
"priority" => 86,
"timestamp" => "Dec 26 18:19:08",
"logsource" => "kibana",
"program" => "sshd",
"pid" => "1304",
"severity" => 6,
"facility" => 10,
"facility_label" => "security/authorization",
"severity_label" => "Informational"
}
{
"message" => "pam_unix(sshd:session): session opened for user root by (uid=0)\n",
"@version" => "1",
"@timestamp" => "2018-12-26T10:19:08.000Z",
"host" => "192.168.3.116",
"priority" => 86,
"timestamp" => "Dec 26 18:19:08",
"logsource" => "kibana",
"program" => "sshd",
"pid" => "1304",
"severity" => 6,
"facility" => 10,
"facility_label" => "security/authorization",
"severity_label" => "Informational"
}
###########################################################################################
虛擬機logstash
[root@logstash ~]# curl -v www.baidu.com
顯示以下：
* About to connect() to www.baidu.com port 80 (#0)
* Trying 14.215.177.39...
* Connected to www.baidu.com (14.215.177.39) port 80 (#0)
> GET / HTTP/1.1
> User-Agent: curl/7.29.0
> Host: www.baidu.com
> Accept: */*
>
< HTTP/1.1 200 OK
< Accept-Ranges: bytes
< Cache-Control: private, no-cache, no-store, proxy-revalidate, no-transform
< Connection: Keep-Alive
< Content-Length: 2381
< Content-Type: text/html
< Date: Wed, 26 Dec 2018 11:46:32 GMT
< Etag: "588604dd-94d"
< Last-Modified: Mon, 23 Jan 2017 13:27:57 GMT
< Pragma: no-cache
< Server: bfe/1.0.8.18
< Set-Cookie: BDORZ=27315; max-age=86400; domain=.baidu.com; path=/
<
<!DOCTYPE html>
... ...
[root@logstash ~]# vim baidu.sh
#!/bin/bash
exec 5<>/dev/tcp/www.baidu.com/80
echo -ne "GET / HTTP/1.1\r\n" >&5
echo -ne "User-Agent: curl/7.29.0\r\n" >&5
echo -ne "Host: www.baidu.com\r\n" >&5
echo -ne "\r\n" >&5
cat <&5
exec 5<&-
[root@logstash ~]# chmod +x baidu.sh
[root@logstash ~]# ./baidu.sh
顯示以下：
HTTP/1.1 200 OK
Accept-Ranges: bytes
Cache-Control: private, no-cache, no-store, proxy-revalidate, no-transform
Connection: Keep-Alive
Content-Length: 2381
Content-Type: text/html
Date: Wed, 26 Dec 2018 11:51:50 GMT
Etag: "588604dd-94d"
Last-Modified: Mon, 23 Jan 2017 13:27:57 GMT
Pragma: no-cache
Server: bfe/1.0.8.18
Set-Cookie: BDORZ=27315; max-age=86400; domain=.baidu.com; path=/
<!DOCTYPE html>
[root@logstash fd]# cd /proc/self/fd
[root@logstash fd]# exec 6<>/etc/passwd
[root@logstash fd]# ll
總用量 0
... ...
lrwx------ 1 root root 64 12月 26 19:11 6 -> /etc/passwd 這裏的6是/etc/passwd的快捷方式
[root@logstash fd]# cat /etc/passwd
[root@logstash fd]# cat /etc/passwd | wc -l
20
[root@logstash fd]# cat ./6 查看當前目錄下的6，就至關於查看/etc/passwd
[root@logstash fd]# cat ./6 | wc -l
20
######################################################################################################
虛擬機web
[root@web httpd]# cat /var/log/httpd/access_log
192.168.3.254 - - [27/Dec/2018:10:51:23 +0800] "GET / HTTP/1.1" 403 4897 "-" "curl/7.29.0"
[root@web httpd]# vim /etc/httpd/conf/httpd.conf
196 LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" combined
197 LogFormat "%h %l %u %t \"%r\" %>s %b" common
198
199 <IfModule logio_module>
200 # You need to enable mod_logio.c to use %I and %O
201 LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\" %I %O" combinedio
202 </IfModule>
... ...
217 CustomLog "logs/access_log" combined
apache官方文檔
http://httpd.apache.org/
中文版的翻譯能夠百度一下：
金步國
http://www.jinbuguo.com/
http://www.jinbuguo.com/apache/menu22/mod/mod_log_config.html#logformat
[root@logstash ~]# cd /opt/logstash/vendor/bundle/jruby/1.9/gems/logstash-patterns-core-2.0.5/patterns/
[root@logstash patterns]# ls
aws bro firewalls haproxy junos mcollective mongodb postgresql redis
bacula exim grok-patterns java linux-syslog mcollective-patterns nagios rails ruby
[root@logstash patterns]# vim grok-patterns
這是創做者提早寫好的正則表達式變量，咱們能夠調用
[root@logstash patterns]# cat grok-patterns
USERNAME [a-zA-Z0-9._-]+
USER %{USERNAME}
EMAILLOCALPART [a-zA-Z][a-zA-Z0-9_.+-=:]+
EMAILADDRESS %{EMAILLOCALPART}@%{HOSTNAME}
HTTPDUSER %{EMAILADDRESS}|%{USER}
INT (?:[+-]?(?:[0-9]+))
BASE10NUM (?<![0-9.+-])(?>[+-]?(?:(?:[0-9]+(?:\.[0-9]+)?)|(?:\.[0-9]+)))
NUMBER (?:%{BASE10NUM})
BASE16NUM (?<![0-9A-Fa-f])(?:[+-]?(?:0x)?(?:[0-9A-Fa-f]+))
BASE16FLOAT \b(?<![0-9A-Fa-f.])(?:[+-]?(?:0x)?(?:(?:[0-9A-Fa-f]+(?:\.[0-9A-Fa-f]*)?)|(?:\.[0-9A-Fa-f]+)))\b
POSINT \b(?:[1-9][0-9]*)\b
NONNEGINT \b(?:[0-9]+)\b
WORD \b\w+\b
NOTSPACE \S+
SPACE \s*
DATA .*?
GREEDYDATA .*
QUOTEDSTRING (?>(?<!\\)(?>"(?>\\.|[^\\"]+)+"|""|(?>'(?>\\.|[^\\']+)+')|''|(?>`(?>\\.|[^\\`]+)+`)|``))
UUID [A-Fa-f0-9]{8}-(?:[A-Fa-f0-9]{4}-){3}[A-Fa-f0-9]{12}
# Networking
MAC (?:%{CISCOMAC}|%{WINDOWSMAC}|%{COMMONMAC})
CISCOMAC (?:(?:[A-Fa-f0-9]{4}\.){2}[A-Fa-f0-9]{4})
WINDOWSMAC (?:(?:[A-Fa-f0-9]{2}-){5}[A-Fa-f0-9]{2})
COMMONMAC (?:(?:[A-Fa-f0-9]{2}:){5}[A-Fa-f0-9]{2})
IPV6 ((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:)))(%.+)?
IPV4 (?<![0-9])(?:(?:[0-1]?[0-9]{1,2}|2[0-4][0-9]|25[0-5])[.](?:[0-1]?[0-9]{1,2}|2[0-4][0-9]|25[0-5])[.](?:[0-1]?[0-9]{1,2}|2[0-4][0-9]|25[0-5])[.](?:[0-1]?[0-9]{1,2}|2[0-4][0-9]|25[0-5]))(?![0-9])
IP (?:%{IPV6}|%{IPV4})
HOSTNAME \b(?:[0-9A-Za-z][0-9A-Za-z-]{0,62})(?:\.(?:[0-9A-Za-z][0-9A-Za-z-]{0,62}))*(\.?|\b)
IPORHOST (?:%{IP}|%{HOSTNAME})
HOSTPORT %{IPORHOST}:%{POSINT}
# paths
PATH (?:%{UNIXPATH}|%{WINPATH})
UNIXPATH (/([\w_%!$@:.,~-]+|\\.)*)+
TTY (?:/dev/(pts|tty([pq])?)(\w+)?/?(?:[0-9]+))
WINPATH (?>[A-Za-z]+:|\\)(?:\\[^\\?*]*)+
URIPROTO [A-Za-z]+(\+[A-Za-z+]+)?
URIHOST %{IPORHOST}(?::%{POSINT:port})?
# uripath comes loosely from RFC1738, but mostly from what Firefox
# doesn't turn into %XX
URIPATH (?:/[A-Za-z0-9$.+!*'(){},~:;=@#%&_\-]*)+
#URIPARAM \?(?:[A-Za-z0-9]+(?:=(?:[^&]*))?(?:&(?:[A-Za-z0-9]+(?:=(?:[^&]*))?)?)*)?
URIPARAM \?[A-Za-z0-9$.+!*'|(){},~@#%&/=:;_?\-\[\]<>]*
URIPATHPARAM %{URIPATH}(?:%{URIPARAM})?
URI %{URIPROTO}://(?:%{USER}(?::[^@]*)?@)?(?:%{URIHOST})?(?:%{URIPATHPARAM})?
# Months: January, Feb, 3, 03, 12, December
MONTH \b(?:Jan(?:uary|uar)?|Feb(?:ruary|ruar)?|M(?:a|ä)?r(?:ch|z)?|Apr(?:il)?|Ma(?:y|i)?|Jun(?:e|i)?|Jul(?:y)?|Aug(?:ust)?|Sep(?:tember)?|O(?:c|k)?t(?:ober)?|Nov(?:ember)?|De(?:c|z)(?:ember)?)\b
MONTHNUM (?:0?[1-9]|1[0-2])
MONTHNUM2 (?:0[1-9]|1[0-2])
MONTHDAY (?:(?:0[1-9])|(?:[12][0-9])|(?:3[01])|[1-9])
# Days: Monday, Tue, Thu, etc...
DAY (?:Mon(?:day)?|Tue(?:sday)?|Wed(?:nesday)?|Thu(?:rsday)?|Fri(?:day)?|Sat(?:urday)?|Sun(?:day)?)
# Years?
YEAR (?>\d\d){1,2}
HOUR (?:2[0123]|[01]?[0-9])
MINUTE (?:[0-5][0-9])
# '60' is a leap second in most time standards and thus is valid.
SECOND (?:(?:[0-5]?[0-9]|60)(?:[:.,][0-9]+)?)
TIME (?!<[0-9])%{HOUR}:%{MINUTE}(?::%{SECOND})(?![0-9])
# datestamp is YYYY/MM/DD-HH:MM:SS.UUUU (or something like it)
DATE_US %{MONTHNUM}[/-]%{MONTHDAY}[/-]%{YEAR}
DATE_EU %{MONTHDAY}[./-]%{MONTHNUM}[./-]%{YEAR}
ISO8601_TIMEZONE (?:Z|[+-]%{HOUR}(?::?%{MINUTE}))
ISO8601_SECOND (?:%{SECOND}|60)
TIMESTAMP_ISO8601 %{YEAR}-%{MONTHNUM}-%{MONTHDAY}[T ]%{HOUR}:?%{MINUTE}(?::?%{SECOND})?%{ISO8601_TIMEZONE}?
DATE %{DATE_US}|%{DATE_EU}
DATESTAMP %{DATE}[- ]%{TIME}
TZ (?:[PMCE][SD]T|UTC)
DATESTAMP_RFC822 %{DAY} %{MONTH} %{MONTHDAY} %{YEAR} %{TIME} %{TZ}
DATESTAMP_RFC2822 %{DAY}, %{MONTHDAY} %{MONTH} %{YEAR} %{TIME} %{ISO8601_TIMEZONE}
DATESTAMP_OTHER %{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{TZ} %{YEAR}
DATESTAMP_EVENTLOG %{YEAR}%{MONTHNUM2}%{MONTHDAY}%{HOUR}%{MINUTE}%{SECOND}
HTTPDERROR_DATE %{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{YEAR}
# Syslog Dates: Month Day HH:MM:SS
SYSLOGTIMESTAMP %{MONTH} +%{MONTHDAY} %{TIME}
PROG [\x21-\x5a\x5c\x5e-\x7e]+
SYSLOGPROG %{PROG:program}(?:\[%{POSINT:pid}\])?
SYSLOGHOST %{IPORHOST}
SYSLOGFACILITY <%{NONNEGINT:facility}.%{NONNEGINT:priority}>
HTTPDATE %{MONTHDAY}/%{MONTH}/%{YEAR}:%{TIME} %{INT}
# Shortcuts
QS %{QUOTEDSTRING}
# Log formats
SYSLOGBASE %{SYSLOGTIMESTAMP:timestamp} (?:%{SYSLOGFACILITY} )?%{SYSLOGHOST:logsource} %{SYSLOGPROG}:
COMMONAPACHELOG %{IPORHOST:clientip} %{HTTPDUSER:ident} %{USER:auth} \[%{HTTPDATE:timestamp}\] "(?:%{WORD:verb} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest})" %{NUMBER:response} (?:%{NUMBER:bytes}|-)
COMBINEDAPACHELOG %{COMMONAPACHELOG} %{QS:referrer} %{QS:agent}
HTTPD20_ERRORLOG \[%{HTTPDERROR_DATE:timestamp}\] \[%{LOGLEVEL:loglevel}\] (?:\[client %{IPORHOST:clientip}\] ){0,1}%{GREEDYDATA:errormsg}
HTTPD24_ERRORLOG \[%{HTTPDERROR_DATE:timestamp}\] \[%{WORD:module}:%{LOGLEVEL:loglevel}\] \[pid %{POSINT:pid}:tid %{NUMBER:tid}\]( \(%{POSINT:proxy_errorcode}\)%{DATA:proxy_errormessage}:)?( \[client %{IPORHOST:client}:%{POSINT:clientport}\])? %{DATA:errorcode}: %{GREEDYDATA:message}
HTTPD_ERRORLOG %{HTTPD20_ERRORLOG}|%{HTTPD24_ERRORLOG}
# Log Levels
LOGLEVEL ([Aa]lert|ALERT|[Tt]race|TRACE|[Dd]ebug|DEBUG|[Nn]otice|NOTICE|[Ii]nfo|INFO|[Ww]arn?(?:ing)?|WARN?(?:ING)?|[Ee]rr?(?:or)?|ERR?(?:OR)?|[Cc]rit?(?:ical)?|CRIT?(?:ICAL)?|[Ff]atal|FATAL|[Ss]evere|SEVERE|EMERG(?:ENCY)?|[Ee]merg(?:ency)?)
######################################################################################################
虛擬機web
[root@room9pc01 qemu]# scp '/root/桌面/filebeat-1.2.3-x86_64.rpm' 192.168.3.100:/root/
[root@web ~]# rpm -ivh filebeat-1.2.3-x86_64.rpm
[root@web ~]# vim /etc/filebeat/filebeat.yml
15 - /var/log/httpd/access_log 改
72 document_type: apachelog 打開註釋並修改
278 logstash: 打開註釋
279 # The Logstash hosts
280 hosts: ["192.168.3.200:5044"] 打開註釋並修改
183 # elasticsearch: 註釋
188 # hosts: ["localhost:9200"] 註釋
[root@web ~]# grep -Pv "^\s*(#|$)" /etc/filebeat/filebeat.yml
filebeat:
prospectors:
-
paths:
- /var/log/httpd/access_log
input_type: log
document_type: apachelog
registry_file: /var/lib/filebeat/registry
output:
logstash:
hosts: ["192.168.3.200:5044"]
shipper:
logging:
files:
rotateeverybytes: 10485760 # = 10MB
[root@web ~]# systemctl restart filebeat
把本身的主頁設爲百度
[root@web ~]# curl www.baidu.com -o /var/www/html/index.html
% Total % Received % Xferd Average Speed Time Time Time Current
Dload Upload Total Spent Left Speed
100 2381 100 2381 0 0 104k 0 --:--:-- --:--:-- --:--:-- 110k
######################################################################################################
虛擬機
[root@logstash ~]# cd /etc/logstash/
[root@logstash logstash]# vim logstash.conf
input{
file {
path => ["/tmp/a.log"]
sincedb_path => "/var/lib/logstash/sincedb"
start_position => "beginning"
type => "test log"
}
tcp {
mode => "server"
host=> "0.0.0.0"
port => 8888
type => "tcp log"
}
udp {
port => 8888
type => "udp log"
}
syslog {
port => 514
}
beats {
port => 5044
}
}
filter{
if [type] == "apachelog" {
grok {
match => { message => "%{COMBINEDAPACHELOG}" }
}}
}
output{
stdout{ codec => "rubydebug" }
if [type] == "apachelog" {
elasticsearch {
hosts => ["es1:9200", "es2:9200", "es3:9200"]
index => "weblog-%{+YYYY.MM.dd}"
}}
}
複製web的apache訪問日誌粘貼到/tmp/a.log
[root@logstash patterns]# cat /tmp/a.log
192.168.3.254 - - [27/Dec/2018:10:51:23 +0800] "GET / HTTP/1.1" 403 4897 "-" "curl/7.29.0"
[root@logstash patterns]# curl -XDELETE http://192.168.3.111:9200/*
{"acknowledged":true}[root@logstash patterns]#
[root@logstash logstash]# logstash -f logstash.conf
Settings: Default pipeline workers: 2
Pipeline main started
{
"message" => "192.168.3.254 - - [27/Dec/2018:10:51:23 +0800] \"GET / HTTP/1.1\" 403 4897 \"-\" \"curl/7.29.0\"",
"@version" => "1",
"@timestamp" => "2018-12-27T03:26:18.499Z",
"type" => "apachelog",
"count" => 1,
"input_type" => "log",
"fields" => nil,
"beat" => {
"hostname" => "web",
"name" => "web"
},
"source" => "/var/log/httpd/access_log",
"offset" => 0,
"host" => "web",
"tags" => [
[0] "beats_input_codec_plain_applied"
],
"clientip" => "192.168.3.254",
"ident" => "-",
"auth" => "-",
"timestamp" => "27/Dec/2018:10:51:23 +0800",
"verb" => "GET",
"request" => "/",
"httpversion" => "1.1",
"response" => "403",
"bytes" => "4897",
"referrer" => "\"-\"",
"agent" => "\"curl/7.29.0\""
}
{
"message" => "192.168.3.254 - - [27/Dec/2018:10:51:30 +0800] \"GET / HTTP/1.1\" 403 4897 \"-\" \"curl/7.29.0\"",
"@version" => "1",
"@timestamp" => "2018-12-27T03:26:18.499Z",
"beat" => {
"hostname" => "web",
"name" => "web"
},
"offset" => 91,
"type" => "apachelog",
"fields" => nil,
"source" => "/var/log/httpd/access_log",
"input_type" => "log",
"count" => 1,
"host" => "web",
"tags" => [
[0] "beats_input_codec_plain_applied"
],
"clientip" => "192.168.3.254",
"ident" => "-",
"auth" => "-",
"timestamp" => "27/Dec/2018:10:51:30 +0800",
"verb" => "GET",
"request" => "/",
"httpversion" => "1.1",
"response" => "403",
"bytes" => "4897",
"referrer" => "\"-\"",
"agent" => "\"curl/7.29.0\""
}
真機訪問
http://192.168.3.115:9200/_plugin/head/
會發現跳出來新的信息
還能夠訪問不少次
for i in {1..10} ;do curl 192.168.3.100 ;done
######################################################################################################
虛擬機logstash
man幫助查用tcp/udp協議，傳內容給遠程的某個主機的某個端口,寫的格式模版
[root@logstash ~]# man bash
... ...
REDIRECTION
... ...
Bash handles several filenames specially when they are used in redirections, as described in the following table:
/dev/fd/fd
If fd is a valid integer, file descriptor fd is duplicated.
/dev/stdin
File descriptor 0 is duplicated.
/dev/stdout
File descriptor 1 is duplicated.
/dev/stderr
File descriptor 2 is duplicated.
/dev/tcp/host/port 這是用tcp協議遠程傳內容給遠程的某個主機的某個端口,寫的格式模版
If host is a valid hostname or Internet address, and port is an integer port number or service name, bash attempts to open a
TCP connection to the corresponding socket.
/dev/udp/host/port 這是用udp協議遠程傳內容給遠程的某個主機的某個端口,寫的格式模版
If host is a valid hostname or Internet address, and port is an integer port number or service name, bash attempts to open a
UDP connection to the corresponding socket.
... ...
######################################################################################################
虛擬機logstash
[root@logstash ~]# echo $$
11729
[root@logstash ~]# pstree -p
systemd(1)─┬─agetty(496)
... ...
│ └─sshd(13431)───bash(13433)
... ...
另開一個新終端，進入python界面
[root@logstash ~]# python
Python 2.7.5 (default, Aug 4 2017, 00:39:18)
[GCC 4.8.5 20150623 (Red Hat 4.8.5-16)] on linux2
Type "help", "copyright", "credits" or "license" for more information.
>>>
原終端
[root@logstash ~]# pstree -p
systemd(1)─┬─agetty(496)
... ...
│ └─sshd(13431)───bash(13433)───python(13472) 多了一個,python，它的pid號是13472
... ...
[root@logstash ~]# cd /proc/13472/fd
[root@logstash fd]# ll 如今只有0 1 2這三個快捷方式
總用量 0
lrwx------ 1 root root 64 12月 27 13:14 0 -> /dev/pts/2
lrwx------ 1 root root 64 12月 27 13:14 1 -> /dev/pts/2
lrwx------ 1 root root 64 12月 27 13:14 2 -> /dev/pts/2
寫入一個新文件
[root@logstash fd]# vim /root/a.txt
aaa
aaaa
aaaaa
另外一個新終端,在python界面
輸入如下內容
>>> f =open("/root/a.txt")
>>> type(f)
<type 'file'>
>>> f.fileno()
3
原終端
[root@logstash fd]# ll 多了 3 這個快捷方式
總用量 0
lrwx------ 1 root root 64 12月 27 13:14 0 -> /dev/pts/2
lrwx------ 1 root root 64 12月 27 13:14 1 -> /dev/pts/2
lrwx------ 1 root root 64 12月 27 13:14 2 -> /dev/pts/2
lr-x------ 1 root root 64 12月 27 13:20 3 -> /root/a.txt 多了3這個快捷方式，它指向的源文件是/root/a.txt
[root@logstash fd]# cat ./3 讀當前目錄下的3，就至關於讀源文件/root/a.txt
aaa
aaaa
aaaaa
[root@logstash fd]# cat /root/a.txt 能夠發現和讀3這個快捷方式的內容同樣
aaa
aaaa
aaaaa
另外一個新終端,在python界面輸入如下內容>>> import os 導入os模塊>>> os.read(3,99999) 讀取os模塊，會發現內容是同樣的'aaa\naaaa\naaaaa\n'>>> os.read(3,99999)''>>> f.seek(0,0)>>> abc = os.read(3,99999) 設置abc這個變量的值>>> print(abc) 讀abc這個變量，就至關於讀源文件/root/a.txtaaaaaaaaaaaa
-
每一个你不满意的现在，都有一个你没有努力的曾经。