logstas-經常使用插件

date插件

filter {
        date {
            match => ["create_at", "yyyy-MM-dd HH:mm:ss,SSS", "UNIX"]
            target => "@timestamp"
            locale => "cn"
        }
}

    filter{
        date{
                match => ["fieldName", "yyyyMMdd","yyyy-MM-dd"]
                target => "fieldName1"
                timezone => "Asia/Shanghai"
        }
}

match
第一個參數是字段名。
第二個參數是格式化模式
第三個參數是要轉換的時間格式，以下表

ISO8601 - 應解析任何有效的ISO8601時間戳，如2011-04-19T03:44:01.103Z
UNIX - 將解析float或int值，表示自1346149001.132以及1326149001.132以來的秒數（以秒爲單位）
UNIX_MS - 將分析int值表示unix時間（以毫秒爲單位），如1366125117000
TAI64N - 將解析tai64n時間值

#輸入
input {
    file {
        path => ["文件路徑"]
        #自定義類型
        type => "自定義"
        start_position => "beginning"
    }
}

#過濾器
filter{
    #去除換行符，只對string類型有效
    #去除反斜槓和單引號
    mutate{
    gsub => [ "message", "\r", "" ] ,
    gsub => ["message", "[\\]", "/", "message","'", "" ]
}

#逗號分割
mutate {  
    split => ["message",","]     
}

#分割後，字段命名與賦值
mutate{
    add_field =>   {
        "id" => "%{[message][0]}"
        "mydate" => "%{[message][1]}"
        "user" => "%{[message][2]}"
        "pc" => "%{[message][3]}"
        "to_user" => "%{[message][4]}"
        "cc" => "%{[message][5]}"
        "bcc" => "%{[message][6]}"
        "from_user" => "%{[message][7]}"
        "size" => "%{[message][8]}"
        "attachments" => "%{[message][9]}"
        "content" => "%{[message][10]}"
    } 
}

#字段裏的日期識別，以及時區轉換，生成date
date {
    match => [ "mydate", "MM/dd/yyyy HH:mm:ss" ]
    target => "date"
    locale => "en"
    timezone => "+00:00"  
}

#刪除無用字段
mutate {  
    remove_field => "message"    
    remove_field => "mydate"    
    remove_field => "@version"    
    remove_field => "host"    
    remove_field => "path"    
}
#將兩個字段轉換爲整型
mutate{
convert => { "size" => "integer" }
convert => { "attachments" => "integer" }
}
}

#輸出，輸出目標爲es
output {
    #stdout { codec => rubydebug }
    elasticsearch {
        #目標主機
        host => ["目標主機1","目標主機2"]
        #協議類型
        protocol => "http"
        #索引名
        index =>"自定義"
    }  
}

mongodb

./logstash-plugin install logstash-output-mongodb

output {
    mongodb {
                        collection => "base"
                        database => "fragment"
                        uri => "mongodb://192.168.199.7:27017"
                 }
}