Zabbix4.0歷史數據的持久化

時間 2019-11-07

原文原文鏈接

（一）背景介紹
zabbix是一個大型的分佈式的監控系統，監控的範圍比較廣，是目前比較流行的監控系統，可是因爲自身的緣由，歷史數據不能持久保存，若是數據庫的數據大於100G左右查詢或其餘的速度會很是的慢，會觸發不少問題，通常的zabbix歷史數據會不超過一個月（按實際得到的數據比例計算），咱們通常保存七天。通常爲了業務的須要，每每會須要很長的歷史數據來進行查看和排查問題，這就須要使zabbix的歷史數據進行長久保存（不能存數據庫，而能夠存ES存儲）。php

（二）環境
zabbix：zabbix4.0.1（安裝部署省略）
ES：5.5.2 （安裝部署省略）前端

（三）具體的配置mysql

3.一、首先修改zabbix_server.conf文件，啓用歷史數據的配置，具體以下：web

$grep '^[a-Z]' /etc/zabbix/zabbix_server.conf 
LogFile=/tmp/zabbix_server.log
LogFileSize=0
PidFile=/var/run/zabbix/zabbix_server.pid
SocketDir=/var/run/zabbix
DBHost=localhost
DBName=zabbix
DBUser=zabbix
DBPassword=zabbix
HistoryStorageURL=http://X.X.X.X:9200
HistoryStorageTypes=uint,db1,str,log,text
HistoryStorageDateIndex=1
SNMPTrapperFile=/var/log/snmptrap/snmptrap.log
Timeout=4
AlertScriptsPath=/usr/lib/zabbix/alertscripts
ExternalScripts=/usr

########################下邊是具體的解釋####################################sql

### Option: HistoryStorageURL
#       History storage HTTP[S] URL.
#
# Mandatory: no
# Default:
# HistoryStorageURL=
HistoryStorageURL=http://X.X.X.X:9200

### Option: HistoryStorageTypes
#       Comma separated list of value types to be sent to the history storage.
#
# Mandatory: no
# Default:
# HistoryStorageTypes=uint,db1,str,log,text
HistoryStorageTypes=uint,db1,str,log,text

### Option: HistoryStorageDateIndex
# Enable preprocessing of history values in history storage to store values in different indices based on date.
#       0 - disable
#       1 - enable
#
# Mandatory: no
# Default:
HistoryStorageDateIndex=1

備註：下邊是ES所支持存儲的數據類型
數據庫

3.二、修改zabbix前端的配置文件,添加global $DB,$HISTORY;json

$vim /etc/zabbix/web/zabbix.conf.php 

<?php
// Zabbix GUI configuration file.
global $DB,$HISTORY;
#global $DB;

$DB['TYPE']     = 'MYSQL';
$DB['SERVER']   = 'localhost';
$DB['PORT']     = '0';
$DB['DATABASE'] = 'zabbix';
$DB['USER']     = 'zabbix';
$DB['PASSWORD'] = 'zabbix';

// Schema name. Used for IBM DB2 and PostgreSQL.
$DB['SCHEMA'] = '';

$ZBX_SERVER      = 'X.X.X.X';
$ZBX_SERVER_PORT = '10051';
$ZBX_SERVER_NAME = 'zabbix';

$IMAGE_FORMAT_DEFAULT = IMAGE_FORMAT_PNG;

// Elasticsearch url (can be string if same url is used for all types).
$HISTORY['url']   = 'http://X.X.X.X:9200';
// Value types stored in Elasticsearch.
$HISTORY['types'] = ['uint', 'text', 'log', 'str', 'db1'];

//--------------------------------------------------------------------------------------------------------------------------
//若是不一樣類型使用不一樣的 ES 集羣，能夠按以下進行配置
#$HISTORY['url']   = [
#    'uint' => 'http://X.X.X.X:9200 ',
#    'text' => 'http://X.X.X.X:9200 '
#];
#$HISTORY['types'] = ['uint', 'text'];

注意的要點：
一、HISTORY['url']和HISTORY['types']被更新。
二、定義zabbix GUI全局數據源路徑文件 global $DB, $HISTORY;(告訴zabbix先從數據庫裏讀取，沒有的話取HISTORY讀取)vim

3.三、在ES上依次建立五個索引、五個模板、五個pipline.app

##########一、建立索引必須建立##############
# uint mapping

curl -X PUT \
 http://10.114.23.118:9200/uint \
 -H 'content-type:application/json' \
 -d '

{
   "settings" : {
      "index" : {
         "number_of_replicas" : 1,
         "number_of_shards" : 5
      }
   },
   "mappings" : {
      "values" : {
         "properties" : {
            "itemid" : {
               "type" : "long"
            },
            "clock" : {
               "format" : "epoch_second",
               "type" : "date"
            },
            "value" : {
               "type" : "long"
            }
         }
      }
   }
}
'

# dbl mapping
curl -X PUT \
 http://10.114.23.118:9200/dbl \
 -H 'content-type:application/json' \
 -d '
{
   "settings" : {
      "index" : {
         "number_of_replicas" : 1,
         "number_of_shards" : 5
      }
   },
   "mappings" : {
      "values" : {
         "properties" : {
            "itemid" : {
               "type" : "long"
            },
            "clock" : {
               "format" : "epoch_second",
               "type" : "date"
            },
            "value" : {
               "type" : "double"
            }
         }
      }
   }
}
'

# str mapping
curl -X PUT \
 http://10.114.23.118:9200/str \
 -H 'content-type:application/json' \
 -d '
{
   "settings" : {
      "index" : {
         "number_of_replicas" : 1,
         "number_of_shards" : 5
      }
   },
   "mappings" : {
      "values" : {
         "properties" : {
            "itemid" : {
               "type" : "long"
            },
            "clock" : {
               "format" : "epoch_second",
               "type" : "date"
            },
            "value" : {
               "fields" : {
                  "analyzed" : {
                     "index" : true,
                     "type" : "text",
                     "analyzer" : "standard"
                  }
               },
               "index" : false,
               "type" : "text"
            }
         }
      }
   }
}
'

# text mapping

curl -X PUT \
 http://10.114.23.118:9200/text \
 -H 'content-type:application/json' \
 -d '
{
   "settings" : {
      "index" : {
         "number_of_replicas" : 1,
         "number_of_shards" : 5
      }
   },
   "mappings" : {
      "values" : {
         "properties" : {
            "itemid" : {
               "type" : "long"
            },
            "clock" : {
               "format" : "epoch_second",
               "type" : "date"
            },
            "value" : {
               "fields" : {
                  "analyzed" : {
                     "index" : true,
                     "type" : "text",
                     "analyzer" : "standard"
                  }
               },
               "index" : false,
               "type" : "text"
            }
         }
      }
   }
}
'

# log mapping
curl -X PUT \
 http://10.114.23.118:9200/log \
 -H 'content-type:application/json' \
 -d '
{
   "settings" : {
      "index" : {
         "number_of_replicas" : 1,
         "number_of_shards" : 5
      }
   },
   "mappings" : {
      "values" : {
         "properties" : {
            "itemid" : {
               "type" : "long"
            },
            "clock" : {
               "format" : "epoch_second",
               "type" : "date"
            },
            "value" : {
               "fields" : {
                  "analyzed" : {
                     "index" : true,
                     "type" : "text",
                     "analyzer" : "standard"
                  }
               },
               "index" : false,
               "type" : "text"
            }
         }
      }
   }
}
'

###########二、建立索引模板################

curl -X PUT \
 http://10.114.23.118:9200/_template/uint_template \
 -H 'content-type:application/json' \
 -d '{
   "template": "uint*",
   "index_patterns": ["uint*"],
   "settings" : {
      "index" : {
         "number_of_replicas" : 1,
         "number_of_shards" : 5
      }
   },
   "mappings" : {
      "values" : {
         "properties" : {
            "itemid" : {
               "type" : "long"
            },
            "clock" : {
               "format" : "epoch_second",
               "type" : "date"
            },
            "value" : {
               "type" : "long"
            }
         }
      }
   }
}'

curl -X PUT \
 http://10.114.23.118:9200/_template/text_template \
 -H 'content-type:application/json' \
 -d '{
   "template": "text*",
   "index_patterns": ["text*"],
   "settings" : {
      "index" : {
         "number_of_replicas" : 1,
         "number_of_shards" : 5
      }
   },
   "mappings" : {
      "values" : {
         "properties" : {
            "itemid" : {
               "type" : "long"
            },
            "clock" : {
               "format" : "epoch_second",
               "type" : "date"
            },
            "value" : {
               "fields" : {
                  "analyzed" : {
                     "index" : true,
                     "type" : "text",
                     "analyzer" : "standard"
                  }
               },
               "index" : false,
               "type" : "text"
            }
         }
      }
   }
}'

curl -X PUT \
 http://10.114.23.118:9200/_template/str_template \
 -H 'content-type:application/json' \
 -d '{
   "template": "str*",
   "index_patterns": ["str*"],
   "settings" : {
      "index" : {
         "number_of_replicas" : 1,
         "number_of_shards" : 5
      }
   },
   "mappings" : {
      "values" : {
         "properties" : {
            "itemid" : {
               "type" : "long"
            },
            "clock" : {
               "format" : "epoch_second",
               "type" : "date"
            },
            "value" : {
               "type" : "long"
            }
         }
      }
   }
}'

curl -X PUT \
 http://10.114.23.118:9200/_template/log_template \
 -H 'content-type:application/json' \
 -d '{
   "template": "log*",
   "index_patterns": ["log*"],
   "settings" : {
      "index" : {
         "number_of_replicas" : 1,
         "number_of_shards" : 5
      }
   },
   "mappings" : {
      "values" : {
         "properties" : {
            "itemid" : {
               "type" : "long"
            },
            "clock" : {
               "format" : "epoch_second",
               "type" : "date"
            },
            "value" : {
               "fields" : {
                  "analyzed" : {
                     "index" : true,
                     "type" : "text",
                     "analyzer" : "standard"
                  }
               },
               "index" : false,
               "type" : "text"
            }
         }
      }
   }
}'

curl -X PUT \
 http://10.114.23.118:9200/_template/dbl_template \
 -H 'content-type:application/json' \
 -d '{
   "template": "dbl*",
   "index_patterns": ["dbl*"],
   "settings" : {
      "index" : {
         "number_of_replicas" : 1,
         "number_of_shards" : 5
      }
   },
   "mappings" : {
      "values" : {
         "properties" : {
            "itemid" : {
               "type" : "long"
            },
            "clock" : {
               "format" : "epoch_second",
               "type" : "date"
            },
            "value" : {
               "fields" : {
                  "analyzed" : {
                     "index" : true,
                     "type" : "text",
                     "analyzer" : "standard"
                  }
               },
               "index" : false,
               "type" : "text"
            }
         }
      }
   }
}'

##########建立pipeline。 在將數據放入索引以前，pipeline能對數據進行多種預處理操做##############

curl -X PUT \
 http://10.114.23.118:9200/_ingest/pipeline/uint-pipeline \
 -H 'content-type:application/json' \
 -d '{
  "description": "daily uint index naming",
  "processors": [
    {
      "date_index_name": {
        "field": "clock",
        "date_formats": ["UNIX"],
        "index_name_prefix": "uint-",
        "date_rounding": "d"
      }
    }
  ]
}'

curl -X PUT \
 http://10.114.23.118:9200/_ingest/pipeline/dbl-pipeline \
 -H 'content-type:application/json' \
 -d '{
  "description": "daily dbl index naming",
  "processors": [
    {
      "date_index_name": {
        "field": "clock",
        "date_formats": ["UNIX"],
        "index_name_prefix": "dbl-",
        "date_rounding": "d"
      }
    }
  ]
}'

curl -X PUT \
 http://10.114.23.118:9200/_ingest/pipeline/log-pipeline \
 -H 'content-type:application/json' \
 -d '{
  "description": "daily log index naming",
  "processors": [
    {
      "date_index_name": {
        "field": "clock",
        "date_formats": ["UNIX"],
        "index_name_prefix": "log-",
        "date_rounding": "d"
      }
    }
  ]
}'

curl -X PUT \
 http://10.114.23.118:9200/_ingest/pipeline/text-pipeline \
 -H 'content-type:application/json' \
 -d '{
  "description": "daily text index naming",
  "processors": [
    {
      "date_index_name": {
        "field": "clock",
        "date_formats": ["UNIX"],
        "index_name_prefix": "text-",
        "date_rounding": "d"
      }
    }
  ]
}'

curl -X PUT \
 http://10.114.23.118:9200/_ingest/pipeline/str-pipeline \
 -H 'content-type:application/json' \
 -d '{
  "description": "daily str index naming",
  "processors": [
    {
      "date_index_name": {
        "field": "clock",
        "date_formats": ["UNIX"],
        "index_name_prefix": "str-",
        "date_rounding": "d"
      }
    }
  ]
}'

3.四、在ES上查看是否生成：curl

$curl http://X.X.X.X:9200/_cat/indices?v
health status index uuid pri rep docs.count docs.deleted store.size pri.store.size
green open str ZTkdqhpFR9ijzTfKvtsMrQ 5 1 0 0 2.2kb 1.1kb
green open text TjY6OEGySJ2AGc1rpfOEhA 5 1 0 0 2.2kb 1.1kb
green open log qG-R2UhiQhypP9stSSodkw 5 1 0 0 2.2kb 1.1kb
green open db1 av_V5XbiSDyDdZXW0rO6aw 5 1 0 0 2.2kb 1.1kb
green open .kibana JZBiyypQSRuMxRnHxOLP1Q 1 1 1 0 8kb 4kb
green open unit 2bxxvaMTTFKqTq0EDX5EjA 5 1 0 0 2.2kb 1.1kb

3.五、在kibana上建立相應的索引並查看數據。

就能夠看到歷史數據會存儲到ES中去

（四）、查看是否經過db訪問仍是es,修改該文件（/etc/zabbix/web/zabbix.conf.php）的全局變量就能夠知道訪問的是mysql仍是ES了。

具體能夠參考官方文檔（https://www.zabbix.com/documentation/4.0/manual/appendix/install/elastic_search_setup）