centos7.6下的python3.6.9虛擬環境安裝elastalert

centos7.6安裝python3.6.9+elastalert

1.編譯安裝python3.6.9環境
# 安裝依賴
yum -y install zlib-devel bzip2-devel openssl-devel ncurses-devel sqlite-devel readline-devel tk-devel gdbm-devel db4-devel libpcap-devel xz-devel
# 獲取編譯安裝python3.6.9
mkdir -p /usr/local/python3
wget https://www.python.org/ftp/python/3.6.9/Python-3.6.9.tgz
tar xf Python-3.6.9.tgz
cd Python-3.6.9
./configure --prefix=/usr/local/python3
make && make install
ln -s /usr/local/python3/bin/python-3.6.9/bin/python3.6 /usr/bin/python3
ln -s /usr/local/python3/bin/pip3 /usr/bin/pip3

2.安裝virtualenv虛擬環境
pip3 install virtualenv
# 建立存放虛擬環境的目錄
mkdir -p  /usr/local/venv_py3.6_elastalert-0.2.1

# 建立純淨的虛擬環境
cd /usr/local
git clone https://github.com/Yelp/elastalert.git
cd /usr/local/elastalert

/usr/local/python-3.6.9/bin/virtualenv --no-site-packages --python=/usr/local/python-3.6.9/bin/python3.6 /usr/local/venv_py3.6_elastalert-0.2.1
[root@eus-kibana-elastalert-01:/usr/local/venv_py3.6_elastalert-0.2.1]# source bin/activate
(venv_py3.6_elastalert-0.2.1) [root@eus-kibana-elastalert-01:/usr/local/venv_py3.6_elastalert-0.2.1]#

3.在虛擬的python3.6環境中安裝alasticalert
# 指定庫，安裝依賴，不然可能安裝失敗

(venv_py3.6_elastalert-0.2.1) [root@eus-kibana-elastalert-01:/usr/local/elastalert]# pip install -r requirements.txt -i https://pypi.python.org/simple

# 安裝主程序，不然沒法使用 elastalert-create-index 命令
(venv_py3.6_elastalert-0.2.1) [root@eus-kibana-elastalert-01:/usr/local/elastalert]# python setup.py install

# 運行 elastalert-create-index 配置
(venv_py3.6_elastalert-0.2.1) [root@eus-kibana-elastalert-01:/usr/local/elastalert]# elastalert-create-index
Enter Elasticsearch host: 172.30.0.62
Enter Elasticsearch port: 19200
Use SSL? t/f: f
Enter optional basic-auth username (or leave blank): 
Enter optional basic-auth password (or leave blank): 
Enter optional Elasticsearch URL prefix (prepends a string to the URL of every request): 
New index name? (Default elastalert_status) 
New alias name? (Default elastalert_alerts) 
Name of existing index to copy? (Default None) 
Elastic Version: 7.3.0
Reading Elastic 6 index mappings:
Reading index mapping 'es_mappings/6/silence.json'
Reading index mapping 'es_mappings/6/elastalert_status.json'
Reading index mapping 'es_mappings/6/elastalert.json'
Reading index mapping 'es_mappings/6/past_elastalert.json'
Reading index mapping 'es_mappings/6/elastalert_error.json'
New index elastalert_status created
Done!
(venv_py3.6_elastalert-0.2.1) [root@eus-kibana-elastalert-01:/usr/local/elastalert]# 


# 報錯
(venv_py3.6_elastalert-0.2.1) [root@eus-kibana-elastalert-01:/usr/local/elastalert]# elastalert-test-rule example_rules/my_rule.yml

  File "/usr/local/venv_py3.6_elastalert-0.2.1/lib/python3.6/site-packages/tzlocal/unix.py", line 90, in _get_localzone
    utils.assert_tz_offset(tz)
  File "/usr/local/venv_py3.6_elastalert-0.2.1/lib/python3.6/site-packages/tzlocal/utils.py", line 38, in assert_tz_offset
    raise ValueError(msg)
ValueError: Timezone offset does not match system offset: 28800 != -25200. Please, check your config files.

# 代碼和系統時區不匹配，從新設置爲上海時區
(venv_py3.6_elastalert-0.2.1) [root@eus-kibana-elastalert-01:/usr/local/elastalert]# timedatectl set-timezone Asia/Shanghai
(venv_py3.6_elastalert-0.2.1) [root@eus-kibana-elastalert-01:/usr/local/elastalert]# elastalert-test-rule example_rules/my_rule.yml
INFO:elastalert:Note: In debug mode, alerts will be logged to console but NOT actually sent.
            To send them but remain verbose, use --verbose instead.
WARNING:elasticsearch:GET http://172.30.0.62:19200/logstash-*/_search?ignore_unavailable=true&size=1 [status:400 request:0.004s]
Error running your filter:
RequestError(400, 'parsing_exception', {'error': {'root_cause': [{'type': 'parsing_exception', 'reason': '[term] query malformed, no start_object after query name', 'line': 1, 'col': 151}], 'type': 'parsing_exception', 'reason': '[term] query malformed, no start_object after query name', 'line': 1, 'col': 151}, 'status': 400})
INFO:elastalert:Note: In debug mode, alerts will be logged to console but NOT actually sent.
                To send them but remain verbose, use --verbose instead.
1 rules loaded
INFO:apscheduler.scheduler:Adding job tentatively -- it will be properly scheduled when the scheduler starts
WARNING:elasticsearch:GET http://172.30.0.62:19200/logstash-*/_search?_source_includes=%2A%2C%40timestamp&ignore_unavailable=true&scroll=30s&size=10000 [status:400 request:0.003s]
ERROR:root:Error running query: RequestError(400, 'parsing_exception', '[term] query malformed, no start_object after query name')

Would have written the following documents to writeback index (default is elastalert_status):

elastalert_error - {'message': "Error running query: RequestError(400, 'parsing_exception', '[term] query malformed, no start_object after query name')", 'traceback': ['Traceback (most recent call last):', '  File "/usr/local/venv_py3.6_elastalert-0.2.1/lib/python3.6/site-packages/elastalert-0.2.1-py3.6.egg/elastalert/elastalert.py", line 384, in get_hits', '    **extra_args', '  File "/usr/local/venv_py3.6_elastalert-0.2.1/lib/python3.6/site-packages/elasticsearch/client/utils.py", line 84, in _wrapped', '    return func(*args, params=params, **kwargs)', '  File "/usr/local/venv_py3.6_elastalert-0.2.1/lib/python3.6/site-packages/elasticsearch/client/__init__.py", line 819, in search', '    "GET", _make_path(index, "_search"), params=params, body=body', '  File "/usr/local/venv_py3.6_elastalert-0.2.1/lib/python3.6/site-packages/elasticsearch/transport.py", line 350, in perform_request', '    timeout=timeout,', '  File "/usr/local/venv_py3.6_elastalert-0.2.1/lib/python3.6/site-packages/elasticsearch/connection/http_requests.py", line 156, in perform_request', '    self._raise_error(response.status_code, raw_data)', '  File "/usr/local/venv_py3.6_elastalert-0.2.1/lib/python3.6/site-packages/elasticsearch/connection/base.py", line 181, in _raise_error', '    status_code, error_message, additional_info', "elasticsearch.exceptions.RequestError: RequestError(400, 'parsing_exception', '[term] query malformed, no start_object after query name')"], 'data': {'rule': 'eus-log-elasticsearch-cluster-alert', 'query': {'query': {'bool': {'filter': {'bool': {'must': [{'range': {'@timestamp': {'gt': '2019-09-17T05:06:25.831477Z', 'lte': '2019-09-17T05:21:25.831477Z'}}}, {'term': None}, {'query_string': {'query': 'message: error'}}]}}}}, 'sort': [{'@timestamp': {'order': 'asc'}}]}}}


4.配置elastalert
##############全局配置
[root:/usr/local/elastalert#cp  config.yaml.example config.yaml
#存放elastalert 規則的文件夾，你的elastalert 放到哪裏就放到哪裏就好了
rules_folder: /usr/local/elastalert/example_rules
 
#Elastalert 多久去查詢一下根據定義的規則去elasticsearch 查詢是否有符合規則的字段，若是有就會觸發報警，若是沒有就等待下一次時間再檢查，時間定義的單位從周到秒均可以，具體定義方法以下。
run_every:
 #seconds：1
  minutes: 1
  #hours：1
  #days：1
  #weeks：1
 
#當查詢開始一直到結束，最大的緩存時間。
buffer_time:
  minutes: 15
 
#你的Elasticsearch ip地址
es_host: 172.30.0.52
 
#Elasticsearch 的端口
es_port: 9200
 
#這個是elastalert 在es裏邊寫的index
# The index on es_host which is used for metadata storage
# This can be a unmapped index, but it is recommended that you run
# elastalert-create-index to set a mapping
writeback_index: elastalert_status
 
#若是alert當時沒有發出去重試多久以後放棄發送；
alert_time_limit:
  days: 2

[root@eus-kibana-elastalert-01:/usr/local/elastalert]# egrep -v '^#|^$' config.yaml
rules_folder: example_rules
run_every:
  minutes: 1
buffer_time:
  minutes: 15
es_host: 172.30.0.62
es_port: 19200
writeback_index: elastalert_status
writeback_alias: elastalert_alerts
alert_time_limit:
  days: 2

##############rules 的定義
[root@ws-elk-cluster01:/usr/local/elastalert]#cp example_frequency.yaml my_rule.yaml
vi my_rule.yaml
# Alert when the rate of events exceeds a threshold
#Elasticsearch  機器
es_host: 192.168.115.65
 
#Elasticsearch  端口
es_port: 9200
 
#若是elasticsearch 有認證，填寫用戶名和密碼的地方
# (Optional) basic-auth username and password for Elasticsearch
#es_username: someusername
#es_password: somepassword
 
#rule name 必須是獨一的，否則會報錯，這個定義完成以後，會成爲報警郵件的標題
# (Required)
# Rule name, must be unique
name: ws-elk-cluster-alert
 
#配置一種數據驗證的方式，有 any，blacklist，whitelist，change，frequency，spike，flatline，new_term，cardinality
any：只要有匹配就報警；
blacklist：compare_key字段的內容匹配上 blacklist數組裏任意內容；
whitelist：compare_key字段的內容一個都沒能匹配上whitelist數組裏內容；
change：在相同query_key條件下，compare_key字段的內容，在 timeframe範圍內 發送變化；
frequency：在相同 query_key條件下，timeframe 範圍內有num_events個被過濾出 來的異常；
spike：在相同query_key條件下，先後兩個timeframe範圍內數據量相差比例超過spike_height。其中能夠經過spike_type設置具體漲跌方向是- up,down,both 。還能夠經過threshold_ref設置要求上一個週期數據量的下限，threshold_cur設置要求當前週期數據量的下限，若是數據量不到下限，也不觸發；
flatline：timeframe 範圍內，數據量小於threshold 閾值；
new_term：fields字段新出現以前terms_window_size(默認30天)範圍內最多的terms_size (默認50)個結果之外的數據；
cardinality：在相同 query_key條件下，timeframe範圍內cardinality_field的值超過 max_cardinality 或者低於min_cardinality
 
# (Required)
# Type of alert.
# the frequency rule type alerts when num_events events occur with timeframe time
#我配置的是frequency，這個須要兩個條件知足，在相同 query_key條件下，timeframe 範圍內有num_events個被過濾出來的異常
type: frequency
 
#這個index 是指再kibana 裏邊的index，支持正則匹配，支持多個index，同時若是嫌麻煩直接* 也能夠。
index: customer*
#index: es-nginx*,winlogbeat*
 
#時間出發的次數
num_events: 5
 
#和上邊的參數關聯，也就是說在4分鐘內出發5次會報警
timeframe:
  minutes: 4
 
#這個仍是很是關鍵的地方，就是你但願程序的message裏邊出現了什麼樣的關鍵字就報警，這個其實就是elasticsearch 的query語句，支持 AND&OR等。
filter:
- query:
    query_string:
      query: "message: 錯誤  OR Error"
 
#一但須要報警用那種方式報警，支持以下的方式，同時官方支持自定義，我用常規的郵件方式做爲報警方式。
alert:
- "email"
#在郵件正文會顯示你定義的alert_text
alert_text: "Ref Log http://192.168.254.194"
#報警郵箱的smtp server
smtp_host: mail.chinasoft.cn
#報警郵箱的smtp 端口
smtp_port: 25
#須要把認證信息寫到額外配置文件裏，須要user和password兩個屬性
smtp_auth_file: /usr/local/elastalert/example_rules/smtp_auth_file.yaml
email_reply_to:jack@163.com
from_addr: jack@163.com
 
#接受報警郵箱的地址,能夠寫多個，固然後邊搞個郵件組最好了。
# (required, email specific)
# a list of email addresses to send alerts to
email:
- "jack@163.com"


[root@eus-kibana-elastalert-01:/usr/local/elastalert/example_rules]# egrep -v '^#|^$' my_rule.yml 
es_host: 172.30.0.62
es_port: 19200
name: eus-log-elasticsearch-cluster-alert
type: frequency
index: filebeats-log*
num_events: 5
timeframe:
  hours: 4
filter:
- term:
- query:
    query_string:
      query: "message: error"
alert:
- "email"
email:
- "jack@chinasoft.cn"
alert_text: "Ref Log http://172.30.0.62"
smtp_host: mail.chinasoft.cn
smtp_port: 25
smtp_auth_file: /usr/local/elastalert/example_rules/smtp_auth_file.yaml
email_reply_to: jack@chinasoft.cn
from_addr: jack@chinasoft.cn

 
######################smtp認證文件
[root@ws-elk-cluster01:/usr/local/elastalert]#vi smtp_auth_file.yaml
user: "jack"
password: "jack123"


#經過elastalert-test-rule 測試一下咱們寫的rule 是否有問題
[root@ws-elk-cluster01:/usr/local/elastalert/example_rules]# elastalert-test-rule my_rule.yaml


#配置檢查成功以後，咱們就能夠把程序跑起來了，把全部的日誌直接打在前端，這樣方便驗證
/usr/local/venv_py3.6_elastalert-0.2.1/bin/python3.6 -m elastalert.elastalert --verbose --rule /usr/local/elastalert/example_rules/my_rule.yaml