Elasticsearch 6.5 實現冷熱數據分離

1、安裝Elasticsearch集羣web管理工具Cerebronode

1.1 下載linux

https://github.com/lmenezes/cerebro/releasesnginx

[root@es-master1 software]# wget https://github.com/lmenezes/cerebro/releases/download/v0.8.1/cerebro-0.8.1.zip

1.2 安裝及配置git

[root@es-master1 software]# unzip cerebro-0.8.1.zip
[root@es-master1 software]# ln -s cerebro-0.8.1 cerebro
[root@es-master1 software]# cd cerebro
[root@es-master1 cerebro]# ll
total 16
drwxr-xr-x 2 root root   40 Dec  5 10:40 bin
drwxr-xr-x 3 root root  103 Dec  5 10:40 conf
drwxr-xr-x 2 root root 8192 Dec  5 10:40 lib
-rw-r--r-- 1 root root 1081 Jun 20 11:03 README.md
[root@es-master1 cerebro]# cd conf
[root@es-master1 conf]# vim application.conf
hosts = [
  {
    host = "http://es-master1.linuxplus.com:9200"
    name = "linuxplus"
    auth = {
         username = "root"
         password = "123456"
     }
  }
]
[root@es-master1 conf]# cd ..
[root@es-master1 cerebro]# nohup ./bin/cerebro -Dhttp.port=1234 -Dhttp.address=IP &
[1] 77172

2、配置Elasticsearch實現冷熱數據分離github

節點名稱 服務器類型 存儲數據
es-master1 SATA 元數據
es-master2
es-master3
es-hot1 SSD Hot
es-hot2
es-hot3
es-stale1 SATA Cold
es-stale2

2.1 配置Master節點web

  • Master1節點配置(其餘節點配置相似)shell

[root@es-master1 ~]# cd /etc/elasticsearch/
[root@es-master1 elasticsearch]# vim elasticsearch.yml
cluster.name: linuxplus
node.name: es-master1.linuxplus.com
node.attr.rack: r6
node.master: true
node.data: false
path.data: /var/lib/elasticsearch
path.logs: /var/log/elasticsearch
network.host: 0.0.0.0
discovery.zen.ping.unicast.hosts: ["es-master1.linuxplus.com:9300","es-master2.linuxplus.com:9300","es-master3.linuxplus.com:9300","es-hot1.linuxplus.com:9300","es-hot2.linuxplus.com:9300","es-hot3.linuxplus.com:9300","es-stale1.linuxplus.com:9300","es-stale2.linuxplus.com:9300"]
discovery.zen.minimum_master_nodes: 1
bootstrap.system_call_filter: false

2.2 配置Hot節點json

  • Hot1節點配置(其餘節點配置相似)bootstrap

[root@es-hot1 elasticsearch]# vim elasticsearch.yml
cluster.name: linuxplus
node.name: es-hot1.linuxplus.com     #須要修改
node.attr.rack: r1
node.master: false
node.data: true
path.data: /var/lib/elasticsearch
path.logs: /var/log/elasticsearch
network.host: 10.10.10.24           #須要修改
discovery.zen.ping.unicast.hosts: ["es-master1.linuxplus.com:9300","es-master2.linuxplus.com:9300","es-master3.linuxplus.com:9300"]
discovery.zen.minimum_master_nodes: 1
bootstrap.system_call_filter: false
node.attr.box_type: hot
[root@es-hot1 elasticsearch]# /etc/init.d/elasticsearch start

在Hot節點打tagvim

node.attr.box_type: hot

2.3 配置Stale節點

  • Stale1節點配置(其餘節點配置相似)

[root@es-stale1 elasticsearch]# vim elasticsearch.yml
cluster.name: linuxplus
node.name: es-stale1.linuxplus.com
node.attr.rack: r1
node.master: false
node.data: true
path.data: /var/lib/elasticsearch
path.logs: /var/log/elasticsearch
network.host: 10.10.10.27
discovery.zen.ping.unicast.hosts: ["es-master1.linuxplus.com:9300","es-master2.linuxplus.com:9300","es-master3.linuxplus.com:9300"]
discovery.zen.minimum_master_nodes: 1
bootstrap.system_call_filter: false
node.attr.box_type: cold
[root@es-stale1 elasticsearch]# /etc/init.d/elasticsearch start

在Stale節點打tag

node.attr.box_type: cold

3、實現熱數據寫入指定節點

經過Cerebro修改模板添加以下配置

"settings": {
    "index": {
      "number_of_shards": "3",
      "auto_expand_replicas": "0-1",
      "routing": {
        "allocation": {
          "require": {
            "box_type": "hot"
          }
        }
      }
    }
  },

image.pngimage.png

經過shell腳本將Hot數據(保留7天)遷移到Stale

#!/bin/bash
Time=$(date -d "1 week ago" +"%Y.%m.%d")
Hostname=$(hostname)
arr=("cron" "messages" "secure" "tomcat" "nginx-access" "nginxtcp" "nginxerror" "windows" ".monitoring-es-6" ".monitoring-beats-6" ".monitoring-kibana-6" ".monitoring-logstash-6" "metricbeat-6.5.3")
for var in ${arr[@]}
do
    curl -H "Content-Type: application/json" -XPUT http://$Hostname:9200/$var-$Time/_settings?pretty -d'
    { 
       "settings": { 
             "index.routing.allocation.require.box_type": "cold"
        } 
    }'
done
相關文章
相關標籤/搜索