Azure vm 擴展腳本自動部署Elasticsearch集羣

1、完整過程比較長，我僅給出Azure vm extension script 一鍵部署Elasticsearch集羣的安裝腳本，有須要的同窗，能夠郵件我，我給你完整的ARM Template

若是你不是用Azure VM ，也不要緊，個人bash腳本都是一個個的函數，你能夠本身改形成本身的Linux安裝腳本。

2、此腳本的誕生是爲了解決兩個問題：

1. ELK在線安裝有時候會異常緩慢，致使整個在線安裝腳本奔潰
2. 一個一個vm節點部署Elasticsearch集羣比較繁瑣，此處實現一鍵自動生成集羣

大體的思路是先離線下載好安裝包，存放到本身的容易下載的地方，我此處用的是Azure blob storage，只要達到一個目的：你能經過一個連接，能快速，方便地訪問下載到Elasticsearch的安裝包便可。利用Azure VM extension，調用我給出的腳本，自行下載和安裝。

3、達到的目的：

1. 運行成功後，自動建立多臺虛擬機，而且虛擬機ES之間組成集羣
2. ES集羣獲得初步優化，vm system在適配ES上也獲得初步優化

4、安裝腳本：

#!/bin/bash
# Copyright (c) Microsoft.  All rights reserved.
# Licensed under ELASTIC LICENSE
##
# Author : Wenbo Yang  July/16/2018
##
# Reference: https://github.com/Azure/azure-diagnostics-tools/blob/master/ELK-Semantic-Logging/ELK/AzureRM/logstash-on-ubuntu/logstash-install-ubuntu.sh
# Reference: https://github.com/Azure/azure-quickstart-templates/blob/master/elasticsearch/scripts/elasticsearch-ubuntu-install.sh
# Reference: https://github.com/Azure/azure-quickstart-templates/blob/master/elasticsearch-vmss/install-elasticsearch.sh
# Reference: https://github.com/Azure/azure-diagnostics-tools/blob/master/ES-MultiNode/es-ubuntu-install.sh
##
 
 
# Log method to control/redirect log output
log()
{
    echo "$1"
    logger "$1"
}
 
if [ "${UID}" -ne 0 ];
then
    log "Script executed without root permissions"
    echo "You must be root to run this program." >&2
    exit 3
fi
 
# TEMP FIX - Re-evaluate and remove when possible
# This is an interim fix for hostname resolution in current VM
grep -q "${HOSTNAME}" /etc/hosts
if [ $? == 0 ]
then
  echo "${HOSTNAME} found in /etc/hosts"
else
  echo "${HOSTNAME} not found in /etc/hosts"
  # Append it to the hosts file if not there
  echo "127.0.0.1 ${HOSTNAME}" >> /etc/hosts
  log "hostname ${HOSTNAME} added to /etc/hosts"
fi
 
 
#Loop through options passed
while getopts :p:c:m:e:k:h optname; do
  log "Option $optname set with value ${OPTARG}"
  case $optname in
    p) #set cluster name
      FIRSTPRIVATEIP=${OPTARG}
      ;;
    c) #set cluster name
      NODECOUNT=${OPTARG}
      ;;
    e) #set master mode
      ES_DOWNLOAD_URL=${OPTARG}
      ;;
    k) #set master mode
      KIBANA_DOWNLOAD_URL=${OPTARG}
      ;;
    h) #show help
      help
      exit 2
      ;;
    \?) #unrecognized option - show help
      echo -e \\n"Option -${BOLD}$OPTARG${NORM} not allowed."
      help
      exit 2
      ;;
  esac
done
 
# Usage: get_discovery_endpoints start_address node_count
# Example: get_discovery_endpoints 10.0.1.4 3
# (returns ["10.0.1.4", "10.0.1.5", "10.0.1.6"]
get_discovery_endpoints()
{
    declare start_address=$1
    declare address_prefix=${start_address%.*}     # Everything up to last dot (not including)
    declare -i address_suffix_start=${start_address##*.}  # Last part of the address, interpreted as a number
    declare retval='['
    declare -i i
    declare -i suffix
     
    for (( i=0; i<$2; ++i )); do suffix=$(( address_suffix_start + i )) retval+="\"${address_prefix}.${suffix}\", " done retval=${retval:0:-2} # Remove last comma and space retval+=']' echo $retval } # Install Oracle Java install_java() { log "Installing Java" add-apt-repository -y ppa:webupd8team/java apt-get -y update > /dev/null
    echo debconf shared/accepted-oracle-license-v1-1 select true | sudo debconf-set-selections
    echo debconf shared/accepted-oracle-license-v1-1 seen true | sudo debconf-set-selections
    apt-get -y install oracle-java8-installer  > /dev/null
    java -version
    if [ $? -ne 0 ]; then
        log "Java installation failed"
        exit 1
    fi
}
 
# Install ES with Debian Package manually
install_es()
{
    log "Installing Elaticsearch"
    sudo wget -q "$ES_DOWNLOAD_URL" -O elasticsearch.deb
    sudo dpkg -i elasticsearch.deb
}
 
# Install Kibana with Debina Package manually
install_kibana()
{
    log "Installing Kibana"
    sudo wget -q "$KIBANA_DOWNLOAD_URL" -O kibana.deb
    sudo dpkg -i kibana.deb
}
 
 
# Configure elasticsearch
configure_es()
{
    log "Update configuration"
    mv /etc/elasticsearch/elasticsearch.yml /etc/elasticsearch/elasticsearch.bak
    echo "cluster.name: elasticsearch" >> /etc/elasticsearch/elasticsearch.yml
    echo "node.name: ${HOSTNAME}" >> /etc/elasticsearch/elasticsearch.yml
    declare -i minimum_master_nodes=$((($NODECOUNT / 2) + 1))
    echo "discovery.zen.minimum_master_nodes: 2" >> /etc/elasticsearch/elasticsearch.yml
    discovery_endpoints=$(get_discovery_endpoints $FIRSTPRIVATEIP $NODECOUNT)
    echo $discovery_endpoints
    echo "discovery.zen.ping.unicast.hosts: $discovery_endpoints" >> /etc/elasticsearch/elasticsearch.yml
    IPADDRESS=$(ip route get 8.8.8.8 | awk 'NR==1 {print $NF}')
    echo "network.host: ${IPADDRESS}" >> /etc/elasticsearch/elasticsearch.yml
    echo "http.port: 9200" >> /etc/elasticsearch/elasticsearch.yml
    echo "bootstrap.memory_lock: true" >> /etc/elasticsearch/elasticsearch.yml
 
    echo "node.master: true" >> /etc/elasticsearch/elasticsearch.yml
    echo "node.data: true" >> /etc/elasticsearch/elasticsearch.yml
    echo "path.data: /var/lib/elasticsearch" >> /etc/elasticsearch/elasticsearch.yml
    echo "path.logs: /var/log/elasticsearch" >> /etc/elasticsearch/elasticsearch.yml
    sudo update-rc.d elasticsearch defaults 95 10
    service elasticsearch start
    #sudo systemctl stop elasticsearch.service
    sleep 30
 
    if [ <code>systemctl is-failed elasticsearch.service</code> == 'failed' ];
    then
        log &quot;Elasticsearch unit failed to start&quot;
        exit 1
    fi
}
 
# Configure kibana
configure_kibana()
{
    IPADDRESS=$(ip route get 8.8.8.8 | awk 'NR==1 {print $NF}')
    echo &quot;server.host: \&quot;${IPADDRESS}\&quot;&quot; &gt;&gt; /etc/kibana/kibana.yml
    echo &quot;elasticsearch.url: \&quot;http://${IPADDRESS}:9200\&quot;&quot; &gt;&gt; /etc/kibana/kibana.yml
    sudo update-rc.d kibana defaults 95 10
    service kibana start
    #sudo systemctl stop kibana.service
    sleep 10
     
    if [ <code>systemctl is-failed kibana.service</code> == 'failed' ];
    then
        log &quot;Kibana unit failed to start&quot;
        exit 1
    fi   
}
 
configure_system()
{
    echo &quot;options timeout:1 attempts:5&quot; &gt;&gt; /etc/resolvconf/resolv.conf.d/head
    resolvconf -u
    ES_HEAP=<code>free -m |grep Mem | awk '{if ($2/2 &gt;31744)  print 31744;else printf &quot;%.0f&quot;, $2/2;}'</code>
    echo &quot;ES_JAVA_OPTS=\&quot;-Xms${ES_HEAP}m -Xmx${ES_HEAP}m\&quot;&quot; &gt;&gt; /etc/default/elasticsearch
    echo &quot;JAVA_HOME=$JAVA_HOME&quot; &gt;&gt; /etc/default/elasticsearch
    echo 'MAX_OPEN_FILES=65536' &gt;&gt; /etc/default/elasticsearch
    echo 'MAX_LOCKED_MEMORY=unlimited' &gt;&gt; /etc/default/elasticsearch
    
    #https://www.elastic.co/guide/en/elasticsearch/reference/current/setting-system-settings.html#systemd
    mkdir -p /etc/systemd/system/elasticsearch.service.d
    touch /etc/systemd/system/elasticsearch.service.d/override.conf
    echo '[Service]' &gt;&gt; /etc/systemd/system/elasticsearch.service.d/override.conf
    echo 'LimitMEMLOCK=infinity' &gt;&gt; /etc/systemd/system/elasticsearch.service.d/override.conf
}
 
log &quot; ---------------begin------------------- &quot;
install_java
log &quot;Master Node install elasticsearch and kibana&quot;
log &quot;Install configure and start elasticsearch&quot;
install_es
configure_system
configure_es
log &quot;Install configure and start kibana&quot;
install_kibana
configure_kibana