#!/bin/sh
# Init script for logstash
# Maintained by Elasticsearch
# Generated by pleaserun.
# Implemented based on LSB Core 3.1:
# * Sections: 20.2, 20.3
#
### BEGIN INIT INFO
# Provides: logstash
# Required-Start: $remote_fs $syslog
# Required-Stop: $remote_fs $syslog
# Default-Start: 2 3 4 5
# Default-Stop: 0 1 6
# Short-Description:
# Description: Starts Logstash as a daemon.
### END INIT INFOjava
PATH=/sbin:/usr/sbin:/bin:/usr/bin
export PATHnode
if [ `id -u` -ne 0 ]; then
echo "You need root privileges to run this script"
exit 1
filinux
name=logstash
pidfile="/var/run/$name.pid"git
LS_USER=root
LS_GROUP=root
LS_HOME=/var/lib/logstash
LS_HEAP_SIZE="1000m"
LS_LOG_DIR=/var/log/logstash
LS_LOG_FILE="${LS_LOG_DIR}/$name.log"
LS_CONF_DIR=/etc/logstash/conf.d/
LS_OPEN_FILES=16384
LS_NICE=19
LS_OPTS=""github
[ -r /etc/default/$name ] && . /etc/default/$name
[ -r /etc/sysconfig/$name ] && . /etc/sysconfig/$nameweb
program=/usr/share/logstash/bin/logstash
args="-f ${LS_CONF_DIR} -l ${LS_LOG_FILE} ${LS_OPTS}"redis
start() {npm
LS_JAVA_OPTS="${LS_JAVA_OPTS} -Djava.io.tmpdir=${LS_HOME}"
HOME=${LS_HOME}
export PATH HOME LS_HEAP_SIZE LS_JAVA_OPTS LS_USE_GC_LOGGINGjson
# chown doesn't grab the suplimental groups when setting the user:group - so we have to do it for it.
# Boy, I hope we're root here.
SGROUPS=$(id -Gn "$LS_USER" | tr " " "," | sed 's/,$//'; echo '')bootstrap
if [ ! -z $SGROUPS ]
then
EXTRA_GROUPS="--groups $SGROUPS"
fi
# set ulimit as (root, presumably) first, before we drop privileges
ulimit -n ${LS_OPEN_FILES}
# Run the program!
nice -n ${LS_NICE} chroot --userspec $LS_USER:$LS_GROUP $EXTRA_GROUPS / sh -c "
cd $LS_HOME
ulimit -n ${LS_OPEN_FILES}
exec \"$program\" $args
" > "${LS_LOG_DIR}/$name.stdout" 2> "${LS_LOG_DIR}/$name.err" &
# Generate the pidfile from here. If we instead made the forked process
# generate it there will be a race condition between the pidfile writing
# and a process possibly asking for status.
echo $! > $pidfile
echo "$name started."
return 0
}
stop() {
# Try a few times to kill TERM the program
if status ; then
pid=`cat "$pidfile"`
echo "Killing $name (pid $pid) with SIGTERM"
kill -TERM $pid
# Wait for it to exit.
for i in 1 2 3 4 5 ; do
echo "Waiting $name (pid $pid) to die..."
status || break
sleep 1
done
if status ; then
if [ "$KILL_ON_STOP_TIMEOUT" -eq 1 ] ; then
echo "Timeout reached. Killing $name (pid $pid) with SIGKILL. This may result in data loss."
kill -KILL $pid
echo "$name killed with SIGKILL."
else
echo "$name stop failed; still running."
fi
else
echo "$name stopped."
fi
fi
}
status() {
if [ -f "$pidfile" ] ; then
pid=`cat "$pidfile"`
if kill -0 $pid > /dev/null 2> /dev/null ; then
# process by this pid is running.
# It may not be our pid, but that's what you get with just pidfiles.
# TODO(sissel): Check if this process seems to be the same as the one we
# expect. It'd be nice to use flock here, but flock uses fork, not exec,
# so it makes it quite awkward to use in this case.
return 0
else
return 2 # program is dead but pid file exists
fi
else
return 3 # program is not running
fi
}
force_stop() {
if status ; then
stop
status && kill -KILL `cat "$pidfile"`
fi
}
case "$1" in
start)
status
code=$?
if [ $code -eq 0 ]; then
echo "$name is already running"
else
start
code=$?
fi
exit $code
;;
stop) stop ;;
force-stop) force_stop ;;
status)
status
code=$?
if [ $code -eq 0 ] ; then
echo "$name is running"
else
echo "$name is not running"
fi
exit $code
;;
restart)
stop && start
;;
*)
echo "Usage: $SCRIPTNAME {start|stop|force-stop|status|restart}" >&2
exit 3
;;
esac
exit $?
######安裝redis
# yum –y install tcl gcc //安裝測試依賴工具
# tar xf redis-3.2.3.tar.gz //解壓至當前目錄
# cd redis-3.2.3 //切換至redis源碼目錄
# make //編譯
# make test //有可能會失敗，只是看看，不用在乎
# make install //安裝
# mkdir /opt/redis/{db,conf} -pv //建立redis安裝目錄
# cp redis.conf /opt/redis/conf/ //複製配置文件至redis安裝目錄
# cd src
# cp redis-benchmark redis-check-aof redis-cli redis-server mkreleasehdr.sh /opt/redis/ //複製各文件至redis安裝目錄
# ln -sv /opt/redis/redis-cli /usr/bin/ //將redis-cli可執行文件連接至/usr/bin目錄，方便在任何目錄下均可直接執行`redis-cli`
# vim /opt/redis/conf/redis.conf //修改redis.conf 中的 `daemonize`爲`yes`,讓server以守護進程在後臺執行，這一步能夠不作，由於後面要執行的腳本會自動建立這個文件，且這個值會設置爲`yes`
daemonize yes
make install僅僅在你的系統上安裝了二進制文件，不會替你默認配置init腳本和配置文件，爲了把它用在生產環境而安裝它，在源碼目錄的utils目錄下Redis爲系統提供了
這樣的一個腳本install_server.sh
# cd /opt/soft/redis-3.2.3/utils
# ./install_server.sh //執行sh格式的安裝腳本
Welcome to the redis service installer
This script will help you easily set up a running redis server
Please select the redis port for this instance: [6379]
Selecting default: 6379
Please select the redis config file name [/etc/redis/6379.conf] /opt/redis/conf/redis.conf
Please select the redis log file name [/var/log/redis_6379.log]
Selected default - /var/log/redis_6379.log
Please select the data directory for this instance [/var/lib/redis/6379] /opt/redis/db/6379.db
Please select the redis executable path [/usr/local/bin/redis-server]
Selected config:
Port : 6379
Config file : /opt/redis/conf/redis.conf
Log file : /var/log/redis_6379.log
Data dir : /opt/redis/db/6379.db
Executable : /usr/local/bin/redis-server
Cli Executable : /usr/local/bin/redis-cli
Is this ok? Then press ENTER to go on or Ctrl-C to abort.
Copied /tmp/6379.conf => /etc/init.d/redis_6379
Installing service...
Successfully added to chkconfig!
Successfully added to runlevels 345!
Starting Redis server...
Installation successful!
# chkconfig --add redis_6379 //將redis加入系統服務
# chkconfig redis_6379 on //加入開機啓動
# vim /opt/redis/conf/redis.conf
requirepass Carsing2582# //設置密碼【坑1】：這步先不作，若是作了後面的測試鏈接redis的時候就須要認證）
bind 0.0.0.0 //redis要打開遠程訪問容許，在 redis.conf 中 bind 這一行改爲 0.0.0.0(運行任何機器鏈接，爲了測試先這樣)【此爲坑2】
# /etc/init.d/redis_6379 restart
# ss -an| grep 6379 //端口已經被監聽，證實服務已提供
LISTEN 0 128 127.0.0.1:6379 *:*
TIME-WAIT 0 0 127.0.0.1:6379 127.0.0.1:34506
###修改 elasticsearch配置文件 存放所收集到的日誌
cd /etc/elasticsearch/
vim elasticsearch.yml
cluster.name: ELK ###集羣名稱
node.name: node-1 ###節點編號
node.master: true
node.data: true
path.data: /ELK/data ###elasticsearch數據路徑
path.logs: /ELK/logs ###elasticsearch日誌路徑
network.host: elk02 ###本機主機名
discovery.zen.ping.unicast.hosts: ["elk02", "elk03"] ###自動發現節點主機名
discovery.zen.minimum_master_nodes: 2
http.cors.enabled: true
http.cors.allow-origin: "*"
xpack.security.enabled: false
啓動異常：ERROR: bootstrap checks failed
system call filters failed to install; check the logs and fix your configuration or disable system call filters at your own risk
問題緣由：由於Centos6不支持SecComp，而ES5.2.1默認bootstrap.system_call_filter爲true進行檢測，因此致使檢測失敗，失敗後直接致使ES不能啓動。詳見 ：https://github.com/elastic/elasticsearch/issues/22899
解決方法：在elasticsearch.yml中配置bootstrap.system_call_filter爲false，注意要在Memory下面:
bootstrap.memory_lock: false
bootstrap.system_call_filter: false
設置線程池
thread_pool:
bulk:
size: 5
queue_size: 1000
search:
size: 5
queue_size: 1000
index:
size: 5
queue_size: 1000
#xpack.monitoring.enabled: false
#xpack.monitoring.exporters.my_local:
# type: local
indices.cache.filter.expire: 30d ###設置索引30天過時
index.cache.filter: 30d
vim /etc/elasticsearch/jvm.options ###內存酌情分配
-Xms8g
-Xmx8g
vi /etc/security/limits.conf
elasticsearch soft memlock unlimited
elasticsearch hard memlock unlimited
elasticsearch soft nofile 65536
elasticsearch hard nofile 131072
elasticsearch soft nproc 2048
elasticsearch hard nproc 4096
* soft nofile 65536
* hard nofile 65536
* soft memlock unlimited
* hard memlock unlimited
建立ELK數據、日誌存放路徑
mkdir /ELK/data
mkdir /ELK/logs
chown -R elasticsearch:elasticsearch /ELK
啓動服務
service elasticsearch start
啓動服務以後查看/ELK/logs路徑中有日誌，說明服務正常
查看端口9200、9300
9200端口是API 用來鏈接服務的接口
9300是內部服務
PS：報錯
max number of threads [1024] for user [lish] likely too low, increase to at least [2048]
解決：切換到root用戶，進入limits.d目錄下修改配置文件。
vi /etc/security/limits.d/90-nproc.conf
修改以下內容：
* soft nproc 1024
#修改成
* soft nproc 2048
##修改主機名，定義dns
vim /etc/sysconfig/network
NETWORKING=yes
HOSTNAME=elk02
GATEWAY=172.25.0.254
vim /etc/hosts
127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4
::1 localhost localhost.localdomain localhost6 localhost6.localdomain6
172.25.11.77 elk01
172.25.11.78 elk02
172.25.11.79 elk03
注：重啓生效 reboot
###配置kibana
cd /etc/kibana/
vim kibana.yml
server.port: 5601 ##定義kibana端口
server.host: "172.25.11.77" ##服務主機IP
elasticsearch.url: "http://172.25.11.78:9200" ##url調取elasticsearch
elasticsearch.url: "http://172.25.11.79:9200" ##url調取elasticsearch
######啓動kibana
service kibana start
查看端口是否存在
netstat -anp | grep 5601
tcp 0 0 172.25.11.77:5601 0.0.0.0:* LISTEN 8492/node
###web頁面上訪問
####安裝elasticsearch-analysis-ik-5.0.0.zip 分詞器
####安裝elasticsearch-head插件
因爲elasticsearch5.0版本變化較大，目前elasticsearch5.0 暫時不支持直接安裝，可是head做者提供了另外一種安裝方法
###安裝elasticsearch-head
cd /usr/share/elasticsearch
git clone git://github.com/mobz/elasticsearch-head.git
chmod -R 777 elasticsearch-head
cd elasticsearch-head/
PS:先修改license
vim package.json
"license": "Apache-2.0",
npm install --force ###安裝完node就會有npm命令
cd /usr/share/elasticsearch/elasticsearch-head/node_modules/grunt/bin
./grunt server > /dev/null 2>&1 &
###安裝node
node-v6.10.1-linux-x64.tar
解壓
tar -xvf node-v6.10.1-linux-x64.tar
mv node-v6.10.1-linux-x64 /usr/share/elasticsearch/
vim /etc/profile
export NODE_HOME=/usr/share/elasticsearch/node-v6.10.1-linux-x64
export PATH=$PATH:$NODE_HOME/bin
source /etc/profile
cd /usr/share/elasticsearch/node-v6.10.1-linux-x64/bin/
./node -v
v6.10.1
npm -v
3.10.10
###安裝grunt
######cd /usr/share/elasticsearch/elasticsearch-head
######npm install grunt-cli
grunt --version
#### 修改head目錄下的Gruntfile.js配置，head默認監聽127.0.0.1
vim Gruntfile.js
hostname: '*',
connect: {
server: {
options: {
port: 9100,
hostname: '*', ####修改此處
base: '.',
keepalive: true
}
}
cd _site/
vim app.js
this.base_uri = this.config.base_uri || this.prefs.get("app-base_uri") || "http://172.25.11.78:9200";
####elasticsearch配置容許跨域訪問
修改elasticsearch配置文件elasticsearch.yml
cd /etc/elasticsearch/
vim elasticsearch.yml
http.cors.enabled: true
http.cors.allow-origin: "*"
####重啓elasticsearch，並啓動node
service elasticsearch restart ###啓動elasticsearch
cd /usr/share/elasticsearch/elasticsearch-head/ ###啓動node
①#######node_modules/grunt/bin/grunt server > /dev/null 2>&1 &
②#######/usr/share/elasticsearch/elasticsearch-head/node_modules/grunt/bin/grunt server > /dev/null 2>&1 &
查看集羣信息
curl '172.25.11.78:9200 '
ip heap.percent ram.percent cpu load_1m load_5m load_15m node.role master name
172.25.11.78 13 33 0 0.01 0.01 0.00 mdi - node-1
172.25.11.79 10 33 0 0.00 0.02 0.00 mdi * node-2
#####安裝x-pack（marvel） 進行性能監控
配置dns
vim /etc/resolv.conf
nameserver 210.73.88.1
kibana上安裝server端
cd /usr/share/kibana/
bin/kibana-plugin install x-pack
es上安裝agent端
cd /usr/share/elasticsearch/
bin/elasticsearch-plugin install x-pack
卸載
bin/elasticsearch-plugin remove x-pack
PS:
安裝完須要重啓kibana和es 默認用戶名密碼elastic/changeme
查看license時間 默認一個月
curl -XGET -u elastic:1qaz2wsx 'http://172.25.11.78:9200/_xpack/license'
替換license 默認密碼changeme
curl -XPUT -u elastic 'http://172.25.11.78:9200/_xpack/license?acknowledge=true' -H "Content-Type: application/json" -d @zhaishaomin-fa370f7b-6eb7-4730-9b73-89b3d93471e3-v5.json
因爲是基本的license，因此須要禁用安全性
vim /etc/kibana/kibana.yml
vim /etc/elasticsearch/elasticsearch.yml
xpack.security.enabled: false ####false爲禁用
####安裝logstash
rpm -ivh logstash-5.0.0.rpm
############################################################################
PS:安裝出現一下錯誤：
warning: rpmts_HdrFromFdno: Header V3 RSA/SHA256 Signature, key ID fd431d51: NOKEY
解決：
rpm --import /etc/pki/rpm-gpg/RPM*
############################################################################
cd /usr/share/logstash/
mkdir config
cp /etc/logstash/logstash.yml /usr/share/logstash/config/
cp /etc/logstash/log4j2.properties /usr/share/logstash/config/
######cd /etc/logstash/conf.d#######
vim input.conf
input{
#stdin{}
file{
path => "/opt/soft/sub-test.cm-inv.com_access_80.log"
start_position => "beginning" ####end(從末尾讀)beginning（從頭開始讀）
stat_interval => 1
type => "access_log"
tags=> ["acc"]
}
}
filter{
grok{
match => [ "message" , "\[%{HTTPDATE:time_local}\] \"%{IP:client_ip}\" \"%{WORD:verb} (%{NOTSPACE:request}|-) (HTTP/%{NUMBER:http_version}|-)\" \"(%{GREEDYDATA:http_cookie}|-)\" \"(%{WORD:http_x_forword_for}|-)\" (%{GREEDYDATA:nomean}|-) (?<user>[a-zA-Z._-]+|-)(?<status>[0-9._-]+|-) (?:%{NUMBER:body_bytes_sent:int}|-) (%{BASE16FLOAT:request_time:float}|-) \"(%{GREEDYDATA:http_did}|-)\" \"(%{GREEDYDATA:http_x_up_calling_line_id}|-)\" \"(%{NOTSPACE:http_referrer}|-)\" \"%{GREEDYDATA:http_user_agent}\" (%{BASE16FLOAT}|-)(%{NUMBER:content_length}|-)(%{GREEDYDATA:upstream_addr}|-)"]
remove_field => [ "message","host","@version" ]
}
date{
match=>["timestamp","dd/MMM/yyyy:HH:mm:ss Z"]
target=>"@timestamp"
}
geoip{
source => "client_ip"
}
}
###########
output{
elasticsearch {
hosts => "192.168.125.129:9200"
index => "logstash-%{type}-%{+YYYY.MM.dd}"
}
stdout {codec=> rubydebug }
}
################
/usr/share/logstash/bin/logstash -f input.conf
bin/logstash -e ‘input { stdin {} } output { stdout {codec=>rubydebug} }’
刪除一個月的索引腳本
#!/bin/bash
data=`date -d '-1 months' +%Y-%m-%d`
curl -XDELETE 'http://172.25.11.77:9200/*-'${data}''