線上日誌分析與其餘一些腳本

對一些線上經常使用的腳本進行了一下總結和說明，省得之後忘記了~

一·線上發佈API集羣的代碼腳本：

#!/bin/bash
#Author CCC

host='
10.44.22.113
10.44.22.113
10.44.22.112
10.44.22.112
10.44.22.113
10.44.22.113
10.44.22.114
10.44.22.114
10.44.22.115
10.44.22.115
10.44.22.119
10.44.22.119
'
#basePath='/var/www'--------------------定義要發佈的文件路徑
basePath='/root/xxxx/2016-01-27'----------定義源文件的路徑
check_port()---------檢查端口已經已經起來了。
{

ss=0;
count=0;

ip=$2;
port=$1;
for ((i=0;i<120;i++))
do
#    $((count++))
    if [[ -n $(ssh $ip "netstat -ntlp | grep $port") ]]
    then
        ss=1;
        break;
    else
        ss=0;
    fi
sleep 1
done

return $ss;

}


check_return()---------檢查返回內容
{
  if [[ "$1" == "0" ]]
  then
      echo $2
      exit
  fi
  echo $2
  host_ip=""
}

sed -i "s/HEAP_MAX=2g/HEAP_MAX=6g/g" $basePath/query/bin/properties/jvm.properties------------------------修改一些JVM參數，由於目前打包是公共打包，參數可能配置不許確
sed -i "s/PERM_SIZE=128m/PERM_SIZE=256m/g" $basePath/query/bin/properties/jvm.properties
sed -i "s/HEAP_INIT=2g/HEAP_INIT=6g/g" $basePath/query/bin/properties/jvm.properties
sed -i "s/YOUNG_GEN_SIZE=256m/YOUNG_GEN_SIZE=4g/g" $basePath/query/bin/properties/jvm.properties
for i in $host--------------循環進行每臺機器的發佈
do
      echo "deploy $i"
      ssh -o StrictHostKeyChecking=no $i "source /etc/profile;sh /var/www/query/bin/stop.sh"
      rsync -e "ssh -o StrictHostKeyChecking=no" -av  $basePath/query $i:/var/www/ --delete >> /dev/null 2>&1
      ssh -o StrictHostKeyChecking=no $i "source /etc/profile;sh /var/www/query/bin/restart.sh"
  check_port 82 $i
  check_return $? $i
  sleep 10

done

2、線上GET方式查看API集羣中API的返回結果：

#!sbin/bash
#author CCC



providers="`cat server_ip`" ----------找到同目錄底下的ip列表

for host in $providers
do 

RESULT=`curl 'http://'$host':82/productapi/xxxxxxx?bbb=23451243&22=1'` ----------循環執行命令和請求
echo $host    $RESULT ------打印日誌

done

3、查看服務器列表的時間，與上一個類似：

#!sbin/bash
#author CCC

providers="`cat server_ip`"

for host in $providers
do
ssh -o StrictHostKeyChecking=no $host "date"
done

4、集羣日誌分析與返回，是個用的比較多的腳本 *****：

#!/bin/bash
#Author CCC

host=' -----------host列表
10.44.44.211
10.44.44.211
10.44.44.211
10.44.44.211
10.44.44.211
10.44.44.211
10.44.44.211
10.44.44.211
10.44.44.11
10.44.44.11
10.44.44.11
10.44.44.11
10.44.44.11
10.44.44.11
10.44.44.11
10.44.44.11
10.44.44.11
10.44.44.11
10.44.44.11
10.44.44.11
10.44.44.11
10.44.44.11
10.44.44.11
10.44.44.11
'
for i in $host 
do
      echo "deploy $i"
     # ssh $i 'grep "java.net.UnknownHostException: xxxxxxxxx.idc2" /d1/joblog/query/dailylog.2015-12-18_09|awk "{print NR==1}"' ------------查到每臺機器上面的xx目錄下這個錯誤的內容，而且找到第一列的內容打印
     # ssh $i "grep find_gift /d1/joblog/query/accesslog.2015-12-30|grep 20633051|awk '{if(\$NF>200){print \$0}}'" ----找到每一個機器上面find_gift接口中商品id是20633051的接口返回時間大於200ms的內容並打印
     # ssh $i "grep find_xxx_lighter  /d1/joblog/query/accesslog.2016-01-12_19|grep 20633051|awk '{print \$NF}'|sort|uniq -c"----找到xxxx接口中內容包含20633051商品的返回時間（最後一列）排序並計數
     # ssh $i "grep product.php /d1/joblog/query/dailylog.2015-12-28*|awk '{print \$NF}'|tr -d ms|sort|uniq -c" ----找到product.php的響應時間，替換到ms關鍵詞，排序計數
      ssh $i "grep promo_product /d1/joblog/query/accesslog.2016-01-24|grep 20260420|wc -l"----同上上
     #  ssh $i "grep 'Request:' /d1/joblog/query/dailylog.2016-01-05*|grep 'Use time:'|awk {'print \$NF'}|tr -d ms|awk '{if(\$1>500){print \$1}}'|wc -l"----找到外部請求超過500ms的接口並計數
     # ssh $i "sed -n '/^10:00:/','/^10:20:/p' /d1/joblog/query/accesslog.2016-01-13| grep find_lighter|awk '{print \$NF}'|sort|uniq -c"----找到日誌在10點到10點20之間lighter接口的響應時間
done

5、全量清理Redis：

host='
10.66.66.66
10.66.66.66
10.66.66.66
10.66.66.66
10.66.66.66
10.66.66.66
10.66.66.66
'
ports='
8279
8379
8479
8579
'
for h in $host
do
for p in $ports
do
/usr/local/bin/redis-cli -h $h -p $p FLUSHALL
sleep 10
done
done

6、整理內容發郵件：

DATE=`date -d "-1 hour" +%Y-%m-%d_%H_%M`  -------時間定義
cd /root/analyze -------目錄
python query.py > /root/analyze/result_$DATE.txt----運行python腳本並輸出到文件中
mail -s "aaaaa Statistics @ $DATE" aaaaaaaaaaa@qq.com < /root/analyze/result_$DATE.txt ---發送郵件

 7、項目總體發佈腳本：

#!bin/bash
#author CCC  

cd /home/cxht/svn/cxht/XXXXX-parent  -----------更新代碼與打包
svn update
#/home/cxht/apache-maven-3.2.1/bin/mvn -e -U clean install package -Dmaven.test.skip=true
cd /home/cxht/svn/cxht/XXXXX-parent/XXXXX-package
/home/cxht/apache-maven-3.2.1/bin/mvn -e -U clean install -Dmaven.test.skip=true
/home/cxht/apache-maven-3.2.1/bin/mvn package

kill -9 `ps -ef|grep /home/cxht/tomcat|grep -v grep|awk '{print $2}'`   ---------殺掉tomcat進程
ps -ef|grep java|grep /var/www/YYYYY/|awk '{print $2}'|xargs -i kill -9 {} ---------殺掉其餘main方法方式運行的程序
rm -fr /var/www/YYYYY/* --------刪除全部代碼

cd /var/www/YYYYY/

find  /home/cxht/svn/cxht/XXXXX-parent/* -name *.gz|xargs -i tar -xvf {} ----------找到全部的gz包解壓
unzip /home/cxht/svn/cxht/XXXXX-parent/XXXXX-web/target/XXXXX-web.war -d /var/www/YYYYY/XXXXX-web --------解壓war包
unzip /home/cxht/svn/cxht/XXXXX-parent/XXXXX-event-maker/target/XXXXX-event-maker-bin.zip -d /var/www/YYYYY/XXXXX-event-maker

#change jvm memory size ------------設置 一些jvm的屬性
sed -i "s/HEAP_INIT=2g/HEAP_INIT=512m/g" `find /var/www/YYYYY/* -name jvm.properties`
sed -i "s/HEAP_MAX=2g/HEAP_MAX=1g/g" `find /var/www/YYYYY/* -name jvm.properties`

#start ------重啓

find /var/www/YYYYY/* -name restart.sh|xargs -i sh {}
sh /home/cxht/tomcat/bin/startup.sh
sh /var/www/YYYYY/XXXXX-event-maker/XXXXX-event-maker/bin/main.sh restart