shell腳本分析nginx日誌

注意：複製該腳本到linux系統中使用時，須要先安裝dos2unix工具進行格式轉換

yum install dos2unix -y

 

安裝成功後，使用該命令轉換windows的換行符

dos2unix a.sh

 

正在寫入的文件不能用tar進行壓縮

--------壓縮日誌----------------------
94 access.log
95 tar: access.log: file changed as we read it
96 #### 壓縮日誌失敗 ####

#!/bin/sh #分析nginx日誌 DATE=`date '+%Y%m%d-%H%M'` ARCHIVE=/usr/log_bak/nginx_$DATE.tar.gz MESSAGE=/usr/log_bak/"Nginx_Analysis""$DATE" FILENAME=/data/nginx/logs/access.log BACKNAME=/usr/log_bak/"nginx_bak" function Mail(){ mail -s "***Nginx Report***" 666@qq.com < $MESSAGE } function Bowser(){ key[0]='" 200 [0-9]{3}';word[0]='http 200' key[1]='" 206 [0-9]{3}';word[1]='http 206' key[2]='" 404 [0-9]{3}';word[2]='http 404' key[3]='" 503 [0-9]{3}';word[3]='http 503' ########## # seo/seo.html" target="_blank">
key[4]='Googlebot.*google.com/bot.html';word[4]='Google Browser' key[5]='Baiduspider.*baidu.com/search/spider.html';word[5]='Baidu Browser' key[6]='bingbot.*bing.com/bingbot.htm';word[6]='Bing Browser' #Soso 'Sosospider.*soso.com/webspider.htm' #ÓеÀ 'YoudaoBot.*youdao.com/help/webmaster/spider/' #YahooÖйú 'Yahoo! Slurp China' ########## # key[7]='MSIE';word[7]='MSIE' key[8]='Gecko/.*Firefox';word[8]='Firefox' key[9]='AppleWebKit.*like Gecko';word[9]='Webkit' key[10]='Opera.*Presto';word[10]='Opera' key[11]='Windows NT 6.1';word[11]='Windows 7 訪問' key[12]='Macintosh; Intel Mac OS X';word[12]='Mac OS X 訪問' key[13]='X11.*Linux';word[13]='Linux with X11' key[14]='Android;';word[14]='Android' #WindowsϵÁÐ win2000'Windows NT 5.0' winxp'Windows NT 5.1' winvasta'Windows NT 6.0' win7'Windows NT 6.1
#SymbianOS 'SymbianOS' ########## # key[15]='iPad.*like Mac OS X';word[15]='iPad 訪問' key[16]='Nokia';word[16]='Nokia' key[17]='Nokia5800';word[17]='Nokia5800 XpressMusic' #iPhone 'iPhone.*like Mac OS X' ########## # key[18]='GET /.*.mp3 HTTP';word[18]="訪問 mp3 file" key[19]='GET /.*.jpg HTTP';word[19]="訪問 jpg file" #echo $filename #echo "nginx日誌: ${FILENAME},一共${totle}行,須要處理 ${#key[@]}條" >> $MESSAGE #echo "來源IP$(cat $FILENAME | awk '{print $1}' |sort|uniq|wc -l)" >> $MESSAGE i=4 echo "----瀏覽器來源----" >> $MESSAGE echo "--瀏覽器-----總計------佔比--" >> $MESSAGE while [ $i -lt ${#key[@]} ] do s1=${word[$i]} s2=$(cat $BACKNAME | grep ''"${key[$i]}"'' | wc -l) s3=$(awk 'BEGIN{printf "%.2f%",('$s2'/'$totle')*100}') echo "${s1} ${s2} ${s3}" >> $MESSAGE ((i++)) done if [[ $? == 0 ]]; then echo "分析瀏覽器標示成功" >> $MESSAGE else echo "分析瀏覽器標示失敗" >> $MESSAGE fi echo "--------------------" >> $MESSAGE } Check_http_status() { #grep -ioE "HTTP\/1\.[1|0]\"[[:blank:]][0-9]{3}" access.log #拿到日誌中全部的包含HTTP狀態碼的部分，拿出第二段來判斷，並將結果分配到數組中 codes=(`grep -ioE "HTTP\/1\.[1|0]\"[[:blank:]][0-9]{3}" $BACKNAME | awk -F"[ ]+"    'BEGIN{i=0;j=0;k=0;n=0;p=0;}{ if($2>=100&&$2<200) 
                        {i++} else if($2>=200&&$2<300) {j++} else if($2>=300&&$2<400) {k++} else if($2>=400&&$2<500) {n++} else if($2>=500) {p++} }END{ print i?i:0,j?j:0,k?k:0,n?n:0,p?p:0,i+j+k+n+p }'`) 
echo "--HTTP狀態碼---COUNT---PERCENT------" >> $MESSAGE echo "status[100+]:--${codes[0]}--$(awk 'BEGIN{printf "%.2f%",('${codes[0]}'/'${codes[5]}')*100}')" >> $MESSAGE echo "status[200+]:--${codes[1]}--$(awk 'BEGIN{printf "%.2f%",('${codes[1]}'/'${codes[5]}')*100}')" >> $MESSAGE echo "status[300+]:--${codes[2]}--$(awk 'BEGIN{printf "%.2f%",('${codes[2]}'/'${codes[5]}')*100}')" >> $MESSAGE echo "status[400+]:--${codes[3]}--$(awk 'BEGIN{printf "%.2f%",('${codes[3]}'/'${codes[5]}')*100}')" >> $MESSAGE echo "status[500+]:--${codes[4]}--$(awk 'BEGIN{printf "%.2f%",('${codes[4]}'/'${codes[5]}')*100}')" >> $MESSAGE echo "----全部的狀態碼: ${codes[5]}----" >> $MESSAGE } function IpUrlTime(){ echo "來源IP共--$(cat $BACKNAME | awk '{print $1}' |sort|uniq|wc -l)--個" >> $MESSAGE ip=$(cat $BACKNAME | awk '{print $1}'|sort | uniq -c | sort -nr | head -n 20) echo "----訪問前20個IP統計----" >> $MESSAGE echo "$ip" >> $MESSAGE #經過日誌查看當天訪問頁面的url: url=$(cat $BACKNAME | awk '{print $7}'|sort | uniq -c | sort -nr |head -n 20) echo "----訪問前20個URL統計----" >> $MESSAGE echo "$url" >> $MESSAGE #經過日誌查看當天訪問次數最多的時間段 time=$(awk '{print $4}' $BACKNAME  |cut -c 14-18 | sort | uniq -c | sort -nr | head | head -n 20) echo "----訪問前20個時間點統計----" >> $MESSAGE echo "$time" >> $MESSAGE } #----------start--------------- ip=`ifconfig | grep 'inet addr:'|grep -v '127.0.0.1'|awk -F '[ :]+' '{print $4}'` echo "--------Server $ip---------------" >> $MESSAGE echo "--------$(df -h)---------------" >> $MESSAGE cd /usr/log_bak if [ $? == 0 ] then echo "進入目錄/usr/log_bak" >> $MESSAGE else echo "####進入目錄失敗,退出####" >> $MESSAGE exit 0 fi echo "---------------------" >> $MESSAGE echo "備份日誌:" $(date +"%y-%m-%d %H:%M:%S") >> $MESSAGE echo "---------------------" >> $MESSAGE #bak access.log cp $FILENAME $BACKNAME #check bak if [[ $? == 0 ]] then echo "日誌複製成功" >> $MESSAGE else echo "####日誌複製失敗,退出####" >> $MESSAGE exit 0 fi echo "-------------------------------" >> $MESSAGE echo "分析時間:" $(date +"%y-%m-%d %H:%M:%S") >> $MESSAGE echo "-------------------------------" >> $MESSAGE totle=$(cat $BACKNAME | wc -l) size=$(ls -sh $BACKNAME | awk '{print $1}') echo "nginx日誌,${size},一共${totle}行 " >> $MESSAGE IpUrlTime Check_http_status Bowser echo "--------壓縮日誌----------------------" >> $MESSAGE  #直接備份複製的文件，否者access.log1正在寫入，沒法壓縮 tar czvf $ARCHIVE nginx_bak >> $MESSAGE 2>&1 #判斷catalina.out備份是否成功 if [[ $? == 0 ]] then #建立備份文件的壓縮包 # tar czvf $ARCHIVE $LOG >> log.txt 2>&1 echo "[$ARCHIVE] 日誌壓縮成功!" >> $MESSAGE # clear access.log > $FILENAME if [[ $? == 0 ]] then echo "清空日誌清空日誌成功" >> $MESSAGE rm -f $BACKNAME else echo "###清空日誌失敗 Failed #####" >> $MESSAGE fi #只需保留備份文件的壓縮包便可 else echo "#### 壓縮日誌失敗 ####" >> $MESSAGE exit 0 fi echo "---------------------" >> $MESSAGE echo "結束時間:" $(date +"%y-%m-%d %H:%M:%S") >> $MESSAGE echo "---------------------" >> $MESSAGE Mail