linux統計cdn日誌慢請求

./stat_ip.sh live-https.log-0510.gz 1000

#首先用shell腳本能夠統計出？日誌慢請求查詢時間超過？秒對應的ip和對應的調用次數（傳兩個參數）

#!/bin/bash
log=$1
threshold=$2
function define()
{
ori_log_path="/home/bjliuzezhou/${log}"
tmp_log_path="/home/bjliuzezhou/temp.log"
tmp_log_path2="/home/bjliuzezhou/temp2.log"
confirm_path="/home/bjliuzezhou/previewlist.log"
}
function gather()
{
echo 'gather start-----------------------------------------------------------------'
zcat ${ori_log_path} | grep 'NewsApp'| grep 'previewlist' | awk '{print $(NF-3),$1}'> ${tmp_log_path}
log_num=`cat ${tmp_log_path} | wc -l`
request_time=`awk '{print $1}' ${tmp_log_path} | awk '{sum+=$1}END{print sum}'`
ave_request_time=`echo | awk "{print ${request_time}/${log_num}}" `
cat ${tmp_log_path} | awk -v th=${threshold} '$1>th {print $2}'> ${tmp_log_path2}
log_num2=`cat ${tmp_log_path2} | wc -l`
ratio=`echo | awk "{print ${log_num2}/${log_num}}" `
cat ${tmp_log_path2}| sort -n | uniq -c | sort -n > ${confirm_path}
# awk -f ip_cn.awk ${confirm_path}
rm -f /home/bjliuzezhou/temp.log
rm -f /home/bjliuzezhou/temp2.log
echo 'gather end--------------------------------------------------------------------'
}
function output()
{
echo "request total time is ${request_time}"
echo "request total number is ${log_num}"
echo "aver request time is ${ave_request_time}"
echo "long request total number is ${log_num2}"
echo "long request time ratio is ${ratio}"
}
function main()
{
define
gather
output
}
main

 

 

nohup ./ip_operator.sh preview.log &

#而後將ip後8位置爲0（緣由參照全國ip段劃分），統計出ip段以及對應的次數，而且解析出對應的運營商
#參數要傳剛獲取的preview.log，抓取運營商時間較長，慎重！而且須要注意一個坑，awk腳本中不能執行shell腳本！
#!/bin/bash
cut -d . -f 1,2,3 $1 > temp.log 
sed 's/$/&.0/g' temp.log | sort -n | uniq -c > temp2.log
awk '{system("sh getip.sh " $3)}' temp2.log > zzz.log
rm -rf temp.log

#!/bin/bash
echo | curl ip.cn?ip=$1

 

將最後解析出的temp2.log和zzz.log數據放在excel進行整合，分列，而後去重，開始快樂的統計之旅吧