shell編程系列21--文本處理三劍客之awk中數組的用法及模擬生產環境數據統計

時間 2019-12-07
標籤 shell 編程系列文本處理劍客 awk 數組用法模擬生產環境數據統計欄目 Unix 简体版
原文原文鏈接
shell編程系列21--文本處理三劍客之awk中數組的用法及模擬生產環境數據統計

shell中的數組的用法：
    shell數組中的下標是從0開始的

    array=("Allen" "Mike" "Messi" "Jerry" "Hanmeimei" "Wang")
    打印元素：        echo ${array[2]}
    打印元素個數:        echo ${#array[@]}
    打印某個元素長度:    echo ${#array[3]}
        
    給元素賦值：    array[3]=ui;
    刪除元素:    unset array[2];unset array # 刪除數組
    分片訪問:    echo ${array[@]:1:3}
    元素內容替換:    ${array[@]/e/E} 只替換第一個e；${array[@]//e/E} 替換全部的e

    數組的遍歷：
        for a in ${array[@]}
        do
            echo $a
        done

    awk中數組的用法：
        在awk中，使用數組時，不只能夠使用1.2..n做爲數組小標，也能夠使用字符串做爲數組下標

    典型經常使用例子：
        1、統計主機上全部的tcp連接狀態數，按照每一個tcp狀態分類
            # netstat -an | grep tcp | awk '{arr[$6]++}END{for (i in arr) print i,arr[i]}'
            LAST_ACK 40
            LISTEN 6
            SYN_RECV 105
            ESTABLISHED 3751
            FIN_WAIT1 33
            FIN_WAIT2 9
            CLOSING 3
            TIME_WAIT 2444

        2、計算橫向數據綜合，計算縱向數據總和

            Allen    80    90    96    98
            Mike    93    98    92    91
            Zhang    78    76    87    92
            Jerry    86    89    68    92
            Han    85    95    75    90
            Li    78    88    98    100

        # 代碼以下：
        [root@localhost shell]# awk -f statics.awk student.txt 
        Name                          Yuwen                         Math                          English                       Physical                      total                         
        Allen                         80                            90                            96                            98                            364                           
        Mike                          93                            98                            92                            91                            374                           
        Zhang                         78                            76                            87                            92                            333                           
        Jerry                         86                            89                            68                            92                            335                           
        Han                           85                            95                            75                            90                            345                           
        Li                            78                            88                            98                            100                           364                           
        every_total                   500                           536                           516                           563                           
        [root@localhost shell]# cat statics.awk 
        BEGIN{
            printf "%-30s%-30s%-30s%-30s%-30s%-30s\n","Name","Yuwen","Math","English","Physical","total"
        }
        {
            total=$2+$3+$4+$5
            yuwen_sum+=$2
            math_sum+=$3
            english_sum+=$4
            physical_sum+=$5
            printf "%-30s%-30d%-30d%-30d%-30d%-30d\n",$1,$2,$3,$4,$5,total
        }
        END{
            printf "%-30s%-30d%-30d%-30d%-30d\n","every_total",yuwen_sum,math_sum,english_sum,physical_sum
        }


計算字符串的長度：
[root@localhost shell]# str="test string"
[root@localhost shell]# echo $str
test string
[root@localhost shell]# echo ${#str}
11

# 修改數組元素
array=("Allen" "Mike" "Messi" "Jerry" "Hanmeimei" "Wang")
[root@localhost shell]# array[1]="Jerry"
[root@localhost shell]# echo ${array[@]}
Allen Jerry Messi Jerry Hanmeimei Wang

# 刪除第3個元素
[root@localhost shell]# echo ${array[@]}
Allen Jerry Messi Jerry Hanmeimei Wang
[root@localhost shell]# 
[root@localhost shell]# unset array[2];
[root@localhost shell]# echo ${array[@]}
Allen Jerry Jerry Hanmeimei Wang


# 在數組中刪除下標爲1的元素，即Mike被刪除，再次刪除下標爲1的元素，發現數組不變，說明數組雖然刪除了元素，下標仍是不變保存在內存中
[root@localhost shell]# array=("Allen" "Mike" "Messi" "Jerry" "Hanmeimei" "Wang")
[root@localhost shell]# unset array[1]
[root@localhost shell]# echo ${array[*]}
Allen Messi Jerry Hanmeimei Wang
[root@localhost shell]# unset array[1]
[root@localhost shell]# echo ${array[*]}
Allen Messi Jerry Hanmeimei Wang

# 分片訪問，數組爲1的開始遍歷3個元素
[root@localhost shell]# array=("Allen" "Mike" "Messi" "Jerry" "Hanmeimei" "Wang")
[root@localhost shell]# echo ${array[@]:1:3}
Mike Messi Jerry
# 1到最後
[root@localhost shell]# echo ${array[@]:1}
Mike Messi Jerry Hanmeimei Wang

#替換1個，替換全部
[root@localhost shell]# echo ${array[@]}
Allen Mike Messi Jerry Hanmeimei Wang
[root@localhost shell]# echo ${array[@]/e/E}
AllEn MikE MEssi JErry HanmEimei Wang
[root@localhost shell]# echo ${array[@]//e/E}
AllEn MikE MEssi JErry HanmEimEi Wang

# 遍歷數組
[root@localhost shell]# for a in ${array[@]};do echo $a;done
Allen
Mike
Messi
Jerry
Hanmeimei
Wang


    計算橫向和、縱向和

    Allen    80    90    96    98
    Mike    93    98    92    91
    Zhang    78    76    87    92
    Jerry    86    89    68    92
    Han    85    95    75    90
    Li    78    88    98    100

    [root@localhost shell]# awk -f stu.awk student.txt 
    Name                Yuwen               Math                English             Physical            total               
    Allen               80                  90                  96                  98                  364                 
    Mike                93                  98                  92                  91                  374                 
    Zhang               78                  76                  87                  92                  333                 
    Jerry               86                  89                  68                  92                  335                 
    Han                 85                  95                  75                  90                  345                 
    Li                  78                  88                  98                  100                 364                 
    every_total         500                 536                 516                 563                 
    [root@localhost shell]# cat stu.awk 
    BEGIN{
        printf "%-20s%-20s%-20s%-20s%-20s%-20s\n","Name","Yuwen","Math","English","Physical","total"
    }

    {    total=$2+$3+$4+$5
        yunwen_sum+=$2
        math_sum+=$3
        english_sum+=$4
        physical_sum+=$5
        printf "%-20s%-20d%-20d%-20d%-20d%-20d\n",$1,$2,$3,$4,$5,total
    }
    END{
        printf "%-20s%-20d%-20d%-20d%-20d\n","every_total",yunwen_sum,math_sum,english_sum,physical_sum
    }



# 模擬生產環境數據腳本
[root@localhost shell]# cat insert.sh 
#!/bin/bash
#

function create_random()
{
    min=$1
    max=$(($2-$min+1))
    num=$(date +%s%N)
    echo $(($num%$max+$min))
}

INDEX=1

while true
do
    for user in Allen Mike Jerry Tracy Hanmeimei Lilei
    do
        COUNT=$RANDOM
        NUM1=`create_random 1 $COUNT`
        NUM2=`expr $COUNT - $NUM1`
        echo "`date '+%Y-%m-%d %H:%M:%S'` $INDEX Batches: user:$user insert $COUNT records into datebase:product table:detail, insert $NUM1 records successfully,
        failed $NUM2 records" >> ./db.log.`date +%Y%m%d`
        INDEX=`expr $INDEX + 1`
    done
done


數據格式以下：
db.log.20190608

2019-06-08 10:31:40 15459 Batches: user Jerry insert 5504 records into datebase:product table:detail, insert 5253 records successfully,failed 251 records
2019-06-08 10:31:40 15460 Batches: user Tracy insert 25114 records into datebase:product table:detail, insert 13340 records successfully,failed 11774 records
2019-06-08 10:31:40 15461 Batches: user Hanmeimei insert 13840 records into datebase:product table:detail, insert 5108 records successfully,failed 8732 records
2019-06-08 10:31:40 15462 Batches: user Lilei insert 32691 records into datebase:product table:detail, insert 5780 records successfully,failed 26911 records
2019-06-08 10:31:40 15463 Batches: user Allen insert 25902 records into datebase:product table:detail, insert 14027 records successfully,failed 11875 records
...


1、統計每一個人分別插入了多少條record進數據庫
    輸出結果示例：
    Name    totalrecords
    allen    493082
    mike    349287

    [root@localhost shell]# awk -f exam1.awk  db.log.20190608 
    User                Total records       
    Jerry               42168344            
    Mike                42070170            
    Lilei               42628926            
    Hanmeimei           41141251            
    Tracy               42241015            
    Allen               42283107            
    [root@localhost shell]# cat exam1.awk 
    BEGIN{
        printf "%-20s%-20s\n","User","Total records"
    }

    {
        USER[$6]+=$8
    }

    END{
        for(u in USER)
            printf "%-20s%-20d\n",u,USER[u]
    }


2、統計每一個人分別插入成功了多少record，失敗了多少record

    輸出結果：
    User    Success_record    failed_record
    jerry    342222        2813413
    
    success $14
    failed    $17


    [root@localhost shell]# cat exam2.awk 
    BEGIN{
        printf "%-30s%-30s%-30s\n","User","Success records","Failed records"
    }

    {
        SUCCESS[$6]+=$14
        FAILED[$6]+=$17
    }

    END{
        for(u in SUCCESS)
            printf "%-30s%-30d%-30d\n",u,SUCCESS[u],FAILED[u]
    }
    [root@localhost shell]# awk -f exam2.awk db.log.20190608 
    User                          Success records               Failed records                
    Jerry                         20641524                      21526820                      
    Mike                          21119844                      20950326                      
    Lilei                         21441042                      21187884                      
    Hanmeimei                     20584580                      20556671                      
    Tracy                         20837585                      21403430                      
    Allen                         21024923                      21258184 

3、將例子1和例子2結合起來，一塊兒輸出，輸出每一個人分別插入多少條數據，多少成功，多少失敗，而且要格式化輸出，加上標題
    輸出結果：
    User    Total        success        failed
    tracy    6134139        34113444    1341
    allen    13387878    7897912        14319

    代碼：
    [root@localhost shell]# cat exam3.awk 
    BEGIN{
        printf "%-30s%-30s%-30s%-30s\n","Name","total records","success records","failed records"
    }

    {
        TOTAL_RECORDS[$6]+=$8
        SUCCESS[$6]+=$14
        FAILED[$6]+=$17
    }

    END{
        for(u in TOTAL_RECORDS)
            printf "%-30s%-30d%-30d%-30d\n",u,TOTAL_RECORDS[u],SUCCESS[u],FAILED[u]
    }
    [root@localhost shell]# awk -f exam3.awk db.log.20190608 
    Name                          total records                 success records               failed records                
    Jerry                         42168344                      20641524                      21526820                      
    Mike                          42070170                      21119844                      20950326                      
    Lilei                         42628926                      21441042                      21187884                      
    Hanmeimei                     41141251                      20584580                      20556671                      
    Tracy                         42241015                      20837585                      21403430                      
    Allen                         42283107                      21024923                      21258184 

4、在例子3的基礎上，加上結尾，統計所有插入記錄數，成功記錄數，失敗記錄數
    輸出結果：
    User    Total        success        failed
    tracy    6134139        34113444    1341
    allen    13387878    7897912        14319
        1413413        3141311341    13411

    方法1：
    [root@localhost shell]# cat exam4_b.awk 
    BEGIN{
        printf "%-30s%-30s%-30s%-30s\n","Name","total records","success records","failed records"
    }

    {
        TOTAL_RECORDS[$6]+=$8
        SUCCESS[$6]+=$14
        FAILED[$6]+=$17
    }

    END{
        for(u in TOTAL_RECORDS)
        {
            # 在統計出的結果數組中進行累加
            records_sum+=TOTAL_RECORDS[u]
            success_sum+=SUCCESS[u]
            failed_sum+=FAILED[u]
            printf "%-30s%-30d%-30d%-30d\n",u,TOTAL_RECORDS[u],SUCCESS[u],FAILED[u]
        }

        printf "%-30s%-30d%-30d%-30d\n","",records_sum,success_sum,failed_sum
    }
    [root@localhost shell]# awk -f exam4_b.awk db.log.20190608 
    Name                          total records                 success records               failed records                
    Jerry                         42168344                      20641524                      21526820                      
    Mike                          42070170                      21119844                      20950326                      
    Lilei                         42628926                      21441042                      21187884                      
    Hanmeimei                     41141251                      20584580                      20556671                      
    Tracy                         42241015                      20837585                      21403430                      
    Allen                         42283107                      21024923                      21258184                      
                      252532813                     125649498                     126883315     
    方法2：
    [root@localhost shell]# cat exam4.awk 
    BEGIN{
        printf "%-30s%-30s%-30s%-30s\n","Name","total records","success records","failed records"
    }

    {
        RECORDS[$6]+=$8
        SUCCESS[$6]+=$14
        FAILED[$6]+=$17
        
        # 在原始數據中進行彙總計算
        records_sum+=$8
        success_sum+=$14
        failed_sum+=$17    
    }

    END{
        for(u in RECORDS)
            printf "%-30s%-30d%-30d%-30d\n",u,RECORDS[u],SUCCESS[u],FAILED[u]

        printf "%-30s%-30d%-30d%-30d\n","total",records_sum,success_sum,failed_sum
    }
    [root@localhost shell]# awk -f exam4.awk db.log.20190608 
    Name                          total records                 success records               failed records                
    Jerry                         42168344                      20641524                      21526820                      
    Mike                          42070170                      21119844                      20950326                      
    Lilei                         42628926                      21441042                      21187884                      
    Hanmeimei                     41141251                      20584580                      20556671                      
    Tracy                         42241015                      20837585                      21403430                      
    Allen                         42283107                      21024923                      21258184                      
    total                         252532813                     125649498                     126883315 

5、查找丟失數據的現象，也就是成功+失敗的記錄數不等於一共插入的記錄數，找出這些數據並顯示行號和對應行的日誌信息
    輸出結果：
        
    代碼：
        [root@localhost shell]# awk '{if($8!=$14+$17) print NR,$0}' db.log.20190608 
        5 2019-06-08 10:29:59 5 Batches: user Hanmeimei insert 15688 records into datebase:product table:detail, insert 3986 records successfully,failed 1102 records
        15404 2019-06-08 10:31:39 15404 Batches: user Mike insert 26675 records into datebase:product table:detail, insert 1931 records successfully,failed 24725 records
    
    # 寫入文件的方式
    [root@localhost shell]# awk -f exam5.awk db.log.20190608 
    5 2019-06-08 10:29:59 5 Batches: user Hanmeimei insert 15688 records into datebase:product table:detail, insert 3986 records successfully,failed 1102 records
    15404 2019-06-08 10:31:39 15404 Batches: user Mike insert 26675 records into datebase:product table:detail, insert 1931 records successfully,failed 24725 records
    [root@localhost shell]# cat exam5.awk 
    BEGIN{
    }
    {
        if($8!=$14+$17) 
        print NR,$0
    }