MySQL和hive對比表結構腳本

 1 #!/bin/bash
 2 
 3 source /etc/profile
 4 
 5 runlog='/tmp/zewei/check_schema_log'
 6 hive_database_schema=/tmp/hive_database_schema/hive/
 7 mysql_database_schema=/tmp/hive_database_schema/mysql/
 8 
 9 > $runlog
10 #天天下午一點刪除現有的表結構緩存文件.
11 #防止hive有變更.
12 if [ `date +%k` -eq 13 ] 
13 then
14     rm -rf /tmp/hive_database_schema/hive/*
15 fi
16 
17 while read DB; do
18     HiveDB=`echo $DB |awk '{print $1}'`
19     MysqlDB=`echo $DB |awk '{print $2}'`
20     MysqlHost=`echo $DB |awk '{print $3}'`
21     MysqlPort=`echo $DB |awk '{print $4}'`
22     PartTag=`echo $DB |awk '{print $5}'`
23 
24     connect_mysql="mysql -u TableSchemaCheck -pSchemacheck666 -h $MysqlHost -P $MysqlPort $MysqlDB -BNe"
25     #緩存文件夾不在就建立
26     ls $mysql_database_schema/$MysqlDB > /dev/null || mkdir -p $mysql_database_schema/$MysqlDB
27     ls $hive_database_schema/$HiveDB > /dev/null || mkdir -p $hive_database_schema/$HiveDB
28     
29     #經過part標記檢查是否爲分區庫
30     if [ "$PartTag" == 'part' ]
31     then
32         table_list=`hive -S -e "use $HiveDB; show tables;" | grep "_part$"`
33     else
34         table_list=`hive -S -e "use $HiveDB; show tables;"`
35     fi
36 
37     #對獲取到的tables進行循環檢查
38     for table in $table_list
39     do
40         if [ "$PartTag" == 'part' ]
41         then
42             hive_table_name=$table
43             mysql_table_name=`echo $table | sed 's/_part$//'`
44         else
45             hive_table_name=$table
46             mysql_table_name=$table
47         fi
48 
49         #獲取mysql的表結構
50         $connect_mysql "desc $mysql_table_name;"  > /tmp/mysql_column 
51         if [ $? -ne 0 ] 
52         then 
53             continue
54         else
55             #把大寫轉換爲小寫.由於hive裏面的列名沒有大小寫之分,因此轉換一下
56             cat /tmp/mysql_column |  awk '{print $1}' | tr '[A-Z]' '[a-z]' > $mysql_database_schema/$MysqlDB/$mysql_table_name
57         fi
58 
59         #若是沒有hive的表結構緩存文件就去hive取...而後對分區字段進行刪除,由於這對於MySQL的表結構來講是多餘的
60         if ! ls $hive_database_schema/$HiveDB/$table > /dev/null
61         then
62             hive -S -e "use $HiveDB; desc ${hive_table_name};" | awk '{print $1 }' > /tmp/hive_column
63             part_column=`sed -n  '/\#/,$p' /tmp/hive_column | egrep -v '#|^$'` 
64             for i in $part_column
65             do
66                 sed -i "/\b${i}\b/d" /tmp/hive_column 
67             done
68             egrep -v "#|^$" /tmp/hive_column | tee $hive_database_schema/$HiveDB/$table
69         fi
70 
71         #獲取MD5
72         md5ForMysql=`md5sum $mysql_database_schema/$MysqlDB/$mysql_table_name | awk '{print $1}'`
73         md5ForHive=`md5sum $hive_database_schema/$HiveDB/$hive_table_name | awk '{print $1}'`
74 
75         #MD5不一樣就記錄日誌
76         if [ ! "$md5ForMysql"x == "$md5ForHive"x ]; then
77                 echo -e "HiveDB:\t$HiveDB\t\ttable:\t$hive_table_name\t\tnot equal to MysqlDB table:\t$mysql_table_name" >> $runlog
78         fi
79     done
80 done < /root/script/DBlist
81 
82 #統計並報警.報警閾值在監控裏面設置
83 err_line=`wc -l $runlog | awk '{print $1}'`
84 [報警]