分佈式監控系統開發【day38】:主機存活檢測程序解析（七）

1、目錄結構

 

2、入口

一、文件MonitorServer.py

import os
import sys

if __name__ == "__main__":
    os.environ.setdefault("DJANGO_SETTINGS_MODULE", "CrazyMonitor.settings")

    from monitor.backends.management import execute_from_command_line

    execute_from_command_line(sys.argv)

二、啓動函數management.py

    def start(self):
        '''start monitor server frontend and backend'''
        reactor = data_processing.Datandler(settings)
        reactor.looping()

3、檢測主機須要監控的服務數據是否按時正常彙報

一、功能以下

一、生成全局的監控配置dict

二、循環全部要監控的服務

三、檢測此服務最近的彙報數據

四、檢測 有沒有這個機器的trigger,若是沒有,把機器狀態改爲ok

二、實現代碼

    def looping(self):
        '''
        start looping data ...
        檢測全部主機須要監控的服務的數據有沒有按時彙報上來，只作基本檢測
        :return:
        '''
        #get latest report data
        self.update_or_load_configs() #生成全局的監控配置dict
        count = 0
        while not self.exit_flag:
            print("looping %s".center(50,'-') % count)
            count += 1
            if time.time() - self.config_last_loading_time >= self.config_update_interval:
                print("\033[41;1mneed update configs ...\033[0m")
                self.update_or_load_configs()
                print("monitor dic",self.global_monitor_dic)
            if self.global_monitor_dic:
                for h,config_dic in self.global_monitor_dic.items():
                    print('handling host:\033[32;1m%s\033[0m' %h)
                    for service_id,val in config_dic['services'].items(): #循環全部要監控的服務
                        #print(service_id,val)
                        service_obj,last_monitor_time = val
                        if time.time() - last_monitor_time >= service_obj.interval: #reached the next monitor interval
                            print("\033[33;1mserivce [%s] has reached the monitor interval...\033[0m" % service_obj.name)
                            self.global_monitor_dic[h]['services'][service_obj.id][1] = time.time()
                            #self.load_service_data_and_calulating(h,service_obj)
                            #only do basic data validataion here, alert if the client didn't report data to server in \
                            #the configured time interval
                            self.data_point_validation(h,service_obj) #檢測此服務最近的彙報數據
                        else:
                            next_monitor_time = time.time() - last_monitor_time - service_obj.interval
                            print("service [%s] next monitor time is %s" % (service_obj.name,next_monitor_time))

                    if time.time() - self.global_monitor_dic[h]['status_last_check'] >10:
                        #檢測 有沒有這個機器的trigger,若是沒有,把機器狀態改爲ok
                        trigger_redis_key = "host_%s_trigger*" % (h.id)
                        trigger_keys = self.redis.keys(trigger_redis_key)
                        #print('len grigger keys....',trigger_keys)
                        if len(trigger_keys) ==0: #沒有trigger被觸發,能夠把狀態改成ok了
                            h.status = 1
                            h.save()
                    #looping triggers 這裏是真正根據用戶的配置來監控了
                    #for trigger_id,trigger_obj in config_dic['triggers'].items():
                    #    #print("triggers expressions:",trigger_obj.triggerexpression_set.select_related())
                    #    self.load_service_data_and_calulating(h,trigger_obj)

            time.sleep(self.poll_interval)

4、監控主機是否存活

一、功能以下

一、拼出此服務在redis中存儲的對應key

二、超過監控間隔但數據還沒彙報過來

三、監控主機是否存活

四、要是主機死掉了我就在數據庫你註釋有問題

五、客戶端起來歷來每連上過沒因此沒有數據數據

二、實現代碼

    def data_point_validation(self,host_obj,service_obj):
        '''
        only do basic data validation here, alert if the client didn't report data to server in the configured time interval
        :param h:
        :param service_obj:
        :return:
        '''
        service_redis_key = "StatusData_%s_%s_latest" %(host_obj.id,service_obj.name) #拼出此服務在redis中存儲的對應key
        latest_data_point = self.redis.lrange(service_redis_key,-1,-1)
        if latest_data_point: #data list is not empty,
            latest_data_point = json.loads(latest_data_point[0].decode())
            #print('laste::::',latest_data_point)
            print("\033[41;1mlatest data point\033[0m %s" % latest_data_point)
            latest_service_data,last_report_time = latest_data_point
            monitor_interval = service_obj.interval + self.django_settings.REPORT_LATE_TOLERANCE_TIME
            if time.time() - last_report_time > monitor_interval: #超過監控間隔但數據還沒彙報過來,something wrong with client
                no_data_secs =  time.time() - last_report_time
                msg = '''Some thing must be wrong with client [%s] , because haven't receive data of service [%s] \
                for [%s]s (interval is [%s])\033[0m''' %(host_obj.ip_addr, service_obj.name,no_data_secs, monitor_interval)
                self.trigger_notifier(host_obj=host_obj,trigger_id=None,positive_expressions=None,
                                      msg=msg)
                print("\033[41;1m%s\033[0m" %msg )
                if service_obj.name == 'uptime': #監控主機存活的服務
                    host_obj.status = 3 #unreachable
                    host_obj.save()
                else:
                    host_obj.status = 5 #problem
                    host_obj.save()

        else: # no data at all
            print("\033[41;1m no data for serivce [%s] host[%s] at all..\033[0m" %(service_obj.name,host_obj.name))
            msg = '''no data for serivce [%s] host[%s] at all..''' %(service_obj.name,host_obj.name)
            self.trigger_notifier(host_obj=host_obj,trigger_id=None,positive_expressions=None,msg=msg)
            host_obj.status = 5 #problem
            host_obj.save()
        #print("triggers:", self.global_monitor_dic[host_obj]['triggers'])

5、存活檢測監控項間隔設計

一、實例化

一、循環檢測每臺主機多久沒有給我彙報數據
二、監控配置有可能變動
三、檢測每一個主機每一個服務
四、主機是好的服務全掛了

class DataHandler(object):
    def __init__(self,django_settings,connect_redis=True):
        self.django_settings = django_settings
        self.poll_interval = 3 #每3秒進行一次全局輪訓
        self.config_update_interval = 120 #每120s從新從數據庫加載一次配置數據
        self.config_last_loading_time = time.time()
        self.global_monitor_dic = {}
        self.exit_flag = False
        if connect_redis:
            self.redis = redis_conn.redis_conn(django_settings

二、方法

一、要是主機死掉了我就在數據庫你註釋有問題
二、客戶端起來歷來每連上過沒因此沒有數據數據

    def update_or_load_configs(self):
        '''
        load monitor configs from Mysql DB
        :return:
        '''
        all_enabled_hosts = models.Host.objects.all()
        for h in all_enabled_hosts:
            if h not in self.global_monitor_dic: # new host
                self.global_monitor_dic[h] = {'services':{}, 'triggers':{}}
                '''self.global_monitor_dic ={
                    'h1':{'services'{'cpu':[cpu_obj,0],
                                     'mem':[mem_obj,0]
                                     },
                          'trigger':{t1:t1_obj,}
                        }
                }'''
            #print(h.host_groups.select_related())
            service_list = []
            trigger_list = []
            for group in h.host_groups.select_related():
                #print("grouptemplates:", group.templates.select_related())

                for template in  group.templates.select_related():
                    #print("tempalte:",template.services.select_related())
                    #print("triigers:",template.triggers.select_related())
                    service_list.extend(template.services.select_related())
                    trigger_list.extend(template.triggers.select_related())
                for service in service_list:
                    if service.id not in self.global_monitor_dic[h]['services']: #first loop
                        self.global_monitor_dic[h]['services'][service.id] = [service,0]
                    else:
                        self.global_monitor_dic[h]['services'][service.id][0] = service
                for trigger in trigger_list:
                    #if not self.global_monitor_dic['triggers'][trigger.id]:
                    self.global_monitor_dic[h]['triggers'][trigger.id] = trigger

            #print(h.templates.select_related() )
            #print('service list:',service_list)

            for template in  h.templates.select_related():
                service_list.extend(template.services.select_related())
                trigger_list.extend(template.triggers.select_related())
            for service in service_list:
                if service.id not in self.global_monitor_dic[h]['services']: #first loop
                    self.global_monitor_dic[h]['services'][service.id] = [service,0]
                else:
                    self.global_monitor_dic[h]['services'][service.id][0] = service
            for trigger in trigger_list:
                self.global_monitor_dic[h]['triggers'][trigger.id] = trigger
            #print(self.global_monitor_dic[h])
            #經過這個時間來肯定是否須要更新主機狀態
            self.global_monitor_dic[h].setdefault('status_last_check',time.time())

        self.config_last_loading_time = time.time()
        return True

三、網絡延遲容忍度設置

一、網絡延遲應該能夠自定義容忍度30秒
二、客戶端每30秒給我彙報一次我還活着

REPORT_LATE_TOLERANCE_TIME = 10 #allow service report late than monitor interval no more than defined seconds.