背景 :在公司機器都放在監控當中,分不一樣的業務線,每一個業務線的機器資源利用率也不同,可是從運維角度出發,咱們要資源利用率最大化,減小資源成本,可是每一個業務線去找人比較麻煩,咱們能夠經過zabbix去獲取全部機器的信息,這邊能夠分析,吧機器資源整合。python
腳本在連接:
連接:https://pan.baidu.com/s/1_6UoDCJlO1kQN_KEvo7NZg
提取碼:53ermysql
「」「
#!/usr/local/python/bin/python3.5
import pymysql
import time, datetime
import smtplib
import os
from email.mime.text import MIMEText
from email.header import Headersql
zdbhost = '10.20.2.21'
zdbuser = 'zabbix'
zdbpass = 'zabbix'
zdbport = 3306
zdbname = 'zabbix'數據庫
d = datetime.datetime.now()
day = datetime.date.today()
keys = {
'trends_uint': [
'net.if.in[eth0]',
'net.if.out[eth0]',
'vfs.fs.size[/,used]',
'vm.memory.size[available]',
],
'trends': [
'system.cpu.load[percpu,avg5]',
'system.cpu.util[,idle]',
],
}
class ReportForm:json
def __init__(self): self.conn = pymysql.connect(host=zdbhost, user=zdbuser, passwd=zdbpass, port=zdbport, db=zdbname) self.cursor = self.conn.cursor() self.groupname = 'K8s機器監控' #這裏分組必定要修改不應查不到 self.IpInfoList = self.__getHostList() # return self.IpInfoList def __getHostList(self): sql = '''select groupid from groups where name = '%s' ''' % self.groupname self.cursor.execute(sql) groupid = self.cursor.fetchone()[0] print(groupid) sql = '''select hostid from hosts_groups where groupid = %s''' % groupid self.cursor.execute(sql) hostlist = self.cursor.fetchall() IpInfoList = {} for i in hostlist: hostid = i[0] sql = '''select host from hosts where status = 0 and hostid = %s''' % hostid ret = self.cursor.execute(sql) if ret: IpInfoList[self.cursor.fetchone()[0]] = {'hostid': hostid} return IpInfoList def __getItemid(self, hostid, itemname): sql = '''select itemid from items where hostid = %s and key_ = '%s' ''' % (hostid, itemname) if self.cursor.execute(sql): itemid = self.cursor.fetchone()[0] else: itemid = None return itemid def getTrendsValue(self, itemid, start_time, stop_time): resultlist = {} for type in ['min', 'max', 'avg']: sql = '''select %s(value_%s) as result from trends where itemid = %s and clock >= %s and clock <= %s''' % (type, type, itemid, start_time, stop_time) self.cursor.execute(sql) result = self.cursor.fetchone()[0] if result == None: result = 0 resultlist[type] = result return resultlist def getTrends_uintValue(self, itemid, start_time, stop_time): resultlist = {} for type in ['min', 'max', 'avg']: sql = '''select %s(value_%s) as result from trends_uint where itemid = %s and clock >= %s and clock <= %s''' % (type, type, itemid, start_time, stop_time) self.cursor.execute(sql) result = self.cursor.fetchone()[0] if result: resultlist[type] = int(result) else: resultlist[type] = 0 return resultlist def get_week(self, d): dayscount = datetime.timedelta(days=d.isoweekday()) dayto = d - dayscount sixdays = datetime.timedelta(days=6) dayfrom = dayto - sixdays date_from = datetime.datetime(dayfrom.year, dayfrom.month, dayfrom.day, 0, 0, 0) date_to = datetime.datetime(dayto.year, dayto.month, dayto.day, 23, 59, 59) ts_first = int(time.mktime(datetime.datetime(dayfrom.year, dayfrom.month, dayfrom.day, 0, 0, 0).timetuple())) ts_last = int(time.mktime(datetime.datetime(dayto.year, dayto.month, dayto.day, 23, 59, 59).timetuple())) return ts_first, ts_last def getLastMonthData(self, hostid, table, itemname): ts_first = self.get_week(d)[0] ts_last = self.get_week(d)[1] itemid = self.__getItemid(hostid, itemname) # function = getattr(self, 'get %s Value' % table.capitalize()) function = getattr(self, 'get%sValue' % table.capitalize()) return function(itemid, ts_first, ts_last) def getinfo(self): for ip, resultdict in zabbix.IpInfoList.items(): print("正在查詢 IP:%-15s hostid:%5d 的信息!" % (ip, resultdict['hostid'])) for table, keylists in keys.items(): for key in keylists: print("\t正在統計 key_:%s" % key) data = zabbix.getLastMonthData(resultdict['hostid'], table, key) zabbix.IpInfoList[ip][key] = data def writeToXls(self): dayscount = datetime.timedelta(days=d.isoweekday()) dayto = d - dayscount sixdays = datetime.timedelta(days=6) dayfrom = dayto - sixdays date_from = datetime.date(dayfrom.year, dayfrom.month, dayfrom.day) date_to = datetime.date(dayto.year, dayto.month, dayto.day) '''生成xls文件''' try: import xlsxwriter # 建立文件 workbook = xlsxwriter.Workbook('/usr/monitor/week/%s_%s巡檢報告.xlsx' % (date_from, date_to)) # 建立工做薄 worksheet = workbook.add_worksheet() # 寫入標題(第一行) i = 0 for value in ["主機", "CPU平均空閒值", "CPU最小空閒值", "可用平均內存(單位M)", "可用最小內存(單位M)", "CPU5分鐘負載", "進入最大流量(單位Kbps)", "進入平均流量(單位Kbps)", "出去最大流量(單位Kbps)", "出去平均流量(單位Kbps)"]: worksheet.write(0, i, value) i = i + 1 # 寫入內容: j = 1 for ip, value in self.IpInfoList.items(): worksheet.write(j, 0, ip) worksheet.write(j, 1, '%.2f' % value['system.cpu.util[,idle]']['avg']) worksheet.write(j, 2, '%.2f' % value['system.cpu.util[,idle]']['min']) worksheet.write(j, 3, '%dM' % int(value['vm.memory.size[available]']['avg'] / 1024 / 1024)) worksheet.write(j, 4, '%dM' % int(value['vm.memory.size[available]']['min'] / 1024 / 1024)) worksheet.write(j, 5, '%.2f' % value['system.cpu.load[percpu,avg5]']['avg']) worksheet.write(j, 6, value['net.if.in[eth0]']['max'] / 1000) worksheet.write(j, 7, value['net.if.in[eth0]']['avg'] / 1000) worksheet.write(j, 8, value['net.if.out[eth0]']['max'] / 1000) worksheet.write(j, 9, value['net.if.out[eth0]']['avg'] / 1000) j = j + 1 workbook.close() except Exception as e: print(e) def __del__(self): '''關閉數據庫鏈接''' self.cursor.close() self.conn.close() def sendmail(self): sender = 'lxftty@163.com' receiver = ['lxftty@163.com', '602497030@qq.com'] subject = '上週巡檢報告' smtpserver = 'smtp.163.com' username = 'lxftty@163.com' password = '123asdPP' msg = MIMEText('上週巡檢報告已生成,請檢查。\n報告目錄:/usr/monitor\n因爲郵件沒法發送excel文檔,請使用ftp登陸' '查看。\nftp地址:xxx:端口\nftp用戶名:xxx\n', 'plain', 'utf-8') # 中文需參數‘utf-8',單字節字符不須要 msg['Subject'] = Header(subject, 'utf-8') msg['From'] = 'lxftty@163.com' msg['To'] = "602497030@qq.com" smtp = smtplib.SMTP() smtp.connect('smtp.163.com') smtp.login(username, password) smtp.sendmail(sender, receiver, msg.as_string()) print('發送成功!') smtp.quit()
if name == "main":
zabbix = ReportForm()
zabbix.getinfo()
zabbix.writeToXls()
zabbix.sendmail()
[root@zabbix-server ~]# vim liyonglv.py
[root@zabbix-server ~]# cat liyonglv.py
#!/usr/local/python/bin/python3.5
import pymysql
import time, datetime
import smtplib
import os
from email.mime.text import MIMEText
from email.header import Headervim
zdbhost = '10.20.2.21'
zdbuser = 'zabbix'
zdbpass = 'zabbix'
zdbport = 3306
zdbname = 'zabbix'api
d = datetime.datetime.now()
day = datetime.date.today()
keys = {
'trends_uint': [
'net.if.in[eth0]',
'net.if.out[eth0]',
'vfs.fs.size[/,used]',
'vm.memory.size[available]',
],
'trends': [
'system.cpu.load[percpu,avg5]',
'system.cpu.util[,idle]',
],
}
class ReportForm:數組
def __init__(self): self.conn = pymysql.connect(host=zdbhost, user=zdbuser, passwd=zdbpass, port=zdbport, db=zdbname) self.cursor = self.conn.cursor() self.groupname = 'K8s機器監控' #這裏分組必定要修改不應查不到 self.IpInfoList = self.__getHostList() # return self.IpInfoList def __getHostList(self): sql = '''select groupid from groups where name = '%s' ''' % self.groupname self.cursor.execute(sql) groupid = self.cursor.fetchone()[0] print(groupid) sql = '''select hostid from hosts_groups where groupid = %s''' % groupid self.cursor.execute(sql) hostlist = self.cursor.fetchall() IpInfoList = {} for i in hostlist: hostid = i[0] sql = '''select host from hosts where status = 0 and hostid = %s''' % hostid ret = self.cursor.execute(sql) if ret: IpInfoList[self.cursor.fetchone()[0]] = {'hostid': hostid} return IpInfoList def __getItemid(self, hostid, itemname): sql = '''select itemid from items where hostid = %s and key_ = '%s' ''' % (hostid, itemname) if self.cursor.execute(sql): itemid = self.cursor.fetchone()[0] else: itemid = None return itemid def getTrendsValue(self, itemid, start_time, stop_time): resultlist = {} for type in ['min', 'max', 'avg']: sql = '''select %s(value_%s) as result from trends where itemid = %s and clock >= %s and clock <= %s''' % (type, type, itemid, start_time, stop_time) self.cursor.execute(sql) result = self.cursor.fetchone()[0] if result == None: result = 0 resultlist[type] = result return resultlist def getTrends_uintValue(self, itemid, start_time, stop_time): resultlist = {} for type in ['min', 'max', 'avg']: sql = '''select %s(value_%s) as result from trends_uint where itemid = %s and clock >= %s and clock <= %s''' % (type, type, itemid, start_time, stop_time) self.cursor.execute(sql) result = self.cursor.fetchone()[0] if result: resultlist[type] = int(result) else: resultlist[type] = 0 return resultlist def get_week(self, d): dayscount = datetime.timedelta(days=d.isoweekday()) dayto = d - dayscount sixdays = datetime.timedelta(days=6) dayfrom = dayto - sixdays date_from = datetime.datetime(dayfrom.year, dayfrom.month, dayfrom.day, 0, 0, 0) date_to = datetime.datetime(dayto.year, dayto.month, dayto.day, 23, 59, 59) ts_first = int(time.mktime(datetime.datetime(dayfrom.year, dayfrom.month, dayfrom.day, 0, 0, 0).timetuple())) ts_last = int(time.mktime(datetime.datetime(dayto.year, dayto.month, dayto.day, 23, 59, 59).timetuple())) return ts_first, ts_last def getLastMonthData(self, hostid, table, itemname): ts_first = self.get_week(d)[0] ts_last = self.get_week(d)[1] itemid = self.__getItemid(hostid, itemname) # function = getattr(self, 'get %s Value' % table.capitalize()) function = getattr(self, 'get%sValue' % table.capitalize()) return function(itemid, ts_first, ts_last) def getinfo(self): for ip, resultdict in zabbix.IpInfoList.items(): print("正在查詢 IP:%-15s hostid:%5d 的信息!" % (ip, resultdict['hostid'])) for table, keylists in keys.items(): for key in keylists: print("\t正在統計 key_:%s" % key) data = zabbix.getLastMonthData(resultdict['hostid'], table, key) zabbix.IpInfoList[ip][key] = data def writeToXls(self): dayscount = datetime.timedelta(days=d.isoweekday()) dayto = d - dayscount sixdays = datetime.timedelta(days=6) dayfrom = dayto - sixdays date_from = datetime.date(dayfrom.year, dayfrom.month, dayfrom.day) date_to = datetime.date(dayto.year, dayto.month, dayto.day) '''生成xls文件''' try: import xlsxwriter # 建立文件 workbook = xlsxwriter.Workbook('/usr/monitor/week/%s_%s巡檢報告.xlsx' % (date_from, date_to)) # 建立工做薄 worksheet = workbook.add_worksheet() # 寫入標題(第一行) i = 0 for value in ["主機", "CPU平均空閒值", "CPU最小空閒值", "可用平均內存(單位M)", "可用最小內存(單位M)", "CPU5分鐘負載", "進入最大流量(單位Kbps)", "進入平均流量(單位Kbps)", "出去最大流量(單位Kbps)", "出去平均流量(單位Kbps)"]: worksheet.write(0, i, value) i = i + 1 # 寫入內容: j = 1 for ip, value in self.IpInfoList.items(): worksheet.write(j, 0, ip) worksheet.write(j, 1, '%.2f' % value['system.cpu.util[,idle]']['avg']) worksheet.write(j, 2, '%.2f' % value['system.cpu.util[,idle]']['min']) worksheet.write(j, 3, '%dM' % int(value['vm.memory.size[available]']['avg'] / 1024 / 1024)) worksheet.write(j, 4, '%dM' % int(value['vm.memory.size[available]']['min'] / 1024 / 1024)) worksheet.write(j, 5, '%.2f' % value['system.cpu.load[percpu,avg5]']['avg']) worksheet.write(j, 6, value['net.if.in[eth0]']['max'] / 1000) worksheet.write(j, 7, value['net.if.in[eth0]']['avg'] / 1000) worksheet.write(j, 8, value['net.if.out[eth0]']['max'] / 1000) worksheet.write(j, 9, value['net.if.out[eth0]']['avg'] / 1000) j = j + 1 workbook.close() except Exception as e: print(e) def __del__(self): '''關閉數據庫鏈接''' self.cursor.close() self.conn.close() def sendmail(self): sender = 'lxftty@163.com' receiver = ['lxftty@163.com', '602497030@qq.com'] subject = '上週巡檢報告' smtpserver = 'smtp.163.com' username = 'lxftty@163.com' password = 'asdPP' msg = MIMEText('上週巡檢報告已生成,請檢查。\n報告目錄:/usr/monitor\n因爲郵件沒法發送excel文檔,請使用ftp登陸' '查看。\n機器地址:10.20.2.21:端口\n用戶名:xxx\n', 'plain', 'utf-8') # 中文需參數‘utf-8',單字節字符不須要 msg['Subject'] = Header(subject, 'utf-8') msg['From'] = 'lxftty@163.com' msg['To'] = "602497030@qq.com" smtp = smtplib.SMTP() smtp.connect('smtp.163.com') smtp.login(username, password) smtp.sendmail(sender, receiver, msg.as_string()) print('發送成功!') smtp.quit()
if name == "main":
zabbix = ReportForm()
zabbix.getinfo()
zabbix.writeToXls()
zabbix.sendmail()運維
」「」ide
測試執行效果
郵件也有通知
==============================================
接下來開始對每週報警的級別統計
連接:https://pan.baidu.com/s/1tOW4lPrAAsYMf_fg9y4rtg
提取碼:1ptl
"""
import xlsxwriter
import datetime
import pymysql
import numpy as np
import pandas
author = 'jmmei'
data = '2020/07/22'
averagesql = """
select from_unixtime(clock,'%Y-%m-%d') as time,
count(DISTINCT eventid) as id
from alerts
WHERE mediatypeid = 9 #這裏須要修改,去數據庫對應上
AND message LIKE '%PROBLEM%'
AND message LIKE '%Average%'
AND SUBJECT LIKE '%PROBLEM%'
AND clock BETWEEN UNIX_TIMESTAMP(DATE_SUB(NOW(), INTERVAL 7 DAY)) AND UNIX_TIMESTAMP(DATE_SUB(NOW(), INTERVAL 1 DAY))
group by time
order by time asc
"""
highsql = """
select from_unixtime(clock,'%Y-%m-%d') as time,
count(DISTINCT eventid) as id
from alerts
WHERE mediatypeid = 9
AND message LIKE '%PROBLEM%'
AND message LIKE '%High%'
AND SUBJECT LIKE '%PROBLEM%'
AND clock BETWEEN UNIX_TIMESTAMP(DATE_SUB(NOW(), INTERVAL 7 DAY)) AND UNIX_TIMESTAMP(DATE_SUB(NOW(), INTERVAL 1 DAY))
group by time
order by time asc
"""
warningsql = """
select from_unixtime(clock,'%Y-%m-%d') as time,
count(DISTINCT eventid) as id
from alerts
WHERE mediatypeid = 9
AND message LIKE '%PROBLEM%'
AND message LIKE '%Warning%'
AND SUBJECT LIKE '%PROBLEM%'
AND clock BETWEEN UNIX_TIMESTAMP(DATE_SUB(NOW(), INTERVAL 7 DAY)) AND UNIX_TIMESTAMP(DATE_SUB(NOW(), INTERVAL 1 DAY))
group by time
order by time asc
"""
informationsql = """
select from_unixtime(clock,'%Y-%m-%d') as time,
count(DISTINCT eventid) as id
from alerts
WHERE mediatypeid = 9
AND message LIKE '%PROBLEM%'
AND message LIKE '%Information%'
AND SUBJECT LIKE '%PROBLEM%'
AND clock BETWEEN UNIX_TIMESTAMP(DATE_SUB(NOW(), INTERVAL 7 DAY)) AND UNIX_TIMESTAMP(DATE_SUB(NOW(), INTERVAL 1 DAY))
group by time
order by time asc
"""
disastersql = """
select from_unixtime(clock,'%Y-%m-%d') as time,
count(DISTINCT eventid) as id
from alerts
WHERE mediatypeid = 9
AND message LIKE '%PROBLEM%'
AND message LIKE '%Diasater%'
AND SUBJECT LIKE '%PROBLEM%'
AND clock BETWEEN UNIX_TIMESTAMP(DATE_SUB(NOW(), INTERVAL 7 DAY)) AND UNIX_TIMESTAMP(DATE_SUB(NOW(), INTERVAL 1 DAY))
group by time
order by time asc
"""
#鏈接MySQL數據庫
def get_count(sql):
'''
#zabbix數據庫信息:
zdbhost = '10.20.2.21'
zdbuser = 'zabbix'
zdbpass = 'zabbix'
zdbport = 3306
zdbname = 'zabbix'
'''
conn=pymysql.connect("10.20.2.21", "zabbix", "zabbix", "zabbix", charset='utf8')
cursor = conn.cursor()
#print("===============cursor:",cursor,type(cursor))
cursor.execute(sql)
count = cursor.fetchall()
#print("=================count:",count) rows = np.array(count) #print("============rows:",rows) conn.close() return count
def coloum(data, weekendtime):
#建立一個excel文件
workbook = xlsxwriter.Workbook(weekendtime +".xlsx")
#建立一個工做表,默認sheet1
worksheet = workbook.add_worksheet()
bold = workbook.add_format({'bold': 1})
#表頭
title = ['告警級別', '星期一','星期二','星期三','星期四','星期五','星期六','星期日']
#列名
buname = ['information', 'warning','average', 'high', 'disaster']
formatter = workbook.add_format() formatter.set_border(2)
#定義格式:# 定義標題欄格式對象:邊框加粗1像素,背景色爲灰色,單元格內容居中、加粗,標題字體顏色
title_formatter = workbook.add_format()
title_formatter.set_border(2)
title_formatter.set_bg_color('#4682B4')
title_formatter.set_align('center')
title_formatter.set_bold()
title_formatter.set_color("#F8F8FF")
title_formatter.set_font_size(14)
chart_col = workbook.add_chart({'type': 'column'})
def chart_series(row):
chart_col.add_series(
{
'categories': '=Sheet1!$B$1:$H$1',
'values': '=Sheet1!$B${}:$H${}'.format(row, row),
'line': {'color': 'black'},
'name': '=Sheet1!$A$' + row } )
worksheet.write_row('A1',title,title_formatter) worksheet.write_column('A2',buname,formatter) for i in range (2,7): worksheet.write_row('B{}'.format(i),data[i-2],formatter) print (i) chart_series(str(i)) # 設置圖表的title 和 x,y軸信息 chart_col.set_title({'name': '告警統計/周'}) chart_col.set_x_axis({'name': '告警級別'}) chart_col.set_y_axis({'name': '告警次數'}) # 設置圖表的風格 #chart_col.set_style(9) # 把圖表插入到worksheet以及偏移 worksheet.insert_chart('A10', chart_col, {'x_offset': 25, 'y_offset': 10}) workbook.close()
#判斷二維元組是否爲空,長度是否知足要求,不知足則補0.
#輸入爲一個字典,判斷是否爲空,空則添加數據
def covertdata(jsondata,weektime):
listkey = list(jsondata.keys()) for i in weektime: j = i.strftime("%Y-%m-%d") if listkey: if j not in listkey: jsondata[j] = "0" else: jsondata[j] = "0" #print ("jsondata:>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>",jsondata)
#按照時間對字典進行排序
sort = sorted(jsondata.items(), key=lambda d: d[0])
#將第二列取出來並轉爲列表
array = np.array(sort)
array2 = array[:, 1]
list2 = array2.tolist()
list3 = list(map(lambda x: float(x), list2))
#print("list2:>>>>>>>>>>>>>>>", list2)
#print("list3:>>>>>>>>>>>>>>>", list3)
return list3
if name == 'main':
yesterday = (datetime.date.today() + datetime.timedelta(days=-1)).strftime("%Y-%m-%d")
print(yesterday)
weeklist = pandas.date_range(end=yesterday, periods=7)
informationdata = get_count(informationsql) informationlist = covertdata(dict(informationdata),weeklist) warningdata = get_count(warningsql) warninglist = covertdata(dict(warningdata),weeklist) averagedata = get_count(averagesql) averagelist = covertdata(dict(averagedata),weeklist) highdata = get_count(highsql) highlist = covertdata(dict(highdata),weeklist) disasterdata = get_count(disastersql) disasterlist = covertdata(dict(disasterdata),weeklist) print (informationlist) print (warninglist) print (averagelist) print (highlist) print (disasterlist) data1 = [informationlist, warninglist, averagelist, highlist, disasterlist] coloum(data1,yesterday)
"""
看看執行結果