python爬蟲爬取各個城市歷史天氣及數據可視化

時間 2019-11-11

原文原文鏈接

前言：html

　　爬取網站www.tianqihoubao.com上各城市的歷史天氣數據，並利用matplotlib將數據可視化。數據庫

代碼實現：session

  1 import asyncio
  2 import aiohttp
  3 from lxml import etree
  4 import re
  5 from collections import namedtuple
  6 import matplotlib.pyplot as plt    
  7 
  8 #用來正常顯示中文標籤
  9 plt.rcParams['font.sans-serif']=['SimHei'] 
 10 
 11 Args = namedtuple('Args',['city','year','month'])
 12 
 13 #獲取給定城市列表在給定年列表的以月爲單位的天天平均溫度字典，
 14 #返回格式 {Args(city='wuhan', year=2018, month=4): [21.5, 22.5, ...], Args(city='shanghai', year=2018, month=2): [3.5, 2.5, ...], ...}
 15 def get_weather(citys, years):
 16     tdata = {}
 17     wdata = {}
 18     async def work(args):
 19         url = "http://www.tianqihoubao.com/lishi/%s/month/%d%02d.html" % (args.city, args.year, args.month)
 20         headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36'}
 21         async with aiohttp.ClientSession() as session:
 22             async with session.get(url, headers=headers, timeout=1000, verify_ssl=False) as response:
 23                 html = await response.text()
 24                 ethtml = etree.HTML(html)
 25 
 26                 #從html頁面提取數據，數據格式如： '10℃/-2℃'
 27                 result = [re.sub("\r\n *","",item.text) for item in ethtml.xpath("//table[@class='b']/tr/td") if not re.fullmatch("\r\n *",item.text)]
 28 
 29                 #獲取溫度數據 '10℃/-2℃'，將'10℃/-2℃'格式數據正則提取，求平均值
 30                 tstrs = result[1::3]
 31                 ts = [sum(map(lambda x: int(x), re.findall('(\d+|-\d+)℃\/(\d+|-\d+)℃', tstr)[0]))/2 for tstr in tstrs 
 32                                                                                     if re.fullmatch('(\d+|-\d+)℃\/(\d+|-\d+)℃', tstr)]
 33 
 34                 #獲取天氣數據 '多雲/晴', 
 35                 wstrs = [re.findall('([\u4E00-\u9FA5]+)\/([\u4E00-\u9FA5]+)',wstr)[0] for wstr in result[0::3] 
 36                                                                                     if re.fullmatch('([\u4E00-\u9FA5]+)\/([\u4E00-\u9FA5]+)',wstr)]
 37                 ws = [re.sub(' ','',b) for a in wstrs for b in a]
 38 
 39                 tdata[args] = ts
 40                 wdata[args] = ws
 41                 print(args, 'done')
 42                 return None
 43 
 44     loop = asyncio.get_event_loop()
 45 
 46     #用列表推導是爲全部城市全部年的每個月添加協程
 47     tasks = [asyncio.ensure_future(work(Args(city, year, month))) for city in citys for year in years for month in range(1,13)]
 48     
 49     #開始執行協程，事件循環處理異步IO事件，程序阻塞在這裏，直到全部的協程執行完畢
 50     loop.run_until_complete(asyncio.wait(tasks))
 51     
 52     #return [task.result() for task in tasks]
 53     return (tdata,wdata)
 54 
 55 
 56 #繪製多個城市在某一年的溫度曲線
 57 def temperature_curve(citys, year, data):
 58     for city in citys:
 59         x = range(365)
 60         y = [b for a in [data[Args(city, year, month)] for month in range(1,13)] for b in a][:365]
 61         plt.plot(x, y, label=city)
 62         plt.xticks(list(x)[::31], ['Jan','Feb','Mar','Apr','May','Jun','Jul','Aug','Sept','Oct','Nov','Dec'])
 63         plt.yticks([-10,0,10,20,30,40],['-10℃','0℃','10℃','20℃','30℃','40℃'])
 64         plt.legend()
 65     
 66     plt.title('各城市%d年溫度曲線圖' % year)
 67     plt.ylabel('temperature')
 68     plt.xlabel('month')
 69     plt.show()
 70 
 71 #繪製多個城市在某一年各個天氣比例
 72 def weather_ratio(citys, year, data):
 73 
 74     #建立 天氣-顏色 字典
 75     color_dict = {'晴':'tomato', '多雲':'gold', '陰':'turquoise', '陰天':'turquoise', '雷陣雨':'darkseagreen',
 76                     '暴雨':'black','晴間多雲':'cyan', '陣雨':'cadetblue', '浮塵':'pink', '揚沙':'plum','霾':'dimgray', 
 77                     '小雨':'cornflowerblue', '中雨':'blue', '大雨':'navy', '小雪':'lime', '中雪':'violet', '大雪':'teal',
 78                     '雨夾雪':'sienna', '凍雨':'skyblue', '少雲':'steelblue', '霧':'white'}
 79     for city in citys:
 80         labels = []
 81         sizes = []
 82         colors = []
 83         city_weathers = [b for a in [data[Args(city, year, month)] for month in range(1,13)] for b in a]
 84         for weather in set(city_weathers):
 85             labels.append(weather)
 86             sizes.append(city_weathers.count(weather) * 100 / len(city_weathers))
 87 
 88             #給不一樣天氣添加顏色
 89             colors.append(color_dict[weather])
 90 
 91         #關聯排序，sizes按逆序排序，labels，colors按sizes關聯排序
 92         z = zip(sizes,labels,colors)
 93         z = sorted(z, reverse=True)
 94         sizes,labels,colors = zip(*z)
 95 
 96         #只顯示最頻繁的7中天氣文字標籤（以避免文字間相互遮擋）
 97         labels = list(labels)[:7] + [''] * (len(labels) - 7)
 98 
 99         #使比率最大的天氣突出顯示
100         explode = [0] * len(sizes)
101         explode[0] = 0.1
102 
103         ax = plt.subplot(1, len(citys), citys.index(city) + 1)
104         ax.pie(sizes, labels=labels,autopct='%1.1f%%',shadow=False,startangle=150,colors=colors, explode=explode)
105         ax.set_title('%s %d年天氣餅圖' % (city, year))
106     plt.show()
107     
108 tdata,wdata =  get_weather(citys=['wuhan','shanghai','beijing'], years=[2018])
109 temperature_curve(citys=['wuhan', 'shanghai', 'beijing'], year=2018, data=tdata)
110 weather_ratio(citys=['wuhan','shanghai','beijing'], year=2018, data=wdata)