蘇州市java崗位的薪資情況（2）

時間 2019-11-06

標籤蘇州市 java 崗位薪資情況欄目 Java 简体版

原文原文鏈接

　　上一篇已經統計出了起薪最高的top 10:html

　　接着玩，把top 10 中全部職位的詳細信息爬取下來。某一職位的詳情是這樣：架構

　　咱們須要把工做經驗、學歷、職能、關鍵字爬取下來。app

 1 from urllib.request import urlopen
 2 from urllib.error import HTTPError
 3 from bs4 import BeautifulSoup
 4 import csv
 5 from itertools import chain
 6 import threading
 7
 8 def load_datas():
 9     '''
10     從joblist.csv中裝載數據
11     :return: 數據集 datas
12     '''
13     datas = []
14     with open('high10_url.csv', encoding='utf-8') as fp:
15         r = csv.reader(fp)
16         for row in r:
17             datas.append(row[0])
18     return datas
19
20 def get_desc(url):
21     ''' 爬取職位的詳細信息，包括：經驗， 學歷, 職位, 技能關鍵字 '''
22     try:
23         html = urlopen(url)
24     except HTTPError as e:
25         print('Page was not found', e.filename)
26         return []
27
28     job_desc = []  # 職位詳情
29     try:
30         exp, edu, position, keys = '', '', '', []  # 經驗， 學歷, 職位, 技能關鍵字
31         bsObj = BeautifulSoup(html.read())
32         contents = bsObj.find('p', {'class': 'msg ltype'}).contents
33         exp = contents[2].strip()  # 經驗
34         edu = contents[4].strip()  # 學歷
35         print(edu)
36         a_list = bsObj.findAll('a', {'class': 'el tdn'})
37         for i, a in enumerate(a_list):
38             if i == 0:
39                 position = a.get_text()  # 職位
40             else:
41                 keys.append(a.get_text())  # 技能關鍵字
42         job_desc.append((exp, edu, position, keys))
43     except AttributeError as e:
44         print(e)
45         job_desc = []
46     return job_desc
47
48 def crawl(urls):
49     '''
50     :param urls: 職位詳情
51     '''
52     print('開始爬取數據...')
53     job_desc = [get_desc(url) for url in urls]
54     print('爬取結束')
55     return job_desc
56
57 def save_data(all_jobs, f_name):
58     '''
59     將信息保存到目標文件
60     :param all_jobs: 二維列表，每一個元素是一頁的職位信息
61     '''
62     print('正在保存數據...')
63     with open(f_name, 'w', encoding='utf-8', newline='') as fp:
64         w = csv.writer(fp)
65         # 將二維列表轉換成一維
66         t = list(chain(*all_jobs))
67         w.writerows(t)
68         print('保存結束，共{}條數據'.format(len(t)))
69
70 urls = load_datas()
71 job_desc = crawl(urls)
72 print(job_desc)
73 save_data(job_desc, 'job_desc.csv')

　　high10_url.csv中已經預先存儲了top 10的全部64個url。job_desc.csv中的結果以下：學習

　　學歷列出現了問題，第5行顯示的是「招1人」，實際上這個職位沒有學歷要求，把全部「招x人」的記錄都改爲「無要求」。url

　　接下來能夠按照經驗、學歷、職能分別統計：spa

import csv
import pandas as pd
import numpy as np

def load_datas():
    '''
    從joblist.csv中裝載數據
    :return: 數據集 datas
    '''
    datas = []
    with open('job_desc.csv', encoding='utf-8') as fp:
        r = csv.reader(fp)
        for row in r:
            datas.append(row)
    return datas

def analysis(datas):
    ''' 數據分析 '''
    df = pd.DataFrame({'exp': datas[:, 0],
                       'edu': datas[:, 1],
                       'position': datas[:, 2],
                       'keys': datas[:, 3]})
    count(df, 'exp', '經驗') # 按經驗統計
    count(df, 'edu', '學歷') # 按學歷統計
    count(df, 'position', '職位') # 按職位統計

def count(df, idx, name):
    ''' 分組統計 '''
    print(('按' + name + '分組').center(60, '-'))
    c = df[idx].value_counts(sort=True)
    print(c)

if __name__ == '__main__':
    # 讀取並清洗數據
    datas = np.array(load_datas())
    analysis(datas)