文本分析——分詞、統計詞頻、詞雲

導入包

import os
import sys
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import re
from pandas import Series, DataFrame  

import string
import re
import jieba
import jieba.analyse
import datetime
from wordcloud import WordCloud, ImageColorGenerator
import codecs

導入文件和數據

gongdan = pd.read_excel('Gongdan.xlsx')

數據預處理

gongdan['content'] = [str(i) for i in gongdan['content']]
gongdan['content'] = [''.join(re.findall(u'[\u4e00-\u9fff]+', i)) for i in gongdan['content']]
indexs = list(gongdan['content'][pd.isnull(gongdan['content'])].index)
gongdan = gongdan.drop(indexs)
indexs = list(gongdan['content'][gongdan['content']==''].index)
gongdan = gongdan.drop(indexs)

content = gongdan['content']

cont = ''.join(content)
cont = ''.join(re.findall(u'[\u4e00-\u9fa5]+', cont))

分詞並去除停用詞

stopwords = set()
fr = codecs.open('stopwords.txt', 'r', 'utf-8')
for word in fr:
   stopwords.add(str(word).strip())
fr.close()

jieba.load_userdict("dict.txt")
text = list(jieba.cut(cont, cut_all=False, HMM=True))
text = list(filter(lambda x: x not in stopwords, text))
text = [str(i) for i in text if i != ' ']

Tfidf 算法

from sklearn import feature_extraction
from sklearn.feature_extraction.text import TfidfTransformer  
from sklearn.feature_extraction.text import CountVectorizer  

test = ' '.join(text)
tlist = []
tlist.append(test)

vectorizer=CountVectorizer()#該類會將文本中的詞語轉換爲詞頻矩陣，矩陣元素a[i][j] 表示j詞在i類文本下的詞頻
transformer = TfidfTransformer()#該類會統計每一個詞語的tf-idf權值
tfidf = transformer.fit_transform(vectorizer.fit_transform(tlist))  #第一個fit_transform是計算tf-idf，第二個fit_transform是將文本轉爲詞頻矩陣  

word=vectorizer.get_feature_names()#獲取詞袋模型中的全部詞語  
weight=tfidf.toarray()#將tf-idf矩陣抽取出來，元素a[i][j]表示j詞在i類文本中的tf-idf權重  
tfidf_list = {}
for i in range(len(weight)):#打印每類文本的tf-idf詞語權重，第一個for遍歷全部文本，第二個for便利某一類文本下的詞語權重  
    for j in range(len(word)):  
        tfidf_list[word[j]] = weight[i][j]

詞雲

font_path = 'yahei.ttf'

from PIL import Image
back_coloring = np.array(Image.open('circle.jpg'))

wc = WordCloud(font_path=font_path,  # 設置字體
               background_color="white",  # 背景顏色
               max_words=60,  # 詞雲顯示的最大詞數
               mask=back_coloring,  # 設置背景圖片
               stopwords=stopwords,
               max_font_size=100,  # 字體最大值
               random_state=42,
               width=1000, height=860, margin=2,# 設置圖片默認的大小,可是若是使用背景圖片的話,那麼保存的圖片大小將會按照其大小保存,margin爲詞語邊緣距離
#               prefer_horizontal=1,
               )

wc.generate_from_frequencies(tfidf_list)

plt.imshow(wc, interpolation="bilinear")
plt.axis("off")
plt.figure()
wc.to_file("w.png")

# create coloring from image
image_colors = ImageColorGenerator(back_coloring)
# recolor wordcloud and show
# we could also give color_func=image_colors directly in the constructor
plt.imshow(wc.recolor(color_func=image_colors), interpolation="bilinear")
plt.axis("off")
plt.figure()
plt.imshow(back_coloring, cmap=plt.cm.gray, interpolation="bilinear")
plt.axis("off")
plt.show()