python文本分類

前面博客裏面從謠言百科中爬取到了全部類別（10類）的新聞並以文本的形式存儲。

如今對這些數據進行分類，上代碼：

# -*- coding: utf-8 -*-
""" Created on Fri Mar 9 14:18:49 2018 @author: Administrator """

import os import time import random import jieba import nltk import sklearn from sklearn.naive_bayes import MultinomialNB import numpy as np import pylab as pl import matplotlib.pyplot as plt def MakeWordsSet(words_file): words_set = set() with open(words_file, 'r', encoding='UTF-8') as fp: for line in fp.readlines(): word = line.strip() if len(word)>0 and word not in words_set: # 去重
 words_set.add(word) return words_set def TextProcessing(folder_path, test_size=0.2): folder_list = os.listdir(folder_path)#獲取文件夾下全部子文件夾
    data_list = []#獲取文本數據
    class_list = []#獲取類別數據

    # 全部類別進行循環
    for folder in folder_list: new_folder_path = os.path.join(folder_path, folder)#得到子文件夾路徑
        files = os.listdir(new_folder_path)#得到子文件夾下全部文件
        # 類內循環
        j = 0 for file in files: if j > 410: # 每類text樣本數最多不超過這個
                break with open(os.path.join(new_folder_path, file), 'r', encoding='UTF-8') as fp: raw = fp.read() # print raw
            ## --------------------------------------------------------------------------------
            ## jieba分詞
 word_cut = jieba.cut(raw, cut_all=False) # 精確模式，返回的結構是一個可迭代的genertor
            word_list = list(word_cut) # genertor轉化爲list，每一個詞unicode格式

            ## --------------------------------------------------------------------------------
 data_list.append(word_list) class_list.append(folder) j += 1

    ## 劃分訓練集和測試集
    # train_data_list, test_data_list, train_class_list, test_class_list = sklearn.cross_validation.train_test_split(data_list, class_list, test_size=test_size)
    data_class_list = list(zip(data_list, class_list))#zip函數:接受2個序列做爲參數，返回tuple列表
    random.shuffle(data_class_list)#shuffle() 將序列的全部元素隨機排序。
    index = int(len(data_class_list)*test_size)+1#數據總量*0.2來劃分訓練集和測試集
    train_list = data_class_list[index:]#訓練集爲後0.8的數據
    test_list = data_class_list[:index]#測試集爲前0.2的數據
    train_data_list, train_class_list = zip(*train_list)#訓練數據集
    test_data_list, test_class_list = zip(*test_list)#測試數據集

    # 統計詞頻放入all_words_dict
    all_words_dict = {} for word_list in train_data_list: for word in word_list: if word in all_words_dict: all_words_dict[word] += 1
            else: all_words_dict[word] = 1
    # key函數利用詞頻進行降序排序
    all_words_tuple_list = sorted(all_words_dict.items(), key=lambda f:f[1], reverse=True) # 內建函數sorted參數需爲list
    all_words_list = list(zip(*all_words_tuple_list))[0] return all_words_list, train_data_list, test_data_list, train_class_list, test_class_list def words_dict(all_words_list, deleteN, stopwords_set=set()):# 選取特徵詞:不全爲數字，不是停留子，長度在1到5之間
 feature_words = [] n = 1
    for t in range(deleteN, len(all_words_list), 1): if n > 1000: # feature_words的維度1000
            break
        # print all_words_list[t]
        if not all_words_list[t].isdigit() and all_words_list[t] not in stopwords_set and 1<len(all_words_list[t])<5:# isdigit() 方法檢測字符串是否只由數字組成。
 feature_words.append(all_words_list[t]) n += 1
    return feature_words def TextFeatures(train_data_list, test_data_list, feature_words, flag='nltk'): def text_features(text, feature_words): text_words = set(text) ## -----------------------------------------------------------------------------------
        if flag == 'nltk': ## nltk特徵 dict
            features = {word:1 if word in text_words else 0 for word in feature_words} elif flag == 'sklearn': ## sklearn特徵 list
            features = [1 if word in text_words else 0 for word in feature_words] else: features = [] ## -----------------------------------------------------------------------------------
        return features train_feature_list = [text_features(text, feature_words) for text in train_data_list] test_feature_list = [text_features(text, feature_words) for text in test_data_list] return train_feature_list, test_feature_list def TextClassifier(train_feature_list, test_feature_list, train_class_list, test_class_list, flag='nltk'): ## -----------------------------------------------------------------------------------
    if flag == 'nltk': ## nltk分類器
        train_flist = zip(train_feature_list, train_class_list) test_flist = zip(test_feature_list, test_class_list) classifier = nltk.classify.NaiveBayesClassifier.train(train_flist) test_accuracy = nltk.classify.accuracy(classifier, test_flist) elif flag == 'sklearn': ## sklearn分類器
        classifier = MultinomialNB().fit(train_feature_list, train_class_list) test_accuracy = classifier.score(test_feature_list, test_class_list) else: test_accuracy = [] return test_accuracy if __name__ == '__main__': print("start") ## 文本預處理
    folder_path = 'F:\\test\\demo' all_words_list, train_data_list, test_data_list, train_class_list, test_class_list = TextProcessing(folder_path, test_size=0.2) # 生成stopwords_set
    stopwords_file = 'F:\\test\\stopword.txt' stopwords_set = MakeWordsSet(stopwords_file) ## 文本特徵提取和分類
    # flag = 'nltk'
    flag = 'sklearn' deleteNs = range(0, 1000, 20) test_accuracy_list = [] for deleteN in deleteNs: # feature_words = words_dict(all_words_list, deleteN)
        feature_words = words_dict(all_words_list, deleteN, stopwords_set)#特徵詞;
        train_feature_list, test_feature_list = TextFeatures(train_data_list, test_data_list, feature_words, flag)#得到訓練集以及測試數據集;
        test_accuracy = TextClassifier(train_feature_list, test_feature_list, train_class_list, test_class_list, flag) test_accuracy_list.append(test_accuracy) print(test_accuracy_list) # 結果評價
 plt.figure() plt.plot(deleteNs, test_accuracy_list) plt.title('Relationship of deleteNs and test_accuracy') plt.xlabel('deleteNs') plt.ylabel('test_accuracy') plt.savefig('result.png') print("finished")

 

運行完分類完成！