文章目录
1、简述
如果用到TF-IDF,sklearn中经常会用CountVectorizer与TfidfTransformer两个类。我们总是需要保存TF-IDF的词典,然后计算测试集的TF-IDF,这里要注意sklearn中保存有两种方法:pickle与joblib。这里,我们可以用pickle保存特征,用joblib保存模型。
2、 CountVectorizer 和 Transformer保存和加载
2.1、TF-IDF词典的保存
train_content = segmentWord(X_train)
test_content = segmentWord(X_test)
# replace 必须加,保存训练集的特征
vectorizer = CountVectorizer(decode_error="replace")
tfidftransformer = TfidfTransformer()
# 注意在训练的时候必须用vectorizer.fit_transform、tfidftransformer.fit_transform
# 在预测的时候必须用vectorizer.transform、tfidftransformer.transform
vec_train = vectorizer.fit_transform(train_content)
tfidf = tfidftransformer.fit_transform(vec_train)
# 保存经过fit的vectorizer 与 经过fit的tfidftransformer,预测时使用
feature_path = 'models/feature.pkl'
with open(feature_path, 'wb') as fw:
pickle.dump(vectorizer.vocabulary_, fw)
tfidftransformer_path = 'models/tfidftransformer.pkl'
with open(tfidftransformer_path, 'wb') as fw:
pickle.dump(tfidftransformer, fw)
注意:vectorizer 与tfidftransformer都要保存,而且只能 fit_transform 之后保存,表示vectorizer 与tfidftransformer已经用训练集训练好了。
2.2、TF-IDF加载,测试新数据
# 加载特征
feature_path = 'models/feature.pkl'
loaded_vec = CountVectorizer(decode_error="replace", vocabulary=pickle.load(open(feature_path, "rb")))
# 加载TfidfTransformer
tfidftransformer_path = 'models/tfidftransformer.pkl'
tfidftransformer = pickle.load(open(tfidftransformer_path, "rb"))
#测试用transform,表示测试数据,为list
test_tfidf = tfidftransformer.transform(loaded_vec.transform(test_content))
3、模型的保存和加载
3.1、模型的保存
# clf_model为生成的模型,利用joblib的dump保存
clf_model = trainModel()
joblib.dump(clf_model, "model_"+path)
3.2、模型的加载
# clf_model为模型,利用joblib的load加载
clf_model = joblib.load(model_path)
4、例子
举了一个简单的例子,可以学一下这里到底怎么使用
"""
Author:沙振宇
Time:20191112
Info:简单的情绪识别
"""
from sklearn.feature_extraction.text import TfidfTransformer
from sklearn.feature_extraction.text import CountVectorizer
from sklearn import svm
from sklearn.metrics import accuracy_score
import joblib
import os
import jieba
import datetime
import warnings # 忽略警告
import pickle
warnings.filterwarnings("ignore", category=FutureWarning, module="sklearn", lineno=196)
m_list_allText = []# 全部内容(包含重复标签,顺序)
m_list_allL4ID = []# 全部标签(包含重复标签,顺序)
m_list_allLabel = [] # 模型全部标签(不包含重复标签,顺序)
m_map_id_score = {} # id对应的分值
# 读取文件里面数据,获取标签和内容
def getFile(filename, count = -1):
with open(filename, 'r' ,encoding='utf-8') as fp:
global m_list_allL4ID,m_list_allText
m_list_allL4ID = []
m_list_allText = []
tmp_text = []
m_file_text = fp.readlines()
tmp_lines = len(m_file_text)
for i in range(tmp_lines):
text = m_file_text[i]
if ":" in text:
L4ID = text.split(":")[-2]
Msg = text.split(":")[-1]
m_list_allL4ID.append(L4ID)
m_list_allText.append(Msg)
if L4ID not in m_list_allLabel:
m_list_allLabel.append(L4ID)
tmp_text = []
tmp_text.append(Msg)
# jieba分词
def jiabaToVector(list, isTest, isTFIDF = False):
tmp_list = []
for sentence in list:
tmp_list.append(" ".join(jieba.cut(sentence.strip())))
# 利用TFIDF生成词向量
transformer = TfidfTransformer()
if isTest:
if isTFIDF:
tfidf = transformer.fit_transform(vectorizer.transform(tmp_list))
else:
tfidf = vectorizer.transform(tmp_list)
else:
if isTFIDF:
tfidf = transformer.fit_transform(vectorizer.fit_transform(tmp_list))
else:
tfidf = vectorizer.fit_transform(tmp_list)
return tfidf
# 创建默认的参数
def predict_4(X, Y):
clf = svm.LinearSVC()
clf = clf.fit(X, Y)
return clf
# 将L4转换为整形
def L4ToInt(m_label_l4):
m_label_l4New = []
for i in range(len(m_label_l4)):
m_label_l4New.append(int(m_label_l4[i][1:]))
# print("m_label_l4New:",m_label_l4New)
return m_label_l4New
# 训练SVM模型
def trainSVM(path, linecount = -1):
getFile(path, linecount)
vectorizer = CountVectorizer(decode_error="replace") # 全局向量, replace 必须加,保存训练集的特征
vector_train = jiabaToVector(m_list_allText, False, True)# 生成训练向量
lenall = len(m_list_allText)# 数据大小
print("总集大小:", lenall)# print("总集大小:", lenall)
startT_Train = datetime.datetime.now()# 训练
clf = predict_4(vector_train, m_list_allL4ID)
endT_Train = datetime.datetime.now()
print("训练Time:", (endT_Train - startT_Train).microseconds)
return clf,vectorizer
# 查看完全匹配
def completeLabelDataMatch(path , query):
outList = {}
file_train = os.path.join(path)
with open(file_train, 'r', encoding='UTF-8')as fp:
textlist = fp.readlines()
for text in textlist:
if ":" in text:
conditionId = text.split(":")[-2]
Msg = text.split(":")[-1]
message = Msg.strip("\n")
if query == message:
outList["conditionId"] = conditionId
outList["Score"] = 1
print("Complete labelData match work: %s:%s"%(conditionId,message))
return outList
return False
# 查看query的分值
def SVMMain(path, clf, query, score):
outList = completeLabelDataMatch(path, query)
if outList:
print("outList[\"conditionId\"]:", outList["conditionId"])
print("outList[\"Score\"]:", outList["Score"])
else:
outList = useSVM(clf, query)
if outList["Score"] > score:
return emotionAnalysis(outList["conditionId"])
else:
return "normal"
# 运用SVM模型
def useSVM(clf, query):
outList = {}
querylist = []
querylist.append(query)
vector_test = jiabaToVector(querylist, True, True) # 生成测试向量
startT = datetime.datetime.now()
percent = clf.decision_function(vector_test)
scorelist = []
if len(percent[0]):
scorelist = percent[0]
if len(m_list_allLabel) == len(scorelist):
for i in range(len(scorelist)):
m_map_id_score[m_list_allLabel[i]] = scorelist[i]
pVallist = sorted(scorelist,reverse=True)
percent = max(pVallist)
conditionID = ""
for item in range (len(m_map_id_score)):
if m_map_id_score[m_list_allLabel[item]] == percent:
conditionID = m_list_allLabel[item]
endT = datetime.datetime.now()
print("测试Time:", (endT - startT).microseconds)
outList["conditionId"] = conditionID
outList["Score"] = percent
print("outList[\"conditionId\"]:", outList["conditionId"])
print("outList[\"Score\"]:", outList["Score"])
return outList
# 情绪识别
def emotionAnalysis(label):
negtiveId = ['4000447','4000448','4000453','4000449','4000450','4000451', '4000452','4000454','4000459','4000458','4002227','4000461','4000460','4000465','4000464','4000803','4000468'] # 消极
positiveId = ['4000439','4000440','4000441','4000442','4000462','4000467','4000469','4000496','4000497'] # 积极
print("negtiveId:",negtiveId)
print("positiveId:",positiveId)
if label in negtiveId:
return "negtive"
elif label in positiveId:
return "positive"
else:
return "normal"
# 保存模型和特征
def saveModel(path):
clf, vectorizer = trainSVM(path, -1)
feature_path = path
if "/" in path:
joblib.dump(clf, "model_" + path.split("/")[1])
if ".txt" in path:
feature_path = 'feature_' + path.split("/")[1].split(".txt")[0] + '.pkl'
else:
joblib.dump(clf, "model_" + path)
if ".txt" in path:
feature_path = 'feature_' + path.split(".txt")[0] + '.pkl'
print("模型已经保存,开始保存特征")
with open(feature_path, 'wb') as fw:
pickle.dump(vectorizer.vocabulary_, fw)
print("特征已经保存。。。")
# 使用模型和特征
def useModel(model_path, feature_path):
# 加载模型
clf = joblib.load(model_path)
# 加载特征
loaded_vec = CountVectorizer(decode_error = "replace", vocabulary = pickle.load(open(feature_path, "rb")))
return clf, loaded_vec
if __name__ =="__main__":
path = "../rg_train_20171230_1000008.txt"
# 保存模型
# saveModel(path)
# 加载模型
global vectorizer
clf, vectorizer = useModel("model_rg_train_20171230_1000008.txt","feature_rg_train_20171230_1000008.pkl")
source = "我很开心"
source = source.replace("\r", "")
source = source.replace("\n", "")
source = source.lower()
source = source[0:256]
print("开始匹配")
result = SVMMain(path , clf, source, 0.6)
if result == "normal":
print("中性情绪")
elif result == "negtive":
print("负向情绪")
elif result == "positive":
print("正向情绪")