版权声明:本文为博主原创文章,未经博主允许不得转载。 https://blog.csdn.net/luoxueqian/article/details/78806984
代码及注释如下:
#使用贝叶斯算法实现垃圾邮件过滤
#将一个大字符串解析为字符串列表
def textParse(bigString):
import re
listOfTokens = re.split(r'\W*', bigString)
return [tok.lower() for tok in listOfTokens if len(tok) > 2]
def spamTest():
#import pandas as pd
docList = []; classList = []; fullText = []
for i in range(1,26):
#wordList = textParse(pd.read_csv('email/spam/%d.txt' %i, sep='\n', encoding='utf8'))
wordList = textParse(open('email/spam/%d.txt' % i).read()) #spam文件夹中的邮件全设为1
docList.append(wordList)
fullText.extend(wordList)
classList.append(1)
#wordList = textParse(pd.read_csv('email/ham/%d.txt' % i, sep='\n', encoding='utf8'))
wordList = textParse(open('email/ham/%d.txt' % i).read()) #ham文件夹中的邮件全设为0
docList.append(wordList)
fullText.extend(wordList)
classList.append(0)
vocabList = createVocabList(docList) #将重复出现的单词删掉
trainingSet = list(range(50)); testSet = []
#随机选取10封邮件为测试集
for i in range(10):
randIndex = int(random.uniform(0,len(trainingSet)))
testSet.append(trainingSet[randIndex])
del(trainingSet[randIndex]) #将测试集从训练集中删除
trainMat = []; trainClasses = []
#剩下的40封作为训练集
for docIndex in trainingSet:
trainMat.append(setOfWords2Vec(vocabList, docList[docIndex])) #将文本转换成向量
trainClasses.append(classList[docIndex])
p0V,p1V,pSpam = trainNB0(array(trainMat), array(trainClasses)) #贝叶斯算法来计算概率
errorCount = 0
#测试集分类精度计算
for docIndex in testSet:
wordVector = setOfWords2Vec(vocabList, docList[docIndex])
print("the index %d is classified as: %d, the real class is %d" % (docIndex, classifyNB(array(wordVector), p0V, p1V, pSpam), classList[docIndex]))
if classifyNB(array(wordVector),p0V,p1V,pSpam) != classList[docIndex]:
errorCount += 1
print('the error rate is: ',float(errorCount)/len(testSet))
创建一个包含所有文档中出现的不重复单词列表
def createVocabList(dataSet):
vocabSet = set([]) #创建空集合
for document in dataSet:
vocabSet = vocabSet | set(document) #返回不重复的单词集合
#print(vocabSet)
return list(vocabSet)
def setOfWords2Vec(vocabList, inputSet):
returnVec = [0]*len(vocabList)
for word in inputSet:
if word in vocabList:
returnVec[vocabList.index(word)] = 1
else:print("the word: %s is not in my Vocabulary!" %word)
return returnVec
贝叶斯概率计算
#trainMatrix为输入的词条集合,trainCategory为词条类别
def trainNB0(trainMatrix, trainCategory):
numTrainDocs = len(trainMatrix) #获取词条长度,即分母变量
numWords = len(trainMatrix[0]) #第一段词条中单词个数,即分子变量
pAbusive = sum(trainCategory)/float(numTrainDocs)
p0Num = zeros(numWords);p1Num = zeros(numWords)
p0Denom = 0.0; p1Denom = 0.0
for i in range(numTrainDocs):
if trainCategory[i] == 1:
p1Num += trainMatrix[i]
p1Denom += sum(trainMatrix[i])
else:
p0Num += trainMatrix[i]
p0Denom += sum(trainMatrix[i])
p1Vect = p1Num/p1Denom
p0Vect = p0Num/p0Denom
return p0Vect,p1Vect,pAbusive
def setOfWords2Vec(vocabList, inputSet):
returnVec = [0]*len(vocabList)
for word in inputSet:
if word in vocabList:
returnVec[vocabList.index(word)] = 1
else:print("the word: %s is not in my Vocabulary!" %word)
return returnVec
#分类,取概率高的值
def classifyNB(vec2Classify, p0Vec, p1Vec, pClass1):
p1 = sum(vec2Classify * p1Vec) + log(pClass1)
p0 = sum(vec2Classify * p0Vec) + log(1.0 - pClass1)
if p1 > p0:
return 1
else:
return 0
分类结果:
代码中测试集来自于机器学习实战官网