文本相似在问答系统中有很重要的应用,如基于知识的问答系统(Knowledge-based QA),基于文档的问答系统(Documen-based QA),以及基于FAQ的问答系统(Community-QA)等。像 对于问题的内容,需要进行相似度匹配,从而选择出与问题最接近,同时最合理的答案。本节介绍 基于Jaccard相似度。
算法描述:两句子分词后词语的交集中词语数与并集中词语数之比。
import os
import jieba
import pickle
import logging
import numpy as np
class StopWords(object):
'''
'''
def __init__(self, stopwords_file=stopwords_file ):
self.stopwords = set( [ word.strip() for word in open(stopwords_file, 'r') ] )
def del_stopwords(self, words):
return [ word for word in words if word not in self.stopwords ]
stop_word = StopWords()
# 是否分词、 及其停用词语
def _seg_word(words, jieba_flag=True, del_stopword=False):
if jieba_flag:
word_list = [stop_word.del_stopwords(words) if del_stopword else word for word in jieba.cut(words)]
else:
word_list = [stop_word.del_stopwords(words) if del_stopword else word for word in words]
return word_list
def sim_jaccard(s1, s2):
"""jaccard相似度"""
s1, s2 = set(s1), set(s2)
ret1 = s1.intersection(s2) # 交集
ret2 = s1.union(s2) # 并集
sim = 1.0 * len(ret1) / len(ret2)
return sim
word1 = ['这是什么']
word2 = ['这个什么价钱']
word_sim = sim_jaccard( _seg_word(word1), _seg_word(word2) )