1 import numpy as np
  2 import os
  3 import sys
  4 import pickle
  5 import collections
  6 from sklearn.feature_extraction.text import TfidfVectorizer
  7 from sklearn.decomposition import TruncatedSVD
  8 from gensim import corpora, models, matutils
  9 from utils.tokenizer import load_stopwords, load_samples, tokenizer, word_segment, load_data, read_data_to_list
 10 from gensim.models import doc2vec, Doc2Vec
 11 from sklearn.metrics.pairwise import cosine_similarity
 12 
 13 '''
 14 大体流程：
 15 input：doc vector;threshold
 16 output:cluster
 17 begin
 18     input doc vector
 19     input threshold
 20     first doc as first cluster and it's vector as the center of the cluster
 21     while(doc vectors){
 22         while(clusters){
 23             max_sim,max_cluster = simlarity(doc vector,cluster);
 24         }
 25         if(max_sim > threshold){
 26             max_cluster.put(doc vector);
 27             max_cluster.update_center()
 28         }
 29         else{
 30             build new cluster(doc vector);
 31         }
 32     }
 33 end
 34 '''
 35 class SingelPassCluster(object):
 36 
 37     '''
 38         1.利用tfidf vec计算cossim
 39     '''
 40     def tfidf_vec(self, corpus, pivot=10, slope=0.25):
 41         dictionary = corpora.Dictionary(corpus)  # 形成词典映射
 42         self.dict_size = len(dictionary)
 43         print('dictionary size:{}'.format(len(dictionary)))
 44         corpus = [dictionary.doc2bow(text) for text in corpus]  # 词的向量表示
 45         tfidf = models.TfidfModel(corpus, pivot=pivot, slope=slope)
 46         corpus_tfidf = tfidf[corpus]
 47         return corpus_tfidf
 48 
 49     def get_max_similarity(self, cluster_cores, vector):
 50         max_value = 0
 51         max_index = -1
 52         print('vector:{}'.format(vector))
 53         for k, core in cluster_cores.items():
 54             print('core:{}'.format(core))
 55             similarity = matutils.cossim(vector, core)
 56             if similarity > max_value:
 57                 max_value = similarity
 58                 max_index = k
 59         return max_index, max_value
 60 
 61     def single_pass(self, corpus_vec, corpus, theta):
 62         clusters = {}
 63         cluster_cores = {}
 64         cluster_text = {}
 65         num_topic = 0
 66         cnt = 0
 67         for vector, text in zip(corpus_vec, corpus):
 68             if num_topic == 0:
 69                 clusters.setdefault(num_topic, []).append(vector)
 70                 cluster_cores[num_topic] = vector
 71                 cluster_text.setdefault(num_topic, []).append(text)
 72                 num_topic += 1
 73             else:
 74                 max_index, max_value = self.get_max_similarity(cluster_cores, vector)
 75                 if max_value > theta:
 76                     clusters[max_index].append(vector)
 77                     text_matrix = matutils.corpus2dense(clusters[max_index], num_terms=self.dict_size,
 78                                                         num_docs=len(clusters[max_index])).T  # 稀疏转稠密
 79                     core = np.mean(text_matrix, axis=0)  # 更新簇中心
 80                     core = matutils.any2sparse(core)  # 将稠密向量core转为稀疏向量
 81                     cluster_cores[max_index] = core
 82                     cluster_text[max_index].append(text)
 83                 else:  # 创建一个新簇
 84                     clusters.setdefault(num_topic, []).append(vector)
 85                     cluster_cores[num_topic] = vector
 86                     cluster_text.setdefault(num_topic, []).append(text)
 87                     num_topic += 1
 88             cnt += 1
 89             if cnt % 100 == 0:
 90                 print('processing {}...'.format(cnt))
 91         return clusters, cluster_text
 92 
 93     def fit_transform(self, corpus, raw_data, theta=0.5):
 94         tfidf_vec = self.tfidf_vec(corpus)  # tfidf_vec是稀疏向量
 95         clusters, cluster_text = self.single_pass(tfidf_vec, raw_data, theta)
 96         return clusters, cluster_text
 97 
 98 
 99     '''
100         2.利用doc2vec计算cossim
101     '''
102     def fit(self, doc2vec_model, corpus, raw_data, theta=0.5):
103         doc_vec = self.doc_vec(doc2vec_model, corpus)
104         clusters, cluster_text = self.doc2vec_single_pass(doc_vec, raw_data, theta)
105         return clusters, cluster_text
106 
107     def fit_2(self, doc_vec, text2index, theta):
108         clusters, cluster_text = self.doc2vec_single_pass(doc_vec, text2index, theta)
109         return clusters, cluster_text
110 
111     def doc_vec(self, doc2vec_model, x_train):
112         print('doc2vec infered vec...')
113         infered_vectors_list = []
114         for text, label in x_train:
115             vector = doc2vec_model.infer_vector(text)
116             infered_vectors_list.append(vector)
117             print('infered vector size:{}'.format(len(infered_vectors_list)))
118             if len(infered_vectors_list) >= 100:
119                 break
120         return infered_vectors_list
121 
122     def get_doc2vec_similarity(self, cluster_cores, vector):
123         max_value = 0
124         max_index = -1
125         for k, core in cluster_cores.items():  # core -> np.ndarray
126             similarity = cosine_similarity(vector.reshape(1, -1), core.reshape(1, -1))
127             similarity = similarity[0, 0]
128             if similarity > max_value:
129                 max_value = similarity
130                 max_index = k
131         return max_index, max_value
132 
133     def doc2vec_single_pass(self, corpus_vec, corpus, theta):
134         clusters = {}
135         cluster_cores = {}
136         cluster_text = {}
137         num_topic = 0
138         cnt = 0
139         for vector, text in zip(corpus_vec, corpus):
140             if num_topic == 0:
141                 clusters.setdefault(num_topic, []).append(vector)
142                 cluster_cores[num_topic] = vector
143                 cluster_text.setdefault(num_topic, []).append(text)
144                 num_topic += 1
145             else:
146                 max_index, max_value = self.get_doc2vec_similarity(cluster_cores, vector)
147                 if max_value > theta:
148                     clusters[max_index].append(vector)
149                     core = np.mean(clusters[max_index], axis=0)  # 更新簇中心
150                     cluster_cores[max_index] = core
151                     cluster_text[max_index].append(text)
152                 else:  # 创建一个新簇
153                     clusters.setdefault(num_topic, []).append(vector)
154                     cluster_cores[num_topic] = vector
155                     cluster_text.setdefault(num_topic, []).append(text)
156                     num_topic += 1
157             cnt += 1
158             if cnt % 100 == 0:
159                 print('processing {}...'.format(cnt))
160         return clusters, cluster_text
161 
162 
163 def sim(doc_vec):
164     vector = doc_vec[0]
165     print('vector:{}'.format(type(vector)))
166     for core in doc_vec:
167         similarity = cosine_similarity(vector.reshape(1,-1), core.reshape(1,-1))
168         similarity = similarity[0, 0]
169         print("similarity:{}".format(similarity))
170 
171 if __name__ == '__main__':
172     base_path = os.path.abspath(os.path.join(os.getcwd(), '../..'))
173     process_text = base_path + '/data/process_text.txt'  # 处理后的样本路径
174     doc2vec_path = base_path + '/data/doc2vec.pkl'
175     cluster_result = base_path + '/data/cluster_result.txt'
176     doc_vec_path = base_path + '/data/doc_vec.vec'  # 经过doc2vec推荐的文本向量
177 
178     corpus = load_data(process_text)
179     raw_text = load_samples(process_text)
180 
181     index2corpus = collections.OrderedDict()
182     for index, line in enumerate(raw_text):
183         index2corpus[index] = line
184     text2index = list(index2corpus.keys())
185     print('docs total size:{}'.format(len(text2index)))
186 
187     single_cluster = SingelPassCluster()
188 
189     cal_vec_type = 'doc2vec'
190 
191     if cal_vec_type == 'tfidf':
192         clusters, cluster_text = single_cluster.fit_transform(corpus, text2index, theta=0.4)
193 
194     if cal_vec_type == 'doc2vec':
195         with open(doc_vec_path, 'rb') as file:
196             infered_vectors_list = pickle.load(file)
197         clusters, cluster_text = single_cluster.fit_2(infered_vectors_list, text2index, theta=0.6)
198 
199         '''
200         if os.path.exists(doc2vec_path):
201             print('doc2vec model loading...')
202             doc2vec_model = Doc2Vec.load(doc2vec_path)
203         x_train = read_data_to_list(process_text)
204         clusters, cluster_text = single_cluster.fit(doc2vec_model, x_train, text2index, theta=0.6)
205         '''
206 
207     if cal_vec_type == 'd2vsim':
208         if os.path.exists(doc2vec_path):
209             print('doc2vec model loading...')
210             doc2vec_model = Doc2Vec.load(doc2vec_path)
211         x_train = read_data_to_list(process_text)
212         doc_vec = single_cluster.doc_vec(doc2vec_model, x_train)
213         sim(doc_vec)
214 
215 
216     print("............................................................................................")
217     print("得到的类数量有: {} 个 ...".format(len(clusters)))
218     print("............................................................................................\n")
219     # 按聚类语句数量对聚类结果进行降序排列
220     clusterTopic_list = sorted(cluster_text.items(), key=lambda x: len(x[1]), reverse=True)
221     with open(cluster_result, 'w', encoding='utf-8') as file_write:
222         for k in clusterTopic_list:
223             cluster_text = []
224             for index, value in enumerate(k[1],start=1):
225                 cluster_text.append('(' + str(index) + '): ' + index2corpus[value])
226             cluster_text = '\n'.join(cluster_text)
227             file_write.write("【簇索引】:{} \n【簇中文档数】：{} \n【簇中文档】 ：\n{}".format(k[0], len(k[1]), cluster_text))
228             file_write.write('\n')
229             file_write.flush()