import collections
import tensorflow as tf
def _read_words(filename):
with tf.gfile.GFile(filename, "r") as f:
return f.read().decode("utf-8").replace("\n", "<eos>").split()
def _build_vocab(filename):
data = _read_words(filename)
counter = collections.Counter(data)
count_pairs = sorted(counter.items(), key=lambda x: (-x[1], x[0]))
words, _ = list(zip(*count_pairs))
word_to_id = dict(zip(words, range(len(words))))
return word_to_id
摘自
https://github.com/tensorflow/models/blob/master/tutorials/rnn/ptb/reader.py
经典的把一篇英文文章转成word2id形式的dict的一段python程序
猜你喜欢
转载自blog.csdn.net/jdbc/article/details/80755593
今日推荐
周排行