**目的:**将1.6万条数据随机切分成训练集、开发集、测试集。
参考了python随机读取txt文档某一行的源码,源码如下:
#coding=utf-8
#! /usr/bin/python
import random
import linecache
def hello():
count = len(open('hello.txt','rU').readlines())#获取行数
hellonum=random.randrange(1,count, 1)#生成随机行数
return linecache.getline('hello.txt',hellonum)#随机读取某行
if __name__ == "__main__":
hello()
基于上述源码实现了随机切分实验数据。源码如下:
import random
import linecache
m=0
with open(r'./data/sz_text_all.txt','rb') as f:
data = f.read().decode('utf-8') #python3一定要加上这句不然会编码报错!
#获取txt的总行数!
n = data.count('\n')
print("总行数",n)
#创建字典,将生成的随机数存入该字典
num_dict = {}
#选取随机的数
while m <= n:
i = random.randint(1, (n+1))
if i in num_dict :continue
num_dict[i] = i
print("本次使用的行数",i)
###得到对应的i行的数据
line=linecache.getline(r'./data/sz_text_all.txt',i)
if m<=500:#切分500条开发集
with open('./dev.txt','a' ,encoding='utf-8') as dev_f:
dev_f.write(line)
#print(line)
elif m>500 and m<=1000:#500条测试集
with open('./test.txt','a' ,encoding='utf-8') as test_f:
test_f.write(line)
#print(line)
else :#随机切分1.5万条训练集
with open('./train.txt','a' ,encoding='utf-8') as train_f:
train_f.write(line)
#print(line)
m+=1
print("done!")
参考:http://www.etwiki.cn/python/634.html
https://www.cnblogs.com/botoo/p/7356737.html