版权声明:本文为博主原创文章,未经博主允许不得转载。 https://blog.csdn.net/shaoyou223/article/details/79143833
最近在看Hands-on machine learning with scikit-learn and tensorflow,在第三章加载MNIST数据时,出现了问题。
书中给的代码是:
from sklearn.datasets import fetch_mldata
mnist = fetch_mldata('MNIST original')
我的电脑是win7,64位,用的是python3.5。在加载时无法读取bytes。所以在网上寻找办法,大多都不行,包括本书github里的解决办法页没能解决。最后在简书里找到了一份代码,亲测可用。
下面这份文件可以在官网上找到,其他三个文件也是。
train-images.idx3-ubyte
以下是经过我修改可以返回数据集的完整代码:
import numpy as np import struct import matplotlib.pyplot as plt # 训练集文件 train_images_idx3_ubyte_file = 'C:\\Users\\Administrator\\PycharmProjects\\untitled\\python文件包\\Hands-on machine learning with scikit-learn&tensorflow\\第三章\\mnist\\train-images.idx3-ubyte' # 训练集标签文件 train_labels_idx1_ubyte_file = 'C:\\Users\\Administrator\\PycharmProjects\\untitled\\python文件包\\Hands-on machine learning with scikit-learn&tensorflow\\第三章\\mnist\\train-labels.idx1-ubyte' # 测试集文件 test_images_idx3_ubyte_file = 'C:\\Users\\Administrator\\PycharmProjects\\untitled\\python文件包\\Hands-on machine learning with scikit-learn&tensorflow\\第三章\\mnist\\t10k-images.idx3-ubyte' # 测试集标签文件 test_labels_idx1_ubyte_file = 'C:\\Users\\Administrator\\PycharmProjects\\untitled\\python文件包\\Hands-on machine learning with scikit-learn&tensorflow\\第三章\\mnist\\t10k-labels.idx1-ubyte' def decode_idx3_ubyte(idx3_ubyte_file): """ 解析idx3文件的通用函数 :param idx3_ubyte_file: idx3文件路径 :return: 数据集 """ # 读取二进制数据 bin_data = open(idx3_ubyte_file, 'rb').read() # 解析文件头信息,依次为魔数、图片数量、每张图片高、每张图片宽 offset = 0 fmt_header = '>iiii' magic_number, num_images, num_rows, num_cols = struct.unpack_from(fmt_header, bin_data, offset) print('魔数:%d, 图片数量: %d张, 图片大小: %d*%d' % (magic_number, num_images, num_rows, num_cols)) # 解析数据集 image_size = num_rows * num_cols offset += struct.calcsize(fmt_header) fmt_image = '>' + str(image_size) + 'B' images = np.empty((num_images, num_rows, num_cols)) for i in range(num_images): if (i + 1) % 10000 == 0: print('已解析 %d' % (i + 1) + '张') images[i] = np.array(struct.unpack_from(fmt_image, bin_data, offset)).reshape((num_rows, num_cols)) offset += struct.calcsize(fmt_image) return images def decode_idx1_ubyte(idx1_ubyte_file): """ 解析idx1文件的通用函数 :param idx1_ubyte_file: idx1文件路径 :return: 数据集 """ # 读取二进制数据 bin_data = open(idx1_ubyte_file, 'rb').read() # 解析文件头信息,依次为魔数和标签数 offset = 0 fmt_header = '>ii' magic_number, num_images = struct.unpack_from(fmt_header, bin_data, offset) print('魔数:%d, 图片数量: %d张' % (magic_number, num_images)) # 解析数据集 offset += struct.calcsize(fmt_header) fmt_image = '>B' labels = np.empty(num_images) for i in range(num_images): if (i + 1) % 10000 == 0: print ('已解析 %d' % (i + 1) + '张') labels[i] = struct.unpack_from(fmt_image, bin_data, offset)[0] offset += struct.calcsize(fmt_image) return labels def load_train_images(idx_ubyte_file=train_images_idx3_ubyte_file): """ TRAINING SET IMAGE FILE (train-images-idx3-ubyte): [offset] [type] [value] [description] 0000 32 bit integer 0x00000803(2051) magic number 0004 32 bit integer 60000 number of images 0008 32 bit integer 28 number of rows 0012 32 bit integer 28 number of columns 0016 unsigned byte ?? pixel 0017 unsigned byte ?? pixel ........ xxxx unsigned byte ?? pixel Pixels are organized row-wise. Pixel values are 0 to 255. 0 means background (white), 255 means foreground (black). :param idx_ubyte_file: idx文件路径 :return: n*row*col维np.array对象,n为图片数量 """ return decode_idx3_ubyte(idx_ubyte_file) def load_train_labels(idx_ubyte_file=train_labels_idx1_ubyte_file): """ TRAINING SET LABEL FILE (train-labels-idx1-ubyte): [offset] [type] [value] [description] 0000 32 bit integer 0x00000801(2049) magic number (MSB first) 0004 32 bit integer 60000 number of items 0008 unsigned byte ?? label 0009 unsigned byte ?? label ........ xxxx unsigned byte ?? label The labels values are 0 to 9. :param idx_ubyte_file: idx文件路径 :return: n*1维np.array对象,n为图片数量 """ return decode_idx1_ubyte(idx_ubyte_file) def load_test_images(idx_ubyte_file=test_images_idx3_ubyte_file): """ TEST SET IMAGE FILE (t10k-images-idx3-ubyte): [offset] [type] [value] [description] 0000 32 bit integer 0x00000803(2051) magic number 0004 32 bit integer 10000 number of images 0008 32 bit integer 28 number of rows 0012 32 bit integer 28 number of columns 0016 unsigned byte ?? pixel 0017 unsigned byte ?? pixel ........ xxxx unsigned byte ?? pixel Pixels are organized row-wise. Pixel values are 0 to 255. 0 means background (white), 255 means foreground (black). :param idx_ubyte_file: idx文件路径 :return: n*row*col维np.array对象,n为图片数量 """ return decode_idx3_ubyte(idx_ubyte_file) def load_test_labels(idx_ubyte_file=test_labels_idx1_ubyte_file): """ TEST SET LABEL FILE (t10k-labels-idx1-ubyte): [offset] [type] [value] [description] 0000 32 bit integer 0x00000801(2049) magic number (MSB first) 0004 32 bit integer 10000 number of items 0008 unsigned byte ?? label 0009 unsigned byte ?? label ........ xxxx unsigned byte ?? label The labels values are 0 to 9. :param idx_ubyte_file: idx文件路径 :return: n*1维np.array对象,n为图片数量 """ return decode_idx1_ubyte(idx_ubyte_file) def run(): train_images = load_train_images() train_labels = load_train_labels() test_images = load_test_images() test_labels = load_test_labels() X_train= train_images X_test = test_images y_train = train_labels y_test = test_labels return X_train,X_test,y_train,y_test print('train set label number is '+str(len(y_train))) print('test set label number is '+str(len(y_test))) # 查看前十个数据及其标签以读取是否正确 for i in range(10): print(train_labels[i]) plt.imshow(train_images[i], cmap='gray') plt.show() print('done') if __name__ == '__main__': X_train, X_test, y_train, y_test = run()