我在解决Andrej Karpathy关于神经网络课程的第二个作业(链接)。编程环境是ipython笔记本。当我尝试加载CIFAR10数据时,反复遇到内存错误。我尝试通过谷歌搜索解决方案,但没有任何效果。请在这里帮助我。
from cs231n.data_utils import load_CIFAR10def get_CIFAR10_data(num_training=49000, num_validation=1000, num_test=1000): """ Load the CIFAR-10 dataset from disk and perform preprocessing to prepare it for the two-layer neural net classifier. These are the same steps as we used for the SVM, but condensed to a single function. """ # Load the raw CIFAR-10 data cifar10_dir = 'cs231n/datasets/cifar-10-batches-py' X_train, y_train, X_test, y_test = load_CIFAR10(cifar10_dir) # Subsample the data mask = range(num_training, num_training + num_validation) X_val = X_train[mask] y_val = y_train[mask] mask = range(num_training) X_train = X_train[mask] y_train = y_train[mask] mask = range(num_test) X_test = X_test[mask] y_test = y_test[mask] # Normalize the data: subtract the mean image mean_image = np.mean(X_train, axis=0) X_train -= mean_image X_val -= mean_image X_test -= mean_image # Reshape data to rows X_train = X_train.reshape(num_training, -1) X_val = X_val.reshape(num_validation, -1) X_test = X_test.reshape(num_test, -1) return X_train, y_train, X_val, y_val, X_test, y_test# Invoke the above function to get our data.X_train, y_train, X_val, y_val, X_test, y_test = get_CIFAR10_data()print 'Train data shape: ', X_train.shapeprint 'Train labels shape: ', y_train.shapeprint 'Validation data shape: ', X_val.shapeprint 'Validation labels shape: ', y_val.shapeprint 'Test data shape: ', X_test.shapeprint 'Test labels shape: ', y_test.shape
import cPickle as pickleimport numpy as npimport osdef load_CIFAR_batch(filename): """ load single batch of cifar """ with open(filename, 'rb') as f: datadict = pickle.load(f) X = datadict['data'] Y = datadict['labels'] X = X.reshape(10000, 3, 32, 32).transpose(0,2,3,1).astype("float") Y = np.array(Y) return X, Ydef load_CIFAR10(ROOT): """ load all of cifar """ xs = [] ys = [] for b in range(1,6): f = os.path.join(ROOT, 'data_batch_%d' % (b, )) X, Y = load_CIFAR_batch(f) xs.append(X) ys.append(Y) Xtr = np.concatenate(xs) Ytr = np.concatenate(ys) del X, Y Xte, Yte = load_CIFAR_batch(os.path.join(ROOT, 'test_batch')) return Xtr, Ytr, Xte, Yte
回答:
如果有人在Windows操作系统上遇到相同的问题,请安装x64版本的Python分发。x86版本的内存使用量限制在2GB。