【吐槽】
啊，代码，你这个大猪蹄子
自己写了cifar10的数据接口，跟官方接口load的数据一样，
沾沾自喜，以为自己会写数据接口了
几天之后，突然想，自己的代码为啥有点慢呢，这数据集不大啊
用了官方接口，真快啊。。。
啊啊啊啊啊啊啊啊
但这是好事，至少我明白了一点知识对吧
【lesson】
看了cifar10的接口，发现自己在数据集初始化的地方写的太少了，应该在初始化的时候就把所有数据读进来，这样的话在__getitem__的时候才能快。
人家的初始化：
if self.train: self.train_data = [] self.train_labels = [] for fentry in self.train_list: f = fentry[0] file = os.path.join(self.root, self.base_folder, f) fo = open(file, 'rb') if sys.version_info[0] == 2: entry = pickle.load(fo) else: entry = pickle.load(fo, encoding='latin1') self.train_data.append(entry['data']) if 'labels' in entry: self.train_labels += entry['labels'] else: self.train_labels += entry['fine_labels'] fo.close() self.train_data = np.concatenate(self.train_data) self.train_data = self.train_data.reshape((50000, 3, 32, 32)) self.train_data = self.train_data.transpose((0, 2, 3, 1)) # convert to HWC
人家的getitem
def __getitem__(self, index): """ Args: index (int): Index Returns: tuple: (image, target) where target is index of the target class. """ if self.train: img, target = self.train_data[index], self.train_labels[index] else: img, target = self.test_data[index], self.test_labels[index] # doing this so that it is consistent with all other datasets # to return a PIL Image img = Image.fromarray(img) if self.transform is not None: img = self.transform(img) if self.target_transform is not None: target = self.target_transform(target) return img, target
自己：（都写到getitem里面了）
def __init__(self, root, transforms=transform(), train=True, test=False): self.root = root self.transform = transforms self.train = train self.test = test if self.test: self.train = False def __getitem__(self, item): x = math.floor(item / 10000) + 1 y = item % 10000 if not self.train and not self.test: x = 5 y = 5000+item imgpath = os.path.join(self.root, "data_batch_"+str(x)) with open(imgpath, 'rb') as fo: dict = pickle.load(fo, encoding='bytes') d_decode = {} for k,v in dict.items(): d_decode[k.decode('utf8')] = v dict = d_decode data = dict['data'][y] # 3*32*32==3072 data = np.reshape(data,(3,32,32)) data = data.transpose(1,2,0) data = self.transform(data) label = dict['labels'][y] # label = torch.from_numpy(label) return data, label
附自己的代码和人家的代码全部
人家：
1 base_folder = 'cifar-10-batches-py' 2 url = "https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz" 3 filename = "cifar-10-python.tar.gz" 4 tgz_md5 = 'c58f30108f718f92721af3b95e74349a' 5 train_list = [ 6 ['data_batch_1', 'c99cafc152244af753f735de768cd75f'], 7 ['data_batch_2', 'd4bba439e000b95fd0a9bffe97cbabec'], 8 ['data_batch_3', '54ebc095f3ab1f0389bbae665268c751'], 9 ['data_batch_4', '634d18415352ddfa80567beed471001a'], 10 ['data_batch_5', '482c414d41f54cd18b22e5b47cb7c3cb'], 11 ] 12 13 test_list = [ 14 ['test_batch', '40351d587109b95175f43aff81a1287e'], 15 ] 16 17 def __init__(self, root, train=True, 18 transform=None, target_transform=None, 19 download=False): 20 self.root = os.path.expanduser(root) 21 self.transform = transform 22 self.target_transform = target_transform 23 self.train = train # training set or test set 24 25 if download: 26 self.download() 27 28 if not self._check_integrity(): 29 raise RuntimeError('Dataset not found or corrupted.' + 30 ' You can use download=True to download it') 31 32 # now load the picked numpy arrays 33 if self.train: 34 self.train_data = [] 35 self.train_labels = [] 36 for fentry in self.train_list: 37 f = fentry[0] 38 file = os.path.join(self.root, self.base_folder, f) 39 fo = open(file, 'rb') 40 if sys.version_info[0] == 2: 41 entry = pickle.load(fo) 42 else: 43 entry = pickle.load(fo, encoding='latin1') 44 self.train_data.append(entry['data']) 45 if 'labels' in entry: 46 self.train_labels += entry['labels'] 47 else: 48 self.train_labels += entry['fine_labels'] 49 fo.close() 50 51 self.train_data = np.concatenate(self.train_data) 52 self.train_data = self.train_data.reshape((50000, 3, 32, 32)) 53 self.train_data = self.train_data.transpose((0, 2, 3, 1)) # convert to HWC 54 else: 55 f = self.test_list[0][0] 56 file = os.path.join(self.root, self.base_folder, f) 57 fo = open(file, 'rb') 58 if sys.version_info[0] == 2: 59 entry = pickle.load(fo) 60 else: 61 entry = pickle.load(fo, encoding='latin1') 62 self.test_data = entry['data'] 63 if 'labels' in entry: 64 self.test_labels = entry['labels'] 65 else: 66 self.test_labels = entry['fine_labels'] 67 fo.close() 68 self.test_data = self.test_data.reshape((10000, 3, 32, 32)) 69 self.test_data = self.test_data.transpose((0, 2, 3, 1)) # convert to HWC 70 71 def __getitem__(self, index): 72 """ 73 Args: 74 index (int): Index 75 76 Returns: 77 tuple: (image, target) where target is index of the target class. 78 """ 79 if self.train: 80 img, target = self.train_data[index], self.train_labels[index] 81 else: 82 img, target = self.test_data[index], self.test_labels[index] 83 84 # doing this so that it is consistent with all other datasets 85 # to return a PIL Image 86 img = Image.fromarray(img) 87 88 if self.transform is not None: 89 img = self.transform(img) 90 91 if self.target_transform is not None: 92 target = self.target_transform(target) 93 94 return img, target
