From 179be88c57df299ba6497572aa2ef8c7ff31451d Mon Sep 17 00:00:00 2001 From: zhengya01 <43601548+zhengya01@users.noreply.github.com> Date: Mon, 21 Jan 2019 21:41:32 +0800 Subject: [PATCH] Delete reader_ce.py --- fluid/PaddleCV/metric_learning/reader_ce.py | 177 -------------------- 1 file changed, 177 deletions(-) delete mode 100644 fluid/PaddleCV/metric_learning/reader_ce.py diff --git a/fluid/PaddleCV/metric_learning/reader_ce.py b/fluid/PaddleCV/metric_learning/reader_ce.py deleted file mode 100644 index 06e37c8f..00000000 --- a/fluid/PaddleCV/metric_learning/reader_ce.py +++ /dev/null @@ -1,177 +0,0 @@ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import os -import math -import random -import functools -import numpy as np -import paddle -from imgtool_ce import process_image - -random.seed(0) - -DATA_DIR = "./data/Stanford_Online_Products/" -TRAIN_LIST = './data/Stanford_Online_Products/Ebay_train.txt' -VAL_LIST = './data/Stanford_Online_Products/Ebay_test.txt' - - -def init_sop(mode): - if mode == 'train': - train_data = {} - train_image_list = [] - train_list = open(TRAIN_LIST, "r").readlines() - for i, item in enumerate(train_list): - items = item.strip().split() - if items[0] == 'image_id': - continue - path = items[3] - label = int(items[1]) - 1 - train_image_list.append((path, label)) - if label not in train_data: - train_data[label] = [] - train_data[label].append(path) - #random.shuffle(train_image_list) - print("{} dataset size: {}".format(mode, len(train_data))) - return train_data, train_image_list - else: - val_data = {} - val_image_list = [] - test_image_list = [] - val_list = open(VAL_LIST, "r").readlines() - for i, item in enumerate(val_list): - items = item.strip().split() - if items[0] == 'image_id': - continue - path = items[3] - label = int(items[1]) - val_image_list.append((path, label)) - test_image_list.append(path) - if label not in val_data: - val_data[label] = [] - val_data[label].append(path) - print("{} dataset size: {}".format(mode, len(val_data))) - if mode == 'val': - return val_data, val_image_list - else: - return test_image_list - -def common_iterator(data, settings): - batch_size = settings.train_batch_size - samples_each_class = settings.samples_each_class - assert (batch_size % samples_each_class == 0) - class_num = batch_size // samples_each_class - def train_iterator(): - labs = list(data.keys()) - lab_num = len(labs) - ind = list(range(0, lab_num)) - while True: - #random.shuffle(ind) - ind_sample = ind[:class_num] - for ind_i in ind_sample: - lab = labs[ind_i] - data_list = data[lab] - data_ind = list(range(0, len(data_list))) - #random.shuffle(data_ind) - anchor_ind = data_ind[:samples_each_class] - - for anchor_ind_i in anchor_ind: - anchor_path = DATA_DIR + data_list[anchor_ind_i] - yield anchor_path, lab - - return train_iterator - -def triplet_iterator(data, settings): - batch_size = settings.train_batch_size - assert (batch_size % 3 == 0) - def train_iterator(): - labs = list(data.keys()) - lab_num = len(labs) - ind = list(range(0, lab_num)) - while True: - #random.shuffle(ind) - ind_pos, ind_neg = ind[:2] - lab_pos = labs[ind_pos] - pos_data_list = data[lab_pos] - data_ind = list(range(0, len(pos_data_list))) - #random.shuffle(data_ind) - anchor_ind, pos_ind = data_ind[:2] - - lab_neg = labs[ind_neg] - neg_data_list = data[lab_neg] - #neg_ind = random.randint(0, len(neg_data_list) - 1) - neg_ind = 1 - - anchor_path = DATA_DIR + pos_data_list[anchor_ind] - yield anchor_path, lab_pos - pos_path = DATA_DIR + pos_data_list[pos_ind] - yield pos_path, lab_pos - neg_path = DATA_DIR + neg_data_list[neg_ind] - yield neg_path, lab_neg - - return train_iterator - -def arcmargin_iterator(data, settings): - def train_iterator(): - while True: - for items in data: - path, label = items - path = DATA_DIR + path - yield path, label - return train_iterator - -def image_iterator(data, mode): - def val_iterator(): - for items in data: - path, label = items - path = DATA_DIR + path - yield path, label - def test_iterator(): - for item in data: - path = item - path = DATA_DIR + path - yield [path] - if mode == 'val': - return val_iterator - else: - return test_iterator - -def createreader(settings, mode): - def metric_reader(): - if mode == 'train': - train_data, train_image_list = init_sop('train') - loss_name = settings.loss_name - if loss_name in ["softmax", "arcmargin"]: - return arcmargin_iterator(train_image_list, settings)() - elif loss_name == 'triplet': - return triplet_iterator(train_data, settings)() - else: - return common_iterator(train_data, settings)() - elif mode == 'val': - val_data, val_image_list = init_sop('val') - return image_iterator(val_image_list, 'val')() - else: - test_image_list = init_sop('test') - return image_iterator(test_image_list, 'test')() - - image_shape = settings.image_shape.split(',') - assert(image_shape[1] == image_shape[2]) - image_size = int(image_shape[2]) - #keep_order = False if mode != 'train' or settings.loss_name in ['softmax', 'arcmargin'] else True - keep_order = True - image_mapper = functools.partial(process_image, - mode=mode, color_jitter=False, rotate=False, crop_size=image_size) - reader = paddle.reader.xmap_readers( - image_mapper, metric_reader, 8, 1000, order=keep_order) - return reader - - -def train(settings): - return createreader(settings, "train") - -def test(settings): - return createreader(settings, "val") - -def infer(settings): - return createreader(settings, "test") -- GitLab