flowers.py 7.6 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15
# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
This module will download dataset from
16
http://www.robots.ox.ac.uk/~vgg/data/flowers/102/index.html
17 18
and parse train/test set intopaddle reader creators.

19
This set contains images of flowers belonging to 102 different categories.
20 21 22 23 24 25
The images were acquired by searching the web and taking pictures. There are a
minimum of 40 images for each category.

The database was used in:

Nilsback, M-E. and Zisserman, A. Automated flower classification over a large
26 27
 number of classes.Proceedings of the Indian Conference on Computer Vision,
Graphics and Image Processing (2008)
28 29 30
http://www.robots.ox.ac.uk/~vgg/publications/papers/nilsback08.{pdf,ps.gz}.

"""
31 32 33

from __future__ import print_function

34
import itertools
35
import functools
36
from .common import download
37
import tarfile
38
import six
39
import scipy.io as scio
40 41
from paddle.dataset.image import *
from paddle.reader import *
42 43
import os
import numpy as np
44
from multiprocessing import cpu_count
45 46
from six.moves import cPickle as pickle
from six.moves import zip
47 48
__all__ = ['train', 'test', 'valid']

49 50 51 52
DATA_URL = 'http://paddlemodels.cdn.bcebos.com/flowers/102flowers.tgz'
LABEL_URL = 'http://paddlemodels.cdn.bcebos.com/flowers/imagelabels.mat'
SETID_URL = 'http://paddlemodels.cdn.bcebos.com/flowers/setid.mat'
DATA_MD5 = '52808999861908f626f3c1f4e79d11fa'
53 54
LABEL_MD5 = 'e0620be6f572b9609742df49c70aed4d'
SETID_MD5 = 'a5357ecc9cb78c4bef273ce3793fc85c'
55 56 57 58 59 60
# In official 'readme', tstid is the flag of test data
# and trnid is the flag of train data. But test data is more than train data.
# So we exchange the train data and test data.
TRAIN_FLAG = 'tstid'
TEST_FLAG = 'trnid'
VALID_FLAG = 'valid'
61 62


63
def default_mapper(is_train, sample):
64 65 66 67
    '''
    map image bytes data to type needed by model input layer
    '''
    img, label = sample
68
    img = load_image_bytes(img)
69
    img = simple_transform(
D
dangqingqing 已提交
70
        img, 256, 224, is_train, mean=[103.94, 116.78, 123.68])
71 72 73
    return img.flatten().astype('float32'), label


74 75 76 77
train_mapper = functools.partial(default_mapper, True)
test_mapper = functools.partial(default_mapper, False)


78 79 80
def reader_creator(data_file,
                   label_file,
                   setid_file,
81
                   dataset_name,
82
                   mapper,
83
                   buffered_size=1024,
84 85
                   use_xmap=True,
                   cycle=False):
86
    '''
87
    1. read images from tar file and
88 89
        merge images into batch files in 102flowers.tgz_batch/
    2. get a reader to read sample from batch file
90 91

    :param data_file: downloaded data file
92
    :type data_file: string
93
    :param label_file: downloaded label file
94 95 96 97
    :type label_file: string
    :param setid_file: downloaded setid file containing information
                        about how to split dataset
    :type setid_file: string
98 99
    :param dataset_name: data set name (tstid|trnid|valid)
    :type dataset_name: string
100
    :param mapper: a function to map image bytes data to type
101 102
                    needed by model input layer
    :type mapper: callable
103 104
    :param buffered_size: the size of buffer used to process images
    :type buffered_size: int
105 106
    :param cycle: whether to cycle through the dataset
    :type cycle: bool
107 108 109
    :return: data reader
    :rtype: callable
    '''
110 111 112 113 114 115 116
    labels = scio.loadmat(label_file)['labels'][0]
    indexes = scio.loadmat(setid_file)[dataset_name][0]
    img2label = {}
    for i in indexes:
        img = "jpg/image_%05d.jpg" % i
        img2label[img] = labels[i - 1]
    file_list = batch_images_from_tar(data_file, dataset_name, img2label)
117 118

    def reader():
119 120 121 122
        while True:
            for file in open(file_list):
                file = file.strip()
                batch = None
123
                with open(file, 'rb') as f:
124 125 126 127
                    if six.PY2:
                        batch = pickle.load(f)
                    else:
                        batch = pickle.load(f, encoding='bytes')
128 129
                data = batch['data']
                labels = batch['label']
130
                for sample, label in zip(data, batch['label']):
131 132 133
                    yield sample, int(label) - 1
            if not cycle:
                break
134

135
    if use_xmap:
C
chengduoZH 已提交
136 137
        cpu_num = int(os.environ.get('CPU_NUM', cpu_count()))
        return xmap_readers(mapper, reader, cpu_num, buffered_size)
138 139
    else:
        return map_readers(mapper, reader)
140 141


142
def train(mapper=train_mapper, buffered_size=1024, use_xmap=True, cycle=False):
143
    '''
144 145 146
    Create flowers training set reader.
    It returns a reader, each sample in the reader is
    image pixels in [0, 1] and label in [1, 102]
147 148 149 150 151 152
    translated from original color image by steps:
    1. resize to 256*256
    2. random crop to 224*224
    3. flatten
    :param mapper:  a function to map sample.
    :type mapper: callable
153 154
    :param buffered_size: the size of buffer used to process images
    :type buffered_size: int
155 156
    :param cycle: whether to cycle through the dataset
    :type cycle: bool
157 158 159 160 161 162
    :return: train data reader
    :rtype: callable
    '''
    return reader_creator(
        download(DATA_URL, 'flowers', DATA_MD5),
        download(LABEL_URL, 'flowers', LABEL_MD5),
163 164 165 166 167 168
        download(SETID_URL, 'flowers', SETID_MD5),
        TRAIN_FLAG,
        mapper,
        buffered_size,
        use_xmap,
        cycle=cycle)
169 170


171
def test(mapper=test_mapper, buffered_size=1024, use_xmap=True, cycle=False):
172
    '''
173 174 175
    Create flowers test set reader.
    It returns a reader, each sample in the reader is
    image pixels in [0, 1] and label in [1, 102]
176 177 178 179 180 181
    translated from original color image by steps:
    1. resize to 256*256
    2. random crop to 224*224
    3. flatten
    :param mapper:  a function to map sample.
    :type mapper: callable
182 183
    :param buffered_size: the size of buffer used to process images
    :type buffered_size: int
184 185
    :param cycle: whether to cycle through the dataset
    :type cycle: bool
186 187 188 189 190 191
    :return: test data reader
    :rtype: callable
    '''
    return reader_creator(
        download(DATA_URL, 'flowers', DATA_MD5),
        download(LABEL_URL, 'flowers', LABEL_MD5),
192 193 194 195 196 197
        download(SETID_URL, 'flowers', SETID_MD5),
        TEST_FLAG,
        mapper,
        buffered_size,
        use_xmap,
        cycle=cycle)
198 199


200
def valid(mapper=test_mapper, buffered_size=1024, use_xmap=True):
201
    '''
202 203 204
    Create flowers validation set reader.
    It returns a reader, each sample in the reader is
    image pixels in [0, 1] and label in [1, 102]
205 206 207 208
    translated from original color image by steps:
    1. resize to 256*256
    2. random crop to 224*224
    3. flatten
209 210 211 212 213 214
    :param mapper:  a function to map sample.
    :type mapper: callable
    :param buffered_size: the size of buffer used to process images
    :type buffered_size: int
    :return: test data reader
    :rtype: callable
215 216 217 218
    '''
    return reader_creator(
        download(DATA_URL, 'flowers', DATA_MD5),
        download(LABEL_URL, 'flowers', LABEL_MD5),
219 220
        download(SETID_URL, 'flowers', SETID_MD5), VALID_FLAG, mapper,
        buffered_size, use_xmap)
221 222 223 224 225 226


def fetch():
    download(DATA_URL, 'flowers', DATA_MD5)
    download(LABEL_URL, 'flowers', LABEL_MD5)
    download(SETID_URL, 'flowers', SETID_MD5)
新手
引导
客服 返回
顶部