test_decoupled_py_reader.py 6.7 KB
Newer Older
S
sneaxiy 已提交
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20
# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import paddle
import paddle.fluid as fluid
import numpy as np
import time
import unittest

21 22 23
EPOCH_NUM = 5
BATCH_SIZE = 16
BATCH_NUM = 10
S
sneaxiy 已提交
24 25 26 27
CLASS_NUM = 10


def random_reader():
S
sneaxiy 已提交
28
    np.random.seed(1)
29
    for i in range(BATCH_SIZE * BATCH_NUM):
S
sneaxiy 已提交
30
        image = np.random.random([784])
31
        label = np.random.randint(low=0, high=CLASS_NUM)
S
sneaxiy 已提交
32 33 34
        yield image, label


S
sneaxiy 已提交
35
def simple_fc_net(places, use_legacy_py_reader, use_double_buffer):
C
cnn 已提交
36
    paddle.seed(1)
L
Leo Chen 已提交
37
    paddle.framework.random._manual_program_seed(1)
S
sneaxiy 已提交
38 39 40 41 42
    startup_prog = fluid.Program()
    main_prog = fluid.Program()

    with fluid.unique_name.guard():
        with fluid.program_guard(main_prog, startup_prog):
43 44 45
            image = fluid.layers.data(
                name='image', shape=[784], dtype='float32'
            )
S
sneaxiy 已提交
46
            label = fluid.layers.data(name='label', shape=[1], dtype='int64')
47 48 49 50 51 52
            py_reader = fluid.io.PyReader(
                feed_list=[image, label],
                capacity=4,
                iterable=not use_legacy_py_reader,
                use_double_buffer=use_double_buffer,
            )
S
sneaxiy 已提交
53 54 55 56 57 58 59
            hidden = image
            for hidden_size in [10, 20, 30]:
                hidden = fluid.layers.fc(
                    hidden,
                    size=hidden_size,
                    act='tanh',
                    bias_attr=fluid.ParamAttr(
60 61 62
                        initializer=fluid.initializer.Constant(value=1.0)
                    ),
                )
S
sneaxiy 已提交
63

64 65 66
            predict_label = fluid.layers.fc(
                hidden, size=CLASS_NUM, act='softmax'
            )
67
            loss = paddle.mean(
68 69
                fluid.layers.cross_entropy(input=predict_label, label=label)
            )
S
sneaxiy 已提交
70 71 72 73 74 75 76

            optimizer = fluid.optimizer.Adam()
            optimizer.minimize(loss)
    return startup_prog, main_prog, py_reader, loss


class TestBase(unittest.TestCase):
77 78 79 80 81 82 83
    def run_main(
        self,
        use_legacy_py_reader,
        with_data_parallel,
        places,
        use_double_buffer,
    ):
S
sneaxiy 已提交
84 85
        scope = fluid.Scope()
        with fluid.scope_guard(scope):
S
sneaxiy 已提交
86
            startup_prog, main_prog, py_reader, loss = simple_fc_net(
87 88
                places, use_legacy_py_reader, use_double_buffer
            )
S
sneaxiy 已提交
89 90 91 92

            reader = paddle.batch(random_reader, batch_size=BATCH_SIZE)

            ps = places if use_double_buffer else fluid.cpu_places(len(places))
S
sneaxiy 已提交
93 94

            py_reader.decorate_sample_list_generator(
95 96
                reader, places=ps if py_reader.iterable else None
            )
S
sneaxiy 已提交
97

S
sneaxiy 已提交
98 99 100 101 102
            exe = fluid.Executor(place=places[0])
            exe.run(startup_prog)

            prog = fluid.CompiledProgram(main_prog)
            if with_data_parallel:
103 104 105
                prog = prog.with_data_parallel(
                    loss_name=loss.name, places=places
                )
S
sneaxiy 已提交
106 107

            step = 0
S
sneaxiy 已提交
108
            step_list = []
S
sneaxiy 已提交
109
            loss_list = []
S
sneaxiy 已提交
110
            start_t = time.time()
S
sneaxiy 已提交
111
            if not py_reader.iterable:
112
                for _ in range(EPOCH_NUM):
S
sneaxiy 已提交
113
                    step = 0
S
sneaxiy 已提交
114 115 116
                    py_reader.start()
                    while True:
                        try:
117 118 119 120 121
                            (L,) = exe.run(
                                program=prog,
                                fetch_list=[loss],
                                use_program_cache=True,
                            )
S
sneaxiy 已提交
122
                            loss_list.append(np.mean(L))
S
sneaxiy 已提交
123 124 125 126
                            step += 1
                        except fluid.core.EOFException:
                            py_reader.reset()
                            break
S
sneaxiy 已提交
127
                    step_list.append(step)
S
sneaxiy 已提交
128
            else:
129
                for _ in range(EPOCH_NUM):
S
sneaxiy 已提交
130
                    step = 0
S
sneaxiy 已提交
131 132 133 134 135 136 137
                    for d in py_reader():
                        assert len(d) == len(places)
                        for i, item in enumerate(d):
                            image = item['image']
                            label = item['label']
                            assert image.shape() == [BATCH_SIZE, 784]
                            assert label.shape() == [BATCH_SIZE, 1]
138 139
                            assert image._place()._equals(ps[i])
                            assert label._place()._equals(ps[i])
140 141 142 143 144 145
                        (L,) = exe.run(
                            program=prog,
                            feed=d,
                            fetch_list=[loss],
                            use_program_cache=True,
                        )
S
sneaxiy 已提交
146
                        loss_list.append(np.mean(L))
S
sneaxiy 已提交
147
                        step += 1
S
sneaxiy 已提交
148
                    step_list.append(step)
S
sneaxiy 已提交
149
            end_t = time.time()
S
sneaxiy 已提交
150 151 152
            ret = {
                "time": end_t - start_t,
                "step": step_list,
153
                "loss": np.array(loss_list),
S
sneaxiy 已提交
154
            }
S
sneaxiy 已提交
155 156
            return ret

S
sneaxiy 已提交
157
    def prepare_places(self, with_data_parallel, with_cpu=True, with_gpu=True):
S
sneaxiy 已提交
158 159 160 161 162
        places = []
        if with_cpu:
            places.append([fluid.CPUPlace()])
            if with_data_parallel:
                places.append([fluid.CPUPlace()] * 2)
S
sneaxiy 已提交
163

S
sneaxiy 已提交
164
        if with_gpu and fluid.core.is_compiled_with_cuda():
S
sneaxiy 已提交
165 166 167 168 169 170 171 172 173 174
            tmp = fluid.cuda_places()
            assert len(tmp) > 0, "no gpu detected"
            if with_data_parallel:
                places.append(tmp)
            places.append([tmp[0]])
        return places

    def test_main(self):
        for with_data_parallel in [True, False]:
            for p in self.prepare_places(with_data_parallel):
S
sneaxiy 已提交
175
                for use_double_buffer in [False, True]:
S
sneaxiy 已提交
176
                    results = []
S
sneaxiy 已提交
177 178 179 180 181
                    for use_legacy_py_reader in [False, True]:
                        ret = self.run_main(
                            use_legacy_py_reader=use_legacy_py_reader,
                            with_data_parallel=with_data_parallel,
                            places=p,
182 183
                            use_double_buffer=use_double_buffer,
                        )
S
sneaxiy 已提交
184 185 186
                        results.append(ret)
                    if not use_double_buffer:
                        diff = np.max(
187 188
                            np.abs(results[0]['loss'] - results[1]['loss'])
                        )
S
sneaxiy 已提交
189
                        self.assertLess(diff, 1e-3)
S
sneaxiy 已提交
190 191 192 193


if __name__ == '__main__':
    unittest.main()