vgg16_fluid.py 11.1 KB
Newer Older
1
#   Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
F
fengjiayi 已提交
2
#
T
typhoonzero 已提交
3 4 5
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
F
fengjiayi 已提交
6
#
T
typhoonzero 已提交
7
#     http://www.apache.org/licenses/LICENSE-2.0
F
fengjiayi 已提交
8
#
T
typhoonzero 已提交
9 10 11 12 13
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
X
Xin Pan 已提交
14
"""VGG16 benchmark in Fluid"""
T
typhoonzero 已提交
15 16 17 18 19 20
from __future__ import print_function

import sys
import time
import numpy as np
import paddle.v2 as paddle
21 22 23
import paddle.fluid as fluid
import paddle.fluid.core as core
import paddle.fluid.profiler as profiler
T
typhoonzero 已提交
24 25 26
import argparse
import functools
import os
27
from paddle.fluid import debuger
T
typhoonzero 已提交
28

T
typhoonzero 已提交
29

T
typhoonzero 已提交
30 31 32 33 34 35 36 37
def str2bool(v):
    if v.lower() in ('yes', 'true', 't', 'y', '1'):
        return True
    elif v.lower() in ('no', 'false', 'f', 'n', '0'):
        return False
    else:
        raise argparse.ArgumentTypeError('Boolean value expected.')

T
typhoonzero 已提交
38

T
typhoonzero 已提交
39 40
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument(
X
Xin Pan 已提交
41
    '--batch_size', type=int, default=16, help="Batch size for training.")
T
typhoonzero 已提交
42 43 44 45 46 47 48 49 50 51 52 53
parser.add_argument(
    '--learning_rate',
    type=float,
    default=1e-3,
    help="Learning rate for training.")
parser.add_argument('--num_passes', type=int, default=50, help="No. of passes.")
parser.add_argument(
    '--device',
    type=str,
    default='CPU',
    choices=['CPU', 'GPU'],
    help="The device type.")
T
typhoonzero 已提交
54
parser.add_argument('--device_id', type=int, default=0, help="The device id.")
T
typhoonzero 已提交
55 56 57 58 59 60 61 62 63
parser.add_argument(
    '--data_format',
    type=str,
    default='NCHW',
    choices=['NCHW', 'NHWC'],
    help='The data order, now only support NCHW.')
parser.add_argument(
    '--data_set',
    type=str,
X
Xin Pan 已提交
64
    default='flowers',
T
typhoonzero 已提交
65 66 67 68 69 70 71
    choices=['cifar10', 'flowers'],
    help='Optional dataset for benchmark.')
parser.add_argument(
    '--local',
    type=str2bool,
    default=True,
    help='Whether to run as local mode.')
G
gongweibao 已提交
72 73 74 75 76 77 78 79 80 81 82

parser.add_argument(
    "--ps_hosts",
    type=str,
    default="",
    help="Comma-separated list of hostname:port pairs")
parser.add_argument(
    "--trainer_hosts",
    type=str,
    default="",
    help="Comma-separated list of hostname:port pairs")
83 84
parser.add_argument(
    "--profile", action='store_true', help="If set, profile a few steps.")
G
gongweibao 已提交
85 86 87 88

# Flags for defining the tf.train.Server
parser.add_argument(
    "--task_index", type=int, default=0, help="Index of task within the job")
T
typhoonzero 已提交
89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111
args = parser.parse_args()


def vgg16_bn_drop(input):
    def conv_block(input, num_filter, groups, dropouts):
        return fluid.nets.img_conv_group(
            input=input,
            pool_size=2,
            pool_stride=2,
            conv_num_filter=[num_filter] * groups,
            conv_filter_size=3,
            conv_act='relu',
            conv_with_batchnorm=True,
            conv_batchnorm_drop_rate=dropouts,
            pool_type='max')

    conv1 = conv_block(input, 64, 2, [0.3, 0])
    conv2 = conv_block(conv1, 128, 2, [0.4, 0])
    conv3 = conv_block(conv2, 256, 3, [0.4, 0.4, 0])
    conv4 = conv_block(conv3, 512, 3, [0.4, 0.4, 0])
    conv5 = conv_block(conv4, 512, 3, [0.4, 0.4, 0])

    drop = fluid.layers.dropout(x=conv5, dropout_prob=0.5)
112
    fc1 = fluid.layers.fc(input=drop, size=4096, act=None)
T
typhoonzero 已提交
113 114
    bn = fluid.layers.batch_norm(input=fc1, act='relu')
    drop2 = fluid.layers.dropout(x=bn, dropout_prob=0.5)
115
    fc2 = fluid.layers.fc(input=drop2, size=4096, act=None)
T
typhoonzero 已提交
116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143
    return fc2


def main():
    if args.data_set == "cifar10":
        classdim = 10
        if args.data_format == 'NCHW':
            data_shape = [3, 32, 32]
        else:
            data_shape = [32, 32, 3]
    else:
        classdim = 102
        if args.data_format == 'NCHW':
            data_shape = [3, 224, 224]
        else:
            data_shape = [224, 224, 3]

    # Input data
    images = fluid.layers.data(name='pixel', shape=data_shape, dtype='float32')
    label = fluid.layers.data(name='label', shape=[1], dtype='int64')

    # Train program
    net = vgg16_bn_drop(images)
    predict = fluid.layers.fc(input=net, size=classdim, act='softmax')
    cost = fluid.layers.cross_entropy(input=predict, label=label)
    avg_cost = fluid.layers.mean(x=cost)

    # Evaluator
F
fengjiayi 已提交
144 145 146
    batch_size = fluid.layers.create_tensor(dtype='int64')
    batch_acc = fluid.layers.accuracy(
        input=predict, label=label, total=batch_size)
T
typhoonzero 已提交
147 148 149 150

    # inference program
    inference_program = fluid.default_main_program().clone()
    with fluid.program_guard(inference_program):
F
fengjiayi 已提交
151
        inference_program = fluid.io.get_inference_program(batch_acc)
T
typhoonzero 已提交
152 153 154 155 156 157

    # Optimization
    optimizer = fluid.optimizer.Adam(learning_rate=args.learning_rate)
    optimize_ops, params_grads = optimizer.minimize(avg_cost)

    # Initialize executor
T
typhoonzero 已提交
158 159
    place = core.CPUPlace() if args.device == 'CPU' else core.CUDAPlace(
        args.device_id)
T
typhoonzero 已提交
160 161 162 163
    exe = fluid.Executor(place)

    # test
    def test(exe):
F
fengjiayi 已提交
164
        test_pass_acc = fluid.average.WeightedAverage()
T
typhoonzero 已提交
165 166 167 168 169 170
        for batch_id, data in enumerate(test_reader()):
            img_data = np.array(map(lambda x: x[0].reshape(data_shape),
                                    data)).astype("float32")
            y_data = np.array(map(lambda x: x[1], data)).astype("int64")
            y_data = y_data.reshape([-1, 1])

F
fengjiayi 已提交
171 172 173 174 175
            outs = exe.run(inference_program,
                           feed={"pixel": img_data,
                                 "label": y_data},
                           fetch_list=[batch_acc, batch_size])
            test_pass_acc.add(value=np.array(outs[0]), weight=np.array(outs[1]))
T
typhoonzero 已提交
176

F
fengjiayi 已提交
177
        return test_pass_acc.eval()
T
typhoonzero 已提交
178 179 180

    def train_loop(exe, trainer_prog):
        iters = 0
T
typhoonzero 已提交
181
        ts = time.time()
F
fengjiayi 已提交
182
        train_pass_acc = fluid.average.WeightedAverage()
T
typhoonzero 已提交
183 184 185 186
        for pass_id in range(args.num_passes):
            # train
            start_time = time.time()
            num_samples = 0
F
fengjiayi 已提交
187
            train_pass_acc.reset()
188 189

            def run_step(batch_id, data):
190 191 192 193 194
                img_data = np.array(
                    map(lambda x: x[0].reshape(data_shape), data)).astype(
                        "float32")
                y_data = np.array(map(lambda x: x[1], data)).astype("int64")
                y_data = y_data.reshape([-1, 1])
T
typhoonzero 已提交
195

196 197 198 199 200
                loss, acc, b_size = exe.run(
                    trainer_prog,
                    feed={"pixel": img_data,
                          "label": y_data},
                    fetch_list=[avg_cost, batch_acc, batch_size])
201 202
                return loss, acc, b_size

X
Xin Pan 已提交
203 204 205
            if args.profile:
                with profiler.profiler('All', 'total',
                                       '/tmp/profile_vgg_%d' % args.task_index):
206
                    for batch_id, data in enumerate(train_reader()):
X
Xin Pan 已提交
207
                        if batch_id > 4: break
208 209
                        run_step(batch_id, data)

X
Xin Pan 已提交
210 211
            total_time = 0.0
            count = 0
212 213 214
            for batch_id, data in enumerate(train_reader()):
                ts = time.time()
                loss, acc, b_size = run_step(batch_id, data)
215 216 217
                iters += 1
                num_samples += len(data)
                train_pass_acc.add(value=acc, weight=b_size)
X
Xin Pan 已提交
218 219 220 221

                duration = time.time() - ts
                total_time += duration
                count += len(data)
222
                print(
223
                    "Pass = %d, Iters = %d, Loss = %f, Accuracy = %f, "
X
Xin Pan 已提交
224 225 226
                    "Speed = %.2f (%.2f) img/s" % (pass_id, iters, loss, acc,
                                                   len(data) / duration,
                                                   count / total_time)
227
                )  # The accuracy is the accumulation of batches, but not the current batch.
T
typhoonzero 已提交
228 229

            pass_elapsed = time.time() - start_time
F
fengjiayi 已提交
230
            pass_train_acc = train_pass_acc.eval()
T
typhoonzero 已提交
231
            pass_test_acc = test(exe)
X
Xin Pan 已提交
232 233 234 235
            print("Task:%d Pass = %d, Training performance = %f imgs/s, "
                  "Train accuracy = %f, Test accuracy = %f\n" %
                  (args.task_index, pass_id, num_samples / pass_elapsed,
                   pass_train_acc, pass_test_acc))
T
typhoonzero 已提交
236 237 238 239 240 241 242 243

    if args.local:
        # Parameter initialization
        exe.run(fluid.default_startup_program())

        # data reader
        train_reader = paddle.batch(
            paddle.reader.shuffle(
T
typhoonzero 已提交
244 245
                paddle.dataset.cifar.train10() if args.data_set == 'cifar10'
                else paddle.dataset.flowers.train(),
T
typhoonzero 已提交
246 247 248 249 250 251 252 253 254
                buf_size=5120),
            batch_size=args.batch_size)
        test_reader = paddle.batch(
            paddle.dataset.cifar.test10()
            if args.data_set == 'cifar10' else paddle.dataset.flowers.test(),
            batch_size=args.batch_size)
        train_loop(exe, fluid.default_main_program())
    else:
        trainers = int(os.getenv("TRAINERS"))  # total trainer count
T
typhoonzero 已提交
255
        print("trainers total: ", trainers)
G
gongweibao 已提交
256

T
typhoonzero 已提交
257 258 259
        training_role = os.getenv(
            "TRAINING_ROLE",
            "TRAINER")  # get the training role: trainer/pserver
G
gongweibao 已提交
260

T
typhoonzero 已提交
261 262
        t = fluid.DistributeTranspiler()
        t.transpile(
G
gongweibao 已提交
263 264
            trainer_id=args.task_index,
            pservers=args.ps_hosts,
T
typhoonzero 已提交
265
            trainers=trainers)
T
typhoonzero 已提交
266 267

        if training_role == "PSERVER":
G
gongweibao 已提交
268 269
            current_endpoint = os.getenv("POD_IP") + ":" + os.getenv(
                "PADDLE_INIT_PORT")
T
typhoonzero 已提交
270 271 272 273
            if not current_endpoint:
                print("need env SERVER_ENDPOINT")
                exit(1)
            pserver_prog = t.get_pserver_program(current_endpoint)
T
typhoonzero 已提交
274 275
            pserver_startup = t.get_startup_program(current_endpoint,
                                                    pserver_prog)
T
typhoonzero 已提交
276 277 278 279 280 281 282 283 284
            exe.run(pserver_startup)
            exe.run(pserver_prog)
        elif training_role == "TRAINER":
            # Parameter initialization
            exe.run(fluid.default_startup_program())

            # data reader
            train_reader = paddle.batch(
                paddle.reader.shuffle(
T
typhoonzero 已提交
285 286
                    paddle.dataset.cifar.train10() if args.data_set == 'cifar10'
                    else paddle.dataset.flowers.train(),
T
typhoonzero 已提交
287 288 289
                    buf_size=5120),
                batch_size=args.batch_size)
            test_reader = paddle.batch(
T
typhoonzero 已提交
290 291
                paddle.dataset.cifar.test10() if args.data_set == 'cifar10' else
                paddle.dataset.flowers.test(),
T
typhoonzero 已提交
292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312
                batch_size=args.batch_size)

            trainer_prog = t.get_trainer_program()
            feeder = fluid.DataFeeder(feed_list=[images, label], place=place)
            # TODO(typhoonzero): change trainer startup program to fetch parameters from pserver
            exe.run(fluid.default_startup_program())
            train_loop(exe, trainer_prog)
        else:
            print("environment var TRAINER_ROLE should be TRAINER os PSERVER")


def print_arguments():
    print('-----------  Configuration Arguments -----------')
    for arg, value in sorted(vars(args).iteritems()):
        print('%s: %s' % (arg, value))
    print('------------------------------------------------')


if __name__ == "__main__":
    print_arguments()
    main()