From 8904a6e252fb64f588495129a4b020096ce311cf Mon Sep 17 00:00:00 2001 From: danleifeng Date: Wed, 5 Feb 2020 08:29:59 +0000 Subject: [PATCH] add dataformat for fp16 --- plsc/entry.py | 10 +++++-- plsc/models/base_model.py | 5 ++-- plsc/models/resnet.py | 61 +++++++++++++++++++++++++-------------- 3 files changed, 50 insertions(+), 26 deletions(-) diff --git a/plsc/entry.py b/plsc/entry.py index 2667b0b..4061867 100644 --- a/plsc/entry.py +++ b/plsc/entry.py @@ -110,6 +110,7 @@ class Entry(object): self.use_fp16 = False self.fp16_user_dict = None + self.data_format = 'NCHW' self.val_targets = self.config.val_targets self.dataset_dir = self.config.dataset_dir @@ -170,6 +171,8 @@ class Entry(object): Whether to use mixed precision training. """ self.use_fp16 = use_fp16 + if(self.use_fp16): + self.data_format = 'NHWC' self.fp16_user_dict = dict() self.fp16_user_dict['init_loss_scaling'] = init_loss_scaling self.fp16_user_dict['incr_every_n_steps'] = incr_every_n_steps @@ -408,7 +411,8 @@ class Entry(object): param_attr=self.param_attr, bias_attr=self.bias_attr, margin=self.margin, - scale=self.scale) + scale=self.scale, + data_format=self.data_format) acc1 = None acc5 = None @@ -618,7 +622,8 @@ class Entry(object): emb = model.build_network(input=image, label=label, - is_train=False) + is_train=False, + data_format=self.data_format) gpu_id = int(os.getenv("FLAGS_selected_gpus", 0)) place = fluid.CUDAPlace(gpu_id) @@ -883,7 +888,6 @@ class Entry(object): strategy = DistributedStrategy() strategy.mode = "collective" strategy.collective_mode = "grad_allreduce" - emb, loss, acc1, acc5, optimizer = self.build_program( True, False, diff --git a/plsc/models/base_model.py b/plsc/models/base_model.py index b13a8ad..4a9349e 100644 --- a/plsc/models/base_model.py +++ b/plsc/models/base_model.py @@ -52,7 +52,8 @@ class BaseModel(object): bias_attr=None, loss_type="dist_softmax", margin=0.5, - scale=64.0): + scale=64.0, + data_format='NCHW'): """ Add the distributed fc layer for the custom model. @@ -76,7 +77,7 @@ class BaseModel(object): "Supported loss types: {}, but given: {}".format( supported_loss_types, loss_type) - emb = self.build_network(input, label, is_train) + emb = self.build_network(input, label, is_train, data_format=data_format) prob = None loss = None if loss_type == "softmax": diff --git a/plsc/models/resnet.py b/plsc/models/resnet.py index 30857f2..efdd8dd 100644 --- a/plsc/models/resnet.py +++ b/plsc/models/resnet.py @@ -28,7 +28,8 @@ class ResNet(BaseModel): def build_network(self, input, label, - is_train=True): + is_train=True, + data_format='NCHW'): layers = self.layers supported_layers = [50, 101, 152] assert layers in supported_layers, \ @@ -45,7 +46,7 @@ class ResNet(BaseModel): conv = self.conv_bn_layer( input=input, num_filters=64, filter_size=3, stride=1, - pad=1, act='prelu', is_train=is_train) + pad=1, act='prelu', is_train=is_train, data_format=data_format) for block in range(len(depth)): for i in range(depth[block]): @@ -53,12 +54,14 @@ class ResNet(BaseModel): input=conv, num_filters=num_filters[block], stride=2 if i == 0 else 1, - is_train=is_train) + is_train=is_train, + data_format=data_format) bn = fluid.layers.batch_norm(input=conv, act=None, epsilon=2e-05, - is_test=False if is_train else True) + is_test=False if is_train else True, + data_layout=data_format) drop = fluid.layers.dropout(x=bn, dropout_prob=0.4, dropout_implementation='upscale_in_train', @@ -74,7 +77,8 @@ class ResNet(BaseModel): emb = fluid.layers.batch_norm(input=fc, act=None, epsilon=2e-05, - is_test=False if is_train else True) + is_test=False if is_train else True, + data_layout=data_format) return emb def conv_bn_layer(self, @@ -85,7 +89,8 @@ class ResNet(BaseModel): pad=0, groups=1, is_train=True, - act=None): + act=None, + data_format='NCHW'): conv = fluid.layers.conv2d( input=input, num_filters=num_filters, @@ -96,13 +101,15 @@ class ResNet(BaseModel): param_attr=fluid.param_attr.ParamAttr( initializer=fluid.initializer.Xavier( uniform=False, fan_in=0.0)), - bias_attr=False) + bias_attr=False, + data_format=data_format) if act == 'prelu': bn = fluid.layers.batch_norm(input=conv, act=None, epsilon=2e-05, momentum=0.9, - is_test=False if is_train else True) + is_test=False if is_train else True, + data_layout=data_format) return fluid.layers.prelu( bn, mode="all", @@ -112,62 +119,74 @@ class ResNet(BaseModel): return fluid.layers.batch_norm(input=conv, act=act, epsilon=2e-05, - is_test=False if is_train else True) + is_test=False if is_train else True, + data_layout=data_format) - def shortcut(self, input, ch_out, stride, is_train): - ch_in = input.shape[1] + def shortcut(self, input, ch_out, stride, is_train, data_format): + if data_format == 'NCHW': + ch_in = input.shape[1] + else: + ch_in = input.shape[-1] if ch_in != ch_out or stride != 1: return self.conv_bn_layer(input, ch_out, 1, stride, - is_train=is_train) + is_train=is_train, + data_format=data_format) else: return input - def bottleneck_block(self, input, num_filters, stride, is_train): + def bottleneck_block(self, input, num_filters, stride, is_train, data_format): if self.layers < 101: bn1 = fluid.layers.batch_norm(input=input, act=None, epsilon=2e-05, - is_test=False if is_train else True) + is_test=False if is_train else True, + data_layout=data_format) conv1 = self.conv_bn_layer(input=bn1, num_filters=num_filters, filter_size=3, pad=1, act='prelu', - is_train=is_train) + is_train=is_train, + data_format=data_format) conv2 = self.conv_bn_layer(input=conv1, num_filters=num_filters, filter_size=3, stride=stride, pad=1, - is_train=is_train) + is_train=is_train, + data_format=data_format) else: bn0 = fluid.layers.batch_norm(input=input, act=None, epsilon=2e-05, - is_test=False if is_train else True) + is_test=False if is_train else True, + data_layout=data_format) conv0 = self.conv_bn_layer(input=bn0, num_filters=num_filters / 4, filter_size=1, pad=0, act='prelu', - is_train=is_train) + is_train=is_train, + data_format=data_format) conv1 = self.conv_bn_layer(input=conv0, num_filters=num_filters / 4, filter_size=3, pad=1, act='prelu', - is_train=is_train) + is_train=is_train, + data_format=data_format) conv2 = self.conv_bn_layer(input=conv1, num_filters=num_filters, filter_size=1, stride=stride, pad=0, - is_train=is_train) + is_train=is_train, + data_format=data_format) - short = self.shortcut(input, num_filters, stride, is_train=is_train) + short = self.shortcut(input, num_filters, stride, is_train=is_train, data_format=data_format) return fluid.layers.elementwise_add(x=short, y=conv2, act=None) -- GitLab