From cb673c14e5963144b08e5b4bb427a981b7aa88ee Mon Sep 17 00:00:00 2001 From: zhangting2020 Date: Tue, 22 Sep 2020 09:05:18 +0000 Subject: [PATCH] add fuse_bn_add_act_ops args --- PaddleCV/image_classification/build_model.py | 3 +- .../image_classification/models/resnet.py | 132 ++++++++++++++---- .../scripts/train/ResNet50_fp16.sh | 1 + .../image_classification/utils/utility.py | 1 + 4 files changed, 105 insertions(+), 32 deletions(-) diff --git a/PaddleCV/image_classification/build_model.py b/PaddleCV/image_classification/build_model.py index 3f7a3a80..d71bea1f 100644 --- a/PaddleCV/image_classification/build_model.py +++ b/PaddleCV/image_classification/build_model.py @@ -41,7 +41,8 @@ def _basic_model(data, model, args, is_train): image_in.stop_gradient = image.stop_gradient net_out = model.net(input=image_in, class_dim=args.class_dim, - data_format=args.data_format) + data_format=args.data_format, + fuse_bn_add_act=args.fuse_bn_add_act_ops) else: net_out = model.net(input=image, class_dim=args.class_dim) softmax_out = fluid.layers.softmax(net_out, use_cudnn=False) diff --git a/PaddleCV/image_classification/models/resnet.py b/PaddleCV/image_classification/models/resnet.py index fcf45358..9940f9d7 100644 --- a/PaddleCV/image_classification/models/resnet.py +++ b/PaddleCV/image_classification/models/resnet.py @@ -31,7 +31,7 @@ class ResNet(): def __init__(self, layers=50): self.layers = layers - def net(self, input, class_dim=1000, data_format="NCHW"): + def net(self, input, class_dim=1000, data_format="NCHW", fuse_bn_add_act=False): layers = self.layers supported_layers = [18, 34, 50, 101, 152] assert layers in supported_layers, \ @@ -77,7 +77,8 @@ class ResNet(): num_filters=num_filters[block], stride=2 if i == 0 and block != 0 else 1, name=conv_name, - data_format=data_format) + data_format=data_format, + fuse_bn_add_act=fuse_bn_add_act) pool = fluid.layers.pool2d( input=conv, pool_type='avg', global_pooling=True, data_format=data_format) @@ -97,7 +98,8 @@ class ResNet(): stride=2 if i == 0 and block != 0 else 1, is_first=block == i == 0, name=conv_name, - data_format=data_format) + data_format=data_format, + fuse_bn_add_act=fuse_bn_add_act) pool = fluid.layers.pool2d( input=conv, pool_type='avg', global_pooling=True, data_format=data_format) @@ -155,7 +157,7 @@ class ResNet(): else: return input - def bottleneck_block(self, input, num_filters, stride, name, data_format): + def bottleneck_block(self, input, num_filters, stride, name, data_format, fuse_bn_add_act): conv0 = self.conv_bn_layer( input=input, num_filters=num_filters, @@ -171,26 +173,56 @@ class ResNet(): act='relu', name=name + "_branch2b", data_format=data_format) - conv2 = self.conv_bn_layer( - input=conv1, - num_filters=num_filters * 4, - filter_size=1, - act=None, - name=name + "_branch2c", - data_format=data_format) + if not fuse_bn_add_act: + conv2 = self.conv_bn_layer( + input=conv1, + num_filters=num_filters * 4, + filter_size=1, + act=None, + name=name + "_branch2c", + data_format=data_format) + short = self.shortcut( + input, + num_filters * 4, + stride, + is_first=False, + name=name + "_branch1", + data_format=data_format) - short = self.shortcut( - input, - num_filters * 4, - stride, - is_first=False, - name=name + "_branch1", - data_format=data_format) + return fluid.layers.elementwise_add( + x=short, y=conv2, act='relu', name=name + ".add.output.5") + else: + name = name + "_branch2c" + conv2 = fluid.layers.conv2d( + input=conv1, + num_filters=num_filters * 4, + filter_size=1, + act=None, + param_attr=ParamAttr(name=name + "_weights"), + bias_attr=False, + name=name + '.conv2d.output.1', + data_format=data_format) + short = self.shortcut( + input, + num_filters * 4, + stride, + is_first=False, + name=name + "_branch1", + data_format=data_format) + bn_name = "bn" + name[3:] + short = fluid.contrib.layers.fused_bn_add_act( + conv2, + short, + param_attr=ParamAttr(name=bn_name + '_scale'), + bias_attr=ParamAttr(bn_name + '_offset'), + moving_mean_name=bn_name + '_mean', + moving_variance_name=bn_name + '_variance', + name=name + ".add.output.5") - return fluid.layers.elementwise_add( - x=short, y=conv2, act='relu', name=name + ".add.output.5") + return short - def basic_block(self, input, num_filters, stride, is_first, name, data_format): + def basic_block(self, input, num_filters, stride, is_first, name, + data_format, fuse_bn_add_act): conv0 = self.conv_bn_layer( input=input, num_filters=num_filters, @@ -199,16 +231,54 @@ class ResNet(): stride=stride, name=name + "_branch2a", data_format=data_format) - conv1 = self.conv_bn_layer( - input=conv0, - num_filters=num_filters, - filter_size=3, - act=None, - name=name + "_branch2b", - data_format=data_format) - short = self.shortcut( - input, num_filters, stride, is_first, name=name + "_branch1", data_format=data_format) - return fluid.layers.elementwise_add(x=short, y=conv1, act='relu') + if not fuse_bn_add_act: + conv1 = self.conv_bn_layer( + input=conv0, + num_filters=num_filters, + filter_size=3, + act=None, + name=name + "_branch2b", + data_format=data_format) + short = self.shortcut( + input, + num_filters, + stride, + is_first, + name=name + "_branch1", + data_format=data_format) + + return fluid.layers.elementwise_add(x=short, y=conv1, act='relu') + else: + name = name + "_branch2b" + conv1 = fluid.layers.conv2d( + input=conv0, + num_filters=num_filters, + filter_size=3, + stride=1, + padding=1, + groups=1, + act=None, + param_attr=ParamAttr(name=name + "_weights"), + bias_attr=False, + name=name + '.conv2d.output.1', + data_format=data_format) + short = self.shortcut( + input, + num_filters, + stride, + is_first, + name=name + "_branch1", + data_format=data_format) + bn_name = "bn" + name[3:] + short = fluid.contrib.layers.fused_bn_add_act( + conv1, + short, + param_attr=ParamAttr(name=bn_name + '_scale'), + bias_attr=ParamAttr(bn_name + '_offset'), + moving_mean_name=bn_name + '_mean', + moving_variance_name=bn_name + '_variance') + + return short def ResNet18(): diff --git a/PaddleCV/image_classification/scripts/train/ResNet50_fp16.sh b/PaddleCV/image_classification/scripts/train/ResNet50_fp16.sh index 6ebd5c01..707db5a6 100755 --- a/PaddleCV/image_classification/scripts/train/ResNet50_fp16.sh +++ b/PaddleCV/image_classification/scripts/train/ResNet50_fp16.sh @@ -30,6 +30,7 @@ python train.py \ --data_format=${DATA_FORMAT} \ --fuse_elewise_add_act_ops=true \ --fuse_bn_act_ops=true \ + --fuse_bn_add_act_ops=false \ --validate=true \ --is_profiler=false \ --profiler_path=profile/ \ diff --git a/PaddleCV/image_classification/utils/utility.py b/PaddleCV/image_classification/utils/utility.py index 45c85102..2d951978 100644 --- a/PaddleCV/image_classification/utils/utility.py +++ b/PaddleCV/image_classification/utils/utility.py @@ -145,6 +145,7 @@ def parse_args(): add_arg('data_format', str, "NCHW", "Tensor data format when training.") add_arg('fuse_elewise_add_act_ops', bool, False, "Whether to use elementwise_act fusion.") add_arg('fuse_bn_act_ops', bool, False, "Whether to use batch_norm and act fusion.") + add_arg('fuse_bn_add_act_ops', bool, False, "Whether to use batch_norm, elementwise_add and act fusion. This is only used for AMP training.") add_arg('use_label_smoothing', bool, False, "Whether to use label_smoothing") add_arg('label_smoothing_epsilon', float, 0.1, "The value of label_smoothing_epsilon parameter") -- GitLab