add fuse_bn_add_act_ops args

PaddleCV/image_classification/build_model.py

@@ -41,7 +41,8 @@ def _basic_model(data, model, args, is_train):
         image_in.stop_gradient = image.stop_gradient
         net_out = model.net(input=image_in,
                             class_dim=args.class_dim,
-                            data_format=args.data_format)
+                            data_format=args.data_format,
+                            fuse_bn_add_act=args.fuse_bn_add_act_ops)
     else:
         net_out = model.net(input=image, class_dim=args.class_dim)
     softmax_out = fluid.layers.softmax(net_out, use_cudnn=False)

PaddleCV/image_classification/models/resnet.py

@@ -31,7 +31,7 @@ class ResNet():
     def __init__(self, layers=50):
         self.layers = layers
 
-    def net(self, input, class_dim=1000, data_format="NCHW"):
+    def net(self, input, class_dim=1000, data_format="NCHW", fuse_bn_add_act=False):
         layers = self.layers
         supported_layers = [18, 34, 50, 101, 152]
         assert layers in supported_layers, \
@@ -77,7 +77,8 @@ class ResNet():
                         num_filters=num_filters[block],
                         stride=2 if i == 0 and block != 0 else 1,
                         name=conv_name,
-                        data_format=data_format)
+                        data_format=data_format,
+                        fuse_bn_add_act=fuse_bn_add_act)
 
             pool = fluid.layers.pool2d(
                 input=conv, pool_type='avg', global_pooling=True, data_format=data_format)
@@ -97,7 +98,8 @@ class ResNet():
                         stride=2 if i == 0 and block != 0 else 1,
                         is_first=block == i == 0,
                         name=conv_name,
-                        data_format=data_format)
+                        data_format=data_format,
+                        fuse_bn_add_act=fuse_bn_add_act)
 
             pool = fluid.layers.pool2d(
                 input=conv, pool_type='avg', global_pooling=True, data_format=data_format)
@@ -155,7 +157,7 @@ class ResNet():
         else:
             return input
 
-    def bottleneck_block(self, input, num_filters, stride, name, data_format):
+    def bottleneck_block(self, input, num_filters, stride, name, data_format, fuse_bn_add_act):
         conv0 = self.conv_bn_layer(
             input=input,
             num_filters=num_filters,
@@ -171,26 +173,56 @@ class ResNet():
             act='relu',
             name=name + "_branch2b",
             data_format=data_format)
-        conv2 = self.conv_bn_layer(
-            input=conv1,
-            num_filters=num_filters * 4,
-            filter_size=1,
-            act=None,
-            name=name + "_branch2c",
-            data_format=data_format)
+        if not fuse_bn_add_act:
+            conv2 = self.conv_bn_layer(
+                input=conv1,
+                num_filters=num_filters * 4,
+                filter_size=1,
+                act=None,
+                name=name + "_branch2c",
+                data_format=data_format)
+            short = self.shortcut(
+                input,
+                num_filters * 4,
+                stride,
+                is_first=False,
+                name=name + "_branch1",
+                data_format=data_format)
 
-        short = self.shortcut(
-            input,
-            num_filters * 4,
-            stride,
-            is_first=False,
-            name=name + "_branch1",
-            data_format=data_format)
+            return fluid.layers.elementwise_add(
+                x=short, y=conv2, act='relu', name=name + ".add.output.5")
+        else:
+            name = name + "_branch2c"
+            conv2 = fluid.layers.conv2d(
+                input=conv1,
+                num_filters=num_filters * 4,
+                filter_size=1,
+                act=None,
+                param_attr=ParamAttr(name=name + "_weights"),
+                bias_attr=False,
+                name=name + '.conv2d.output.1',
+                data_format=data_format)
+            short = self.shortcut(
+                input,
+                num_filters * 4,
+                stride,
+                is_first=False,
+                name=name + "_branch1",
+                data_format=data_format)
+            bn_name = "bn" + name[3:]
+            short = fluid.contrib.layers.fused_bn_add_act(
+                conv2,
+                short,
+                param_attr=ParamAttr(name=bn_name + '_scale'),
+                bias_attr=ParamAttr(bn_name + '_offset'),
+                moving_mean_name=bn_name + '_mean',
+                moving_variance_name=bn_name + '_variance',
+                name=name + ".add.output.5")
 
-        return fluid.layers.elementwise_add(
-            x=short, y=conv2, act='relu', name=name + ".add.output.5")
+            return short
 
-    def basic_block(self, input, num_filters, stride, is_first, name, data_format):
+    def basic_block(self, input, num_filters, stride, is_first, name,
+                    data_format, fuse_bn_add_act):
         conv0 = self.conv_bn_layer(
             input=input,
             num_filters=num_filters,
@@ -199,16 +231,54 @@ class ResNet():
             stride=stride,
             name=name + "_branch2a",
             data_format=data_format)
-        conv1 = self.conv_bn_layer(
-            input=conv0,
-            num_filters=num_filters,
-            filter_size=3,
-            act=None,
-            name=name + "_branch2b",
-            data_format=data_format)
-        short = self.shortcut(
-            input, num_filters, stride, is_first, name=name + "_branch1", data_format=data_format)
-        return fluid.layers.elementwise_add(x=short, y=conv1, act='relu')
+        if not fuse_bn_add_act:
+            conv1 = self.conv_bn_layer(
+                input=conv0,
+                num_filters=num_filters,
+                filter_size=3,
+                act=None,
+                name=name + "_branch2b",
+                data_format=data_format)
+            short = self.shortcut(
+                input,
+                num_filters,
+                stride,
+                is_first,
+                name=name + "_branch1",
+                data_format=data_format)
+
+            return fluid.layers.elementwise_add(x=short, y=conv1, act='relu')
+        else:
+            name = name + "_branch2b"
+            conv1 = fluid.layers.conv2d(
+                input=conv0,
+                num_filters=num_filters,
+                filter_size=3,
+                stride=1,
+                padding=1,
+                groups=1,
+                act=None,
+                param_attr=ParamAttr(name=name + "_weights"),
+                bias_attr=False,
+                name=name + '.conv2d.output.1',
+                data_format=data_format)
+            short = self.shortcut(
+                input,
+                num_filters,
+                stride,
+                is_first,
+                name=name + "_branch1",
+                data_format=data_format)
+            bn_name = "bn" + name[3:]
+            short = fluid.contrib.layers.fused_bn_add_act(
+                conv1,
+                short,
+                param_attr=ParamAttr(name=bn_name + '_scale'),
+                bias_attr=ParamAttr(bn_name + '_offset'),
+                moving_mean_name=bn_name + '_mean',
+                moving_variance_name=bn_name + '_variance')
+                
+            return short
 
 
 def ResNet18():

PaddleCV/image_classification/scripts/train/ResNet50_fp16.sh

@@ -30,6 +30,7 @@ python train.py \
        --data_format=${DATA_FORMAT} \
        --fuse_elewise_add_act_ops=true \
        --fuse_bn_act_ops=true \
+       --fuse_bn_add_act_ops=false \
        --validate=true \
        --is_profiler=false \
        --profiler_path=profile/ \

PaddleCV/image_classification/utils/utility.py

@@ -145,6 +145,7 @@ def parse_args():
     add_arg('data_format',              str,    "NCHW",                 "Tensor data format when training.")
     add_arg('fuse_elewise_add_act_ops', bool,   False,                  "Whether to use elementwise_act fusion.")
     add_arg('fuse_bn_act_ops',          bool,   False,                  "Whether to use batch_norm and act fusion.")
+    add_arg('fuse_bn_add_act_ops',      bool,   False,                  "Whether to use batch_norm, elementwise_add and act fusion. This is only used for AMP training.")
 
     add_arg('use_label_smoothing',      bool,   False,                  "Whether to use label_smoothing")
     add_arg('label_smoothing_epsilon',  float,  0.1,                    "The value of label_smoothing_epsilon parameter")