diff --git a/dygraph/configs/ssd/_base_/ssdlite_ghostnet_320.yml b/dygraph/configs/ssd/_base_/ssdlite_ghostnet_320.yml new file mode 100644 index 0000000000000000000000000000000000000000..6a9e13b5a1ca30a0dee10388a1931ffba0d412eb --- /dev/null +++ b/dygraph/configs/ssd/_base_/ssdlite_ghostnet_320.yml @@ -0,0 +1,42 @@ +architecture: SSD +pretrain_weights: https://paddledet.bj.bcebos.com/models/pretrained/GhostNet_x1_3_ssld_pretrained.pdparams + +SSD: + backbone: GhostNet + ssd_head: SSDHead + post_process: BBoxPostProcess + +GhostNet: + scale: 1.3 + conv_decay: 0.00004 + with_extra_blocks: true + extra_block_filters: [[256, 512], [128, 256], [128, 256], [64, 128]] + feature_maps: [13, 18, 19, 20, 21, 22] + lr_mult_list: [0.25, 0.25, 0.5, 0.5, 0.75] + +SSDHead: + use_sepconv: True + conv_decay: 0.00004 + anchor_generator: + steps: [16, 32, 64, 107, 160, 320] + aspect_ratios: [[2.], [2., 3.], [2., 3.], [2., 3.], [2., 3.], [2., 3.]] + min_ratio: 20 + max_ratio: 95 + base_size: 320 + min_sizes: [] + max_sizes: [] + offset: 0.5 + flip: true + clip: true + min_max_aspect_ratios_order: false + +BBoxPostProcess: + decode: + name: SSDBox + nms: + name: MultiClassNMS + keep_top_k: 200 + score_threshold: 0.01 + nms_threshold: 0.45 + nms_top_k: 400 + nms_eta: 1.0 diff --git a/dygraph/configs/ssd/ssdlite_ghostnet_320_coco.yml b/dygraph/configs/ssd/ssdlite_ghostnet_320_coco.yml new file mode 100644 index 0000000000000000000000000000000000000000..c6eb6c11725bb041f12a1234ebe931cb019bd18e --- /dev/null +++ b/dygraph/configs/ssd/ssdlite_ghostnet_320_coco.yml @@ -0,0 +1,27 @@ +_BASE_: [ + '../datasets/coco_detection.yml', + '../runtime.yml', + '_base_/optimizer_1700e.yml', + '_base_/ssdlite_ghostnet_320.yml', + '_base_/ssdlite320_reader.yml', +] +weights: output/ssdlite_ghostnet_320_coco/model_final + +epoch: 1700 + +LearningRate: + base_lr: 0.2 + schedulers: + - !CosineDecay + max_epochs: 1700 + - !LinearWarmup + start_factor: 0.33333 + steps: 2000 + +OptimizerBuilder: + optimizer: + momentum: 0.9 + type: Momentum + regularizer: + factor: 0.0005 + type: L2 diff --git a/dygraph/ppdet/modeling/backbones/__init__.py b/dygraph/ppdet/modeling/backbones/__init__.py index c46fed17e6e275d62cd4afe084d87ab5521999b0..d027c323cf565b90b84b06088e109f16e9a697a7 100644 --- a/dygraph/ppdet/modeling/backbones/__init__.py +++ b/dygraph/ppdet/modeling/backbones/__init__.py @@ -19,6 +19,7 @@ from . import mobilenet_v1 from . import mobilenet_v3 from . import hrnet from . import blazenet +from . import ghostnet from .vgg import * from .resnet import * @@ -27,3 +28,4 @@ from .mobilenet_v1 import * from .mobilenet_v3 import * from .hrnet import * from .blazenet import * +from .ghostnet import * diff --git a/dygraph/ppdet/modeling/backbones/ghostnet.py b/dygraph/ppdet/modeling/backbones/ghostnet.py new file mode 100644 index 0000000000000000000000000000000000000000..16fb78c0ef273b23bd6665eb29fe0e213baa47af --- /dev/null +++ b/dygraph/ppdet/modeling/backbones/ghostnet.py @@ -0,0 +1,476 @@ +# copyright (c) 2021 PaddlePaddle Authors. All Rights Reserve. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import math +import paddle +from paddle import ParamAttr +import paddle.nn as nn +import paddle.nn.functional as F +from paddle.nn import Conv2D, BatchNorm, AdaptiveAvgPool2D, Linear +from paddle.regularizer import L2Decay +from paddle.nn.initializer import Uniform, KaimingNormal +from ppdet.core.workspace import register, serializable +from numbers import Integral +from ..shape_spec import ShapeSpec +from .mobilenet_v3 import make_divisible, ConvBNLayer + +__all__ = ['GhostNet'] + + +class ExtraBlockDW(nn.Layer): + def __init__(self, + in_c, + ch_1, + ch_2, + stride, + lr_mult, + conv_decay=0., + norm_type='bn', + norm_decay=0., + freeze_norm=False, + name=None): + super(ExtraBlockDW, self).__init__() + self.pointwise_conv = ConvBNLayer( + in_c=in_c, + out_c=ch_1, + filter_size=1, + stride=1, + padding=0, + act='relu6', + lr_mult=lr_mult, + conv_decay=conv_decay, + norm_type=norm_type, + norm_decay=norm_decay, + freeze_norm=freeze_norm, + name=name + "_extra1") + self.depthwise_conv = ConvBNLayer( + in_c=ch_1, + out_c=ch_2, + filter_size=3, + stride=stride, + padding=1, # + num_groups=int(ch_1), + act='relu6', + lr_mult=lr_mult, + conv_decay=conv_decay, + norm_type=norm_type, + norm_decay=norm_decay, + freeze_norm=freeze_norm, + name=name + "_extra2_dw") + self.normal_conv = ConvBNLayer( + in_c=ch_2, + out_c=ch_2, + filter_size=1, + stride=1, + padding=0, + act='relu6', + lr_mult=lr_mult, + conv_decay=conv_decay, + norm_type=norm_type, + norm_decay=norm_decay, + freeze_norm=freeze_norm, + name=name + "_extra2_sep") + + def forward(self, inputs): + x = self.pointwise_conv(inputs) + x = self.depthwise_conv(x) + x = self.normal_conv(x) + return x + + +class SEBlock(nn.Layer): + def __init__(self, num_channels, lr_mult, reduction_ratio=4, name=None): + super(SEBlock, self).__init__() + self.pool2d_gap = AdaptiveAvgPool2D(1) + self._num_channels = num_channels + stdv = 1.0 / math.sqrt(num_channels * 1.0) + med_ch = num_channels // reduction_ratio + self.squeeze = Linear( + num_channels, + med_ch, + weight_attr=ParamAttr( + learning_rate=lr_mult, + initializer=Uniform(-stdv, stdv), + name=name + "_1_weights"), + bias_attr=ParamAttr( + learning_rate=lr_mult, name=name + "_1_offset")) + stdv = 1.0 / math.sqrt(med_ch * 1.0) + self.excitation = Linear( + med_ch, + num_channels, + weight_attr=ParamAttr( + learning_rate=lr_mult, + initializer=Uniform(-stdv, stdv), + name=name + "_2_weights"), + bias_attr=ParamAttr( + learning_rate=lr_mult, name=name + "_2_offset")) + + def forward(self, inputs): + pool = self.pool2d_gap(inputs) + pool = paddle.squeeze(pool, axis=[2, 3]) + squeeze = self.squeeze(pool) + squeeze = F.relu(squeeze) + excitation = self.excitation(squeeze) + excitation = paddle.clip(x=excitation, min=0, max=1) + excitation = paddle.unsqueeze(excitation, axis=[2, 3]) + out = paddle.multiply(inputs, excitation) + return out + + +class GhostModule(nn.Layer): + def __init__(self, + in_channels, + output_channels, + kernel_size=1, + ratio=2, + dw_size=3, + stride=1, + relu=True, + lr_mult=1., + conv_decay=0., + norm_type='bn', + norm_decay=0., + freeze_norm=False, + name=None): + super(GhostModule, self).__init__() + init_channels = int(math.ceil(output_channels / ratio)) + new_channels = int(init_channels * (ratio - 1)) + self.primary_conv = ConvBNLayer( + in_c=in_channels, + out_c=init_channels, + filter_size=kernel_size, + stride=stride, + padding=int((kernel_size - 1) // 2), + num_groups=1, + act="relu" if relu else None, + lr_mult=lr_mult, + conv_decay=conv_decay, + norm_type=norm_type, + norm_decay=norm_decay, + freeze_norm=freeze_norm, + name=name + "_primary_conv") + self.cheap_operation = ConvBNLayer( + in_c=init_channels, + out_c=new_channels, + filter_size=dw_size, + stride=1, + padding=int((dw_size - 1) // 2), + num_groups=init_channels, + act="relu" if relu else None, + lr_mult=lr_mult, + conv_decay=conv_decay, + norm_type=norm_type, + norm_decay=norm_decay, + freeze_norm=freeze_norm, + name=name + "_cheap_operation") + + def forward(self, inputs): + x = self.primary_conv(inputs) + y = self.cheap_operation(x) + out = paddle.concat([x, y], axis=1) + return out + + +class GhostBottleneck(nn.Layer): + def __init__(self, + in_channels, + hidden_dim, + output_channels, + kernel_size, + stride, + use_se, + lr_mult, + conv_decay=0., + norm_type='bn', + norm_decay=0., + freeze_norm=False, + return_list=False, + name=None): + super(GhostBottleneck, self).__init__() + self._stride = stride + self._use_se = use_se + self._num_channels = in_channels + self._output_channels = output_channels + self.return_list = return_list + + self.ghost_module_1 = GhostModule( + in_channels=in_channels, + output_channels=hidden_dim, + kernel_size=1, + stride=1, + relu=True, + lr_mult=lr_mult, + conv_decay=conv_decay, + norm_type=norm_type, + norm_decay=norm_decay, + freeze_norm=freeze_norm, + name=name + "_ghost_module_1") + if stride == 2: + self.depthwise_conv = ConvBNLayer( + in_c=hidden_dim, + out_c=hidden_dim, + filter_size=kernel_size, + stride=stride, + padding=int((kernel_size - 1) // 2), + num_groups=hidden_dim, + act=None, + lr_mult=lr_mult, + conv_decay=conv_decay, + norm_type=norm_type, + norm_decay=norm_decay, + freeze_norm=freeze_norm, + name=name + + "_depthwise_depthwise" # looks strange due to an old typo, will be fixed later. + ) + if use_se: + self.se_block = SEBlock(hidden_dim, lr_mult, name=name + "_se") + self.ghost_module_2 = GhostModule( + in_channels=hidden_dim, + output_channels=output_channels, + kernel_size=1, + relu=False, + lr_mult=lr_mult, + conv_decay=conv_decay, + norm_type=norm_type, + norm_decay=norm_decay, + freeze_norm=freeze_norm, + name=name + "_ghost_module_2") + if stride != 1 or in_channels != output_channels: + self.shortcut_depthwise = ConvBNLayer( + in_c=in_channels, + out_c=in_channels, + filter_size=kernel_size, + stride=stride, + padding=int((kernel_size - 1) // 2), + num_groups=in_channels, + act=None, + lr_mult=lr_mult, + conv_decay=conv_decay, + norm_type=norm_type, + norm_decay=norm_decay, + freeze_norm=freeze_norm, + name=name + + "_shortcut_depthwise_depthwise" # looks strange due to an old typo, will be fixed later. + ) + self.shortcut_conv = ConvBNLayer( + in_c=in_channels, + out_c=output_channels, + filter_size=1, + stride=1, + padding=0, + num_groups=1, + act=None, + lr_mult=lr_mult, + conv_decay=conv_decay, + norm_type=norm_type, + norm_decay=norm_decay, + freeze_norm=freeze_norm, + name=name + "_shortcut_conv") + + def forward(self, inputs): + y = self.ghost_module_1(inputs) + x = y + if self._stride == 2: + x = self.depthwise_conv(x) + if self._use_se: + x = self.se_block(x) + x = self.ghost_module_2(x) + + if self._stride == 1 and self._num_channels == self._output_channels: + shortcut = inputs + else: + shortcut = self.shortcut_depthwise(inputs) + shortcut = self.shortcut_conv(shortcut) + x = paddle.add(x=x, y=shortcut) + + if self.return_list: + return [y, x] + else: + return x + + +@register +@serializable +class GhostNet(nn.Layer): + __shared__ = ['norm_type'] + + def __init__( + self, + scale=1.3, + feature_maps=[6, 12, 15], + with_extra_blocks=False, + extra_block_filters=[[256, 512], [128, 256], [128, 256], [64, 128]], + lr_mult_list=[1.0, 1.0, 1.0, 1.0, 1.0], + conv_decay=0., + norm_type='bn', + norm_decay=0.0, + freeze_norm=False): + super(GhostNet, self).__init__() + if isinstance(feature_maps, Integral): + feature_maps = [feature_maps] + if norm_type == 'sync_bn' and freeze_norm: + raise ValueError( + "The norm_type should not be sync_bn when freeze_norm is True") + self.feature_maps = feature_maps + self.with_extra_blocks = with_extra_blocks + self.extra_block_filters = extra_block_filters + + inplanes = 16 + self.cfgs = [ + # k, t, c, SE, s + [3, 16, 16, 0, 1], + [3, 48, 24, 0, 2], + [3, 72, 24, 0, 1], + [5, 72, 40, 1, 2], + [5, 120, 40, 1, 1], + [3, 240, 80, 0, 2], + [3, 200, 80, 0, 1], + [3, 184, 80, 0, 1], + [3, 184, 80, 0, 1], + [3, 480, 112, 1, 1], + [3, 672, 112, 1, 1], + [5, 672, 160, 1, 2], # SSDLite output + [5, 960, 160, 0, 1], + [5, 960, 160, 1, 1], + [5, 960, 160, 0, 1], + [5, 960, 160, 1, 1] + ] + self.scale = scale + conv1_out_ch = int(make_divisible(inplanes * self.scale, 4)) + self.conv1 = ConvBNLayer( + in_c=3, + out_c=conv1_out_ch, + filter_size=3, + stride=2, + padding=1, + num_groups=1, + act="relu", + lr_mult=1., + conv_decay=conv_decay, + norm_type=norm_type, + norm_decay=norm_decay, + freeze_norm=freeze_norm, + name="conv1") + + # build inverted residual blocks + self._out_channels = [] + self.ghost_bottleneck_list = [] + idx = 0 + inplanes = conv1_out_ch + for k, exp_size, c, use_se, s in self.cfgs: + lr_idx = min(idx // 3, len(lr_mult_list) - 1) + lr_mult = lr_mult_list[lr_idx] + + # for SSD/SSDLite, first head input is after ResidualUnit expand_conv + return_list = self.with_extra_blocks and idx + 2 in self.feature_maps + + ghost_bottleneck = self.add_sublayer( + "_ghostbottleneck_" + str(idx), + sublayer=GhostBottleneck( + in_channels=inplanes, + hidden_dim=int(make_divisible(exp_size * self.scale, 4)), + output_channels=int(make_divisible(c * self.scale, 4)), + kernel_size=k, + stride=s, + use_se=use_se, + lr_mult=lr_mult, + conv_decay=conv_decay, + norm_type=norm_type, + norm_decay=norm_decay, + freeze_norm=freeze_norm, + return_list=return_list, + name="_ghostbottleneck_" + str(idx))) + self.ghost_bottleneck_list.append(ghost_bottleneck) + inplanes = int(make_divisible(c * self.scale, 4)) + idx += 1 + self._update_out_channels( + int(make_divisible(exp_size * self.scale, 4)) + if return_list else inplanes, idx + 1, feature_maps) + + if self.with_extra_blocks: + self.extra_block_list = [] + extra_out_c = int(make_divisible(self.scale * self.cfgs[-1][1], 4)) + lr_idx = min(idx // 3, len(lr_mult_list) - 1) + lr_mult = lr_mult_list[lr_idx] + + conv_extra = self.add_sublayer( + "conv" + str(idx + 2), + sublayer=ConvBNLayer( + in_c=inplanes, + out_c=extra_out_c, + filter_size=1, + stride=1, + padding=0, + num_groups=1, + act="relu6", + lr_mult=lr_mult, + conv_decay=conv_decay, + norm_type=norm_type, + norm_decay=norm_decay, + freeze_norm=freeze_norm, + name="conv" + str(idx + 2))) + self.extra_block_list.append(conv_extra) + idx += 1 + self._update_out_channels(extra_out_c, idx + 1, feature_maps) + + for j, block_filter in enumerate(self.extra_block_filters): + in_c = extra_out_c if j == 0 else self.extra_block_filters[j - + 1][1] + conv_extra = self.add_sublayer( + "conv" + str(idx + 2), + sublayer=ExtraBlockDW( + in_c, + block_filter[0], + block_filter[1], + stride=2, + lr_mult=lr_mult, + conv_decay=conv_decay, + norm_type=norm_type, + norm_decay=norm_decay, + freeze_norm=freeze_norm, + name='conv' + str(idx + 2))) + self.extra_block_list.append(conv_extra) + idx += 1 + self._update_out_channels(block_filter[1], idx + 1, + feature_maps) + + def _update_out_channels(self, channel, feature_idx, feature_maps): + if feature_idx in feature_maps: + self._out_channels.append(channel) + + def forward(self, inputs): + x = self.conv1(inputs['image']) + outs = [] + for idx, ghost_bottleneck in enumerate(self.ghost_bottleneck_list): + x = ghost_bottleneck(x) + if idx + 2 in self.feature_maps: + if isinstance(x, list): + outs.append(x[0]) + x = x[1] + else: + outs.append(x) + + if not self.with_extra_blocks: + return outs + + for i, block in enumerate(self.extra_block_list): + idx = i + len(self.ghost_bottleneck_list) + x = block(x) + if idx + 2 in self.feature_maps: + outs.append(x) + return outs + + @property + def out_shape(self): + return [ShapeSpec(channels=c) for c in self._out_channels]