# copyright (c) 2019 PaddlePaddle Authors. All Rights Reserve. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from __future__ import absolute_import from __future__ import division from __future__ import print_function import paddle from paddle import nn import paddle.nn.functional as F from paddle import ParamAttr import os import sys from ppocr.modeling.necks.intracl import IntraCLBlock __dir__ = os.path.dirname(os.path.abspath(__file__)) sys.path.append(__dir__) sys.path.insert(0, os.path.abspath(os.path.join(__dir__, '../../..'))) from ppocr.modeling.backbones.det_mobilenet_v3 import SEModule class DSConv(nn.Layer): def __init__(self, in_channels, out_channels, kernel_size, padding, stride=1, groups=None, if_act=True, act="relu", **kwargs): super(DSConv, self).__init__() if groups == None: groups = in_channels self.if_act = if_act self.act = act self.conv1 = nn.Conv2D( in_channels=in_channels, out_channels=in_channels, kernel_size=kernel_size, stride=stride, padding=padding, groups=groups, bias_attr=False) self.bn1 = nn.BatchNorm(num_channels=in_channels, act=None) self.conv2 = nn.Conv2D( in_channels=in_channels, out_channels=int(in_channels * 4), kernel_size=1, stride=1, bias_attr=False) self.bn2 = nn.BatchNorm(num_channels=int(in_channels * 4), act=None) self.conv3 = nn.Conv2D( in_channels=int(in_channels * 4), out_channels=out_channels, kernel_size=1, stride=1, bias_attr=False) self._c = [in_channels, out_channels] if in_channels != out_channels: self.conv_end = nn.Conv2D( in_channels=in_channels, out_channels=out_channels, kernel_size=1, stride=1, bias_attr=False) def forward(self, inputs): x = self.conv1(inputs) x = self.bn1(x) x = self.conv2(x) x = self.bn2(x) if self.if_act: if self.act == "relu": x = F.relu(x) elif self.act == "hardswish": x = F.hardswish(x) else: print("The activation function({}) is selected incorrectly.". format(self.act)) exit() x = self.conv3(x) if self._c[0] != self._c[1]: x = x + self.conv_end(inputs) return x class DBFPN(nn.Layer): def __init__(self, in_channels, out_channels, use_asf=False, **kwargs): super(DBFPN, self).__init__() self.out_channels = out_channels self.use_asf = use_asf weight_attr = paddle.nn.initializer.KaimingUniform() self.in2_conv = nn.Conv2D( in_channels=in_channels[0], out_channels=self.out_channels, kernel_size=1, weight_attr=ParamAttr(initializer=weight_attr), bias_attr=False) self.in3_conv = nn.Conv2D( in_channels=in_channels[1], out_channels=self.out_channels, kernel_size=1, weight_attr=ParamAttr(initializer=weight_attr), bias_attr=False) self.in4_conv = nn.Conv2D( in_channels=in_channels[2], out_channels=self.out_channels, kernel_size=1, weight_attr=ParamAttr(initializer=weight_attr), bias_attr=False) self.in5_conv = nn.Conv2D( in_channels=in_channels[3], out_channels=self.out_channels, kernel_size=1, weight_attr=ParamAttr(initializer=weight_attr), bias_attr=False) self.p5_conv = nn.Conv2D( in_channels=self.out_channels, out_channels=self.out_channels // 4, kernel_size=3, padding=1, weight_attr=ParamAttr(initializer=weight_attr), bias_attr=False) self.p4_conv = nn.Conv2D( in_channels=self.out_channels, out_channels=self.out_channels // 4, kernel_size=3, padding=1, weight_attr=ParamAttr(initializer=weight_attr), bias_attr=False) self.p3_conv = nn.Conv2D( in_channels=self.out_channels, out_channels=self.out_channels // 4, kernel_size=3, padding=1, weight_attr=ParamAttr(initializer=weight_attr), bias_attr=False) self.p2_conv = nn.Conv2D( in_channels=self.out_channels, out_channels=self.out_channels // 4, kernel_size=3, padding=1, weight_attr=ParamAttr(initializer=weight_attr), bias_attr=False) if self.use_asf is True: self.asf = ASFBlock(self.out_channels, self.out_channels // 4) def forward(self, x): c2, c3, c4, c5 = x in5 = self.in5_conv(c5) in4 = self.in4_conv(c4) in3 = self.in3_conv(c3) in2 = self.in2_conv(c2) out4 = in4 + F.upsample( in5, scale_factor=2, mode="nearest", align_mode=1) # 1/16 out3 = in3 + F.upsample( out4, scale_factor=2, mode="nearest", align_mode=1) # 1/8 out2 = in2 + F.upsample( out3, scale_factor=2, mode="nearest", align_mode=1) # 1/4 p5 = self.p5_conv(in5) p4 = self.p4_conv(out4) p3 = self.p3_conv(out3) p2 = self.p2_conv(out2) p5 = F.upsample(p5, scale_factor=8, mode="nearest", align_mode=1) p4 = F.upsample(p4, scale_factor=4, mode="nearest", align_mode=1) p3 = F.upsample(p3, scale_factor=2, mode="nearest", align_mode=1) fuse = paddle.concat([p5, p4, p3, p2], axis=1) if self.use_asf is True: fuse = self.asf(fuse, [p5, p4, p3, p2]) return fuse class RSELayer(nn.Layer): def __init__(self, in_channels, out_channels, kernel_size, shortcut=True): super(RSELayer, self).__init__() weight_attr = paddle.nn.initializer.KaimingUniform() self.out_channels = out_channels self.in_conv = nn.Conv2D( in_channels=in_channels, out_channels=self.out_channels, kernel_size=kernel_size, padding=int(kernel_size // 2), weight_attr=ParamAttr(initializer=weight_attr), bias_attr=False) self.se_block = SEModule(self.out_channels) self.shortcut = shortcut def forward(self, ins): x = self.in_conv(ins) if self.shortcut: out = x + self.se_block(x) else: out = self.se_block(x) return out class RSEFPN(nn.Layer): def __init__(self, in_channels, out_channels, shortcut=True, **kwargs): super(RSEFPN, self).__init__() self.out_channels = out_channels self.ins_conv = nn.LayerList() self.inp_conv = nn.LayerList() self.intracl = False if 'intracl' in kwargs.keys() and kwargs['intracl'] is True: self.intracl = kwargs['intracl'] self.incl1 = IntraCLBlock(self.out_channels // 4, reduce_factor=2) self.incl2 = IntraCLBlock(self.out_channels // 4, reduce_factor=2) self.incl3 = IntraCLBlock(self.out_channels // 4, reduce_factor=2) self.incl4 = IntraCLBlock(self.out_channels // 4, reduce_factor=2) for i in range(len(in_channels)): self.ins_conv.append( RSELayer( in_channels[i], out_channels, kernel_size=1, shortcut=shortcut)) self.inp_conv.append( RSELayer( out_channels, out_channels // 4, kernel_size=3, shortcut=shortcut)) def forward(self, x): c2, c3, c4, c5 = x in5 = self.ins_conv[3](c5) in4 = self.ins_conv[2](c4) in3 = self.ins_conv[1](c3) in2 = self.ins_conv[0](c2) out4 = in4 + F.upsample( in5, scale_factor=2, mode="nearest", align_mode=1) # 1/16 out3 = in3 + F.upsample( out4, scale_factor=2, mode="nearest", align_mode=1) # 1/8 out2 = in2 + F.upsample( out3, scale_factor=2, mode="nearest", align_mode=1) # 1/4 p5 = self.inp_conv[3](in5) p4 = self.inp_conv[2](out4) p3 = self.inp_conv[1](out3) p2 = self.inp_conv[0](out2) if self.intracl is True: p5 = self.incl4(p5) p4 = self.incl3(p4) p3 = self.incl2(p3) p2 = self.incl1(p2) p5 = F.upsample(p5, scale_factor=8, mode="nearest", align_mode=1) p4 = F.upsample(p4, scale_factor=4, mode="nearest", align_mode=1) p3 = F.upsample(p3, scale_factor=2, mode="nearest", align_mode=1) fuse = paddle.concat([p5, p4, p3, p2], axis=1) return fuse class LKPAN(nn.Layer): def __init__(self, in_channels, out_channels, mode='large', **kwargs): super(LKPAN, self).__init__() self.out_channels = out_channels weight_attr = paddle.nn.initializer.KaimingUniform() self.ins_conv = nn.LayerList() self.inp_conv = nn.LayerList() # pan head self.pan_head_conv = nn.LayerList() self.pan_lat_conv = nn.LayerList() if mode.lower() == 'lite': p_layer = DSConv elif mode.lower() == 'large': p_layer = nn.Conv2D else: raise ValueError( "mode can only be one of ['lite', 'large'], but received {}". format(mode)) for i in range(len(in_channels)): self.ins_conv.append( nn.Conv2D( in_channels=in_channels[i], out_channels=self.out_channels, kernel_size=1, weight_attr=ParamAttr(initializer=weight_attr), bias_attr=False)) self.inp_conv.append( p_layer( in_channels=self.out_channels, out_channels=self.out_channels // 4, kernel_size=9, padding=4, weight_attr=ParamAttr(initializer=weight_attr), bias_attr=False)) if i > 0: self.pan_head_conv.append( nn.Conv2D( in_channels=self.out_channels // 4, out_channels=self.out_channels // 4, kernel_size=3, padding=1, stride=2, weight_attr=ParamAttr(initializer=weight_attr), bias_attr=False)) self.pan_lat_conv.append( p_layer( in_channels=self.out_channels // 4, out_channels=self.out_channels // 4, kernel_size=9, padding=4, weight_attr=ParamAttr(initializer=weight_attr), bias_attr=False)) self.intracl = False if 'intracl' in kwargs.keys() and kwargs['intracl'] is True: self.intracl = kwargs['intracl'] self.incl1 = IntraCLBlock(self.out_channels // 4, reduce_factor=2) self.incl2 = IntraCLBlock(self.out_channels // 4, reduce_factor=2) self.incl3 = IntraCLBlock(self.out_channels // 4, reduce_factor=2) self.incl4 = IntraCLBlock(self.out_channels // 4, reduce_factor=2) def forward(self, x): c2, c3, c4, c5 = x in5 = self.ins_conv[3](c5) in4 = self.ins_conv[2](c4) in3 = self.ins_conv[1](c3) in2 = self.ins_conv[0](c2) out4 = in4 + F.upsample( in5, scale_factor=2, mode="nearest", align_mode=1) # 1/16 out3 = in3 + F.upsample( out4, scale_factor=2, mode="nearest", align_mode=1) # 1/8 out2 = in2 + F.upsample( out3, scale_factor=2, mode="nearest", align_mode=1) # 1/4 f5 = self.inp_conv[3](in5) f4 = self.inp_conv[2](out4) f3 = self.inp_conv[1](out3) f2 = self.inp_conv[0](out2) pan3 = f3 + self.pan_head_conv[0](f2) pan4 = f4 + self.pan_head_conv[1](pan3) pan5 = f5 + self.pan_head_conv[2](pan4) p2 = self.pan_lat_conv[0](f2) p3 = self.pan_lat_conv[1](pan3) p4 = self.pan_lat_conv[2](pan4) p5 = self.pan_lat_conv[3](pan5) if self.intracl is True: p5 = self.incl4(p5) p4 = self.incl3(p4) p3 = self.incl2(p3) p2 = self.incl1(p2) p5 = F.upsample(p5, scale_factor=8, mode="nearest", align_mode=1) p4 = F.upsample(p4, scale_factor=4, mode="nearest", align_mode=1) p3 = F.upsample(p3, scale_factor=2, mode="nearest", align_mode=1) fuse = paddle.concat([p5, p4, p3, p2], axis=1) return fuse class ASFBlock(nn.Layer): """ This code is refered from: https://github.com/MhLiao/DB/blob/master/decoders/feature_attention.py """ def __init__(self, in_channels, inter_channels, out_features_num=4): """ Adaptive Scale Fusion (ASF) block of DBNet++ Args: in_channels: the number of channels in the input data inter_channels: the number of middle channels out_features_num: the number of fused stages """ super(ASFBlock, self).__init__() weight_attr = paddle.nn.initializer.KaimingUniform() self.in_channels = in_channels self.inter_channels = inter_channels self.out_features_num = out_features_num self.conv = nn.Conv2D(in_channels, inter_channels, 3, padding=1) self.spatial_scale = nn.Sequential( #Nx1xHxW nn.Conv2D( in_channels=1, out_channels=1, kernel_size=3, bias_attr=False, padding=1, weight_attr=ParamAttr(initializer=weight_attr)), nn.ReLU(), nn.Conv2D( in_channels=1, out_channels=1, kernel_size=1, bias_attr=False, weight_attr=ParamAttr(initializer=weight_attr)), nn.Sigmoid()) self.channel_scale = nn.Sequential( nn.Conv2D( in_channels=inter_channels, out_channels=out_features_num, kernel_size=1, bias_attr=False, weight_attr=ParamAttr(initializer=weight_attr)), nn.Sigmoid()) def forward(self, fuse_features, features_list): fuse_features = self.conv(fuse_features) spatial_x = paddle.mean(fuse_features, axis=1, keepdim=True) attention_scores = self.spatial_scale(spatial_x) + fuse_features attention_scores = self.channel_scale(attention_scores) assert len(features_list) == self.out_features_num out_list = [] for i in range(self.out_features_num): out_list.append(attention_scores[:, i:i + 1] * features_list[i]) return paddle.concat(out_list, axis=1)