diff --git a/configs/edvr_l_blur_w_tsa.yaml b/configs/edvr_l_blur_w_tsa.yaml new file mode 100644 index 0000000000000000000000000000000000000000..91ea165f6ad599c5833c97f94b7c37211515d07a --- /dev/null +++ b/configs/edvr_l_blur_w_tsa.yaml @@ -0,0 +1,100 @@ +total_iters: 600000 +output_dir: output_dir +checkpoints_dir: checkpoints +find_unused_parameters: True +# tensor range for function tensor2img +min_max: + (0., 1.) + +model: + name: EDVRModel + tsa_iter: 50000 + generator: + name: EDVRNet + in_nf: 3 + out_nf: 3 + scale_factor: 1 + nf: 128 + nframes: 5 + groups: 8 + front_RBs: 5 + back_RBs: 40 + center: 2 + predeblur: True #False + HR_in: True #False + w_TSA: True + pixel_criterion: + name: CharbonnierLoss + +dataset: + train: + name: REDSDataset + mode: train + gt_folder: data/REDS/train_sharp/X4 + lq_folder: data/REDS/train_blur/X4 + img_format: png + crop_size: 256 + interval_list: [1] + random_reverse: False + number_frames: 5 + use_flip: True + use_rot: True + buf_size: 1024 + scale: 1 + fix_random_seed: 10 + num_workers: 6 + batch_size: 8 + + + test: + name: REDSDataset + mode: test + gt_folder: data/REDS/REDS4_test_sharp/X4 + lq_folder: data/REDS/REDS4_test_blur/X4 + img_format: png + interval_list: [1] + random_reverse: False + number_frames: 5 + batch_size: 1 + use_flip: False + use_rot: False + buf_size: 1024 + scale: 1 + fix_random_seed: 10 + +lr_scheduler: + name: CosineAnnealingRestartLR + learning_rate: !!float 4e-4 + periods: [50000, 100000, 150000, 150000, 150000] + restart_weights: [1, 0.5, 0.5, 0.5, 0.5] + eta_min: !!float 1e-7 + +optimizer: + name: Adam + # add parameters of net_name to optim + # name should in self.nets + net_names: + - generator + beta1: 0.9 + beta2: 0.99 + +validate: + interval: 10000 + save_img: false + + metrics: + psnr: # metric name, can be arbitrary + name: PSNR + crop_border: 0 + test_y_channel: False + ssim: + name: SSIM + crop_border: 0 + test_y_channel: False + +log_config: + interval: 50 + visiual_interval: 5000 + +snapshot_config: + interval: 5000 diff --git a/configs/edvr_l_blur_wo_tsa.yaml b/configs/edvr_l_blur_wo_tsa.yaml new file mode 100644 index 0000000000000000000000000000000000000000..214f054e876ce4d0c056cb2424388648c27a217c --- /dev/null +++ b/configs/edvr_l_blur_wo_tsa.yaml @@ -0,0 +1,99 @@ +total_iters: 600000 +output_dir: output_dir +checkpoints_dir: checkpoints +# tensor range for function tensor2img +min_max: + (0., 1.) + +model: + name: EDVRModel + tsa_iter: False + generator: + name: EDVRNet + in_nf: 3 + out_nf: 3 + scale_factor: 1 + nf: 128 + nframes: 5 + groups: 8 + front_RBs: 5 + back_RBs: 40 + center: 2 + predeblur: True #False + HR_in: True #False + w_TSA: False + pixel_criterion: + name: CharbonnierLoss + +dataset: + train: + name: REDSDataset + mode: train + gt_folder: data/REDS/train_sharp/X4 + lq_folder: data/REDS/train_blur/X4 + img_format: png + crop_size: 256 + interval_list: [1] + random_reverse: False + number_frames: 5 + use_flip: True + use_rot: True + buf_size: 1024 + scale: 1 + fix_random_seed: 10 + num_workers: 6 + batch_size: 8 + + + test: + name: REDSDataset + mode: test + gt_folder: data/REDS/REDS4_test_sharp/X4 + lq_folder: data/REDS/REDS4_test_blur/X4 + img_format: png + interval_list: [1] + random_reverse: False + number_frames: 5 + batch_size: 1 + use_flip: False + use_rot: False + buf_size: 1024 + scale: 1 + fix_random_seed: 10 + +lr_scheduler: + name: CosineAnnealingRestartLR + learning_rate: !!float 2e-4 + periods: [150000, 150000, 150000, 150000] + restart_weights: [1, 0.5, 0.5, 0.5] + eta_min: !!float 1e-7 + +optimizer: + name: Adam + # add parameters of net_name to optim + # name should in self.nets + net_names: + - generator + beta1: 0.9 + beta2: 0.99 + +validate: + interval: 5000 + save_img: false + + metrics: + psnr: # metric name, can be arbitrary + name: PSNR + crop_border: 0 + test_y_channel: False + ssim: + name: SSIM + crop_border: 0 + test_y_channel: False + +log_config: + interval: 10 + visiual_interval: 5000 + +snapshot_config: + interval: 5000 diff --git a/configs/edvr_m_wo_tsa.yaml b/configs/edvr_m_wo_tsa.yaml index 2891fdd91a8b0e096f4ba7cce65e7c5002dcbe55..3817cf1390be47137119b22ef6af6b582f5c2456 100644 --- a/configs/edvr_m_wo_tsa.yaml +++ b/configs/edvr_m_wo_tsa.yaml @@ -92,8 +92,8 @@ validate: test_y_channel: False log_config: - interval: 10 - visiual_interval: 500 + interval: 100 + visiual_interval: 5000 snapshot_config: interval: 5000 diff --git a/docs/en_US/tutorials/video_super_resolution.md b/docs/en_US/tutorials/video_super_resolution.md index 5821cbb111ca598e66059ee97352826483f2e37a..e7650f6ae4644f04e19624615ef53b0f958d78ce 100644 --- a/docs/en_US/tutorials/video_super_resolution.md +++ b/docs/en_US/tutorials/video_super_resolution.md @@ -75,6 +75,8 @@ The metrics are PSNR / SSIM. | EDVR_M_w_tsa_SRx4 | 30.5169 / 0.8699 | | EDVR_L_wo_tsa_SRx4 | 30.8649 / 0.8761 | | EDVR_L_w_tsa_SRx4 | 30.9336 / 0.8773 | +| EDVR_L_wo_tsa_deblur | 34.9587 / 0.9509 | +| EDVR_L_w_tsa_deblur | 35.1473 / 0.9526 | ## 1.4 Model Download @@ -84,6 +86,8 @@ The metrics are PSNR / SSIM. | EDVR_M_w_tsa_SRx4 | REDS | [EDVR_M_w_tsa_SRx4](https://paddlegan.bj.bcebos.com/models/EDVR_M_w_tsa_SRx4.pdparams) | EDVR_L_wo_tsa_SRx4 | REDS | [EDVR_L_wo_tsa_SRx4](https://paddlegan.bj.bcebos.com/models/EDVR_L_wo_tsa_SRx4.pdparams) | EDVR_L_w_tsa_SRx4 | REDS | [EDVR_L_w_tsa_SRx4](https://paddlegan.bj.bcebos.com/models/EDVR_L_w_tsa_SRx4.pdparams) +| EDVR_L_wo_tsa_deblur | REDS | [EDVR_L_wo_tsa_deblur](https://paddlegan.bj.bcebos.com/models/EDVR_L_wo_tsa_deblur.pdparams) +| EDVR_L_w_tsa_deblur | REDS | [EDVR_L_w_tsa_deblur](https://paddlegan.bj.bcebos.com/models/EDVR_L_w_tsa_deblur.pdparams) diff --git a/docs/zh_CN/tutorials/video_super_resolution.md b/docs/zh_CN/tutorials/video_super_resolution.md index ab69bf647279abf738155d81643469a59bf7cd58..f3a04dba10e1a27cf89f3f83be94a39de6fce17e 100644 --- a/docs/zh_CN/tutorials/video_super_resolution.md +++ b/docs/zh_CN/tutorials/video_super_resolution.md @@ -71,6 +71,8 @@ | EDVR_M_w_tsa_SRx4 | 30.5169 / 0.8699 | | EDVR_L_wo_tsa_SRx4 | 30.8649 / 0.8761 | | EDVR_L_w_tsa_SRx4 | 30.9336 / 0.8773 | +| EDVR_L_wo_tsa_deblur | 34.9587 / 0.9509 | +| EDVR_L_w_tsa_deblur | 35.1473 / 0.9526 | ## 1.4 模型下载 @@ -80,7 +82,8 @@ | EDVR_M_w_tsa_SRx4 | REDS | [EDVR_M_w_tsa_SRx4](https://paddlegan.bj.bcebos.com/models/EDVR_M_w_tsa_SRx4.pdparams) | EDVR_L_wo_tsa_SRx4 | REDS | [EDVR_L_wo_tsa_SRx4](https://paddlegan.bj.bcebos.com/models/EDVR_L_wo_tsa_SRx4.pdparams) | EDVR_L_w_tsa_SRx4 | REDS | [EDVR_L_w_tsa_SRx4](https://paddlegan.bj.bcebos.com/models/EDVR_L_w_tsa_SRx4.pdparams) - +| EDVR_L_wo_tsa_deblur | REDS | [EDVR_L_wo_tsa_deblur](https://paddlegan.bj.bcebos.com/models/EDVR_L_wo_tsa_deblur.pdparams) +| EDVR_L_w_tsa_deblur | REDS | [EDVR_L_w_tsa_deblur](https://paddlegan.bj.bcebos.com/models/EDVR_L_w_tsa_deblur.pdparams) diff --git a/ppgan/models/edvr_model.py b/ppgan/models/edvr_model.py index baca93b36fc125c508ba10ccfe465df5cc411265..3fa270d52831ca8092cb96f79360d3e21657069b 100644 --- a/ppgan/models/edvr_model.py +++ b/ppgan/models/edvr_model.py @@ -17,7 +17,7 @@ import paddle.nn as nn from .builder import MODELS from .sr_model import BaseSRModel -from .generators.edvr import ResidualBlockNoBN +from .generators.edvr import ResidualBlockNoBN, DCNPack from ..modules.init import reset_parameters @@ -77,10 +77,10 @@ class EDVRModel(BaseSRModel): def init_edvr_weight(net): def reset_func(m): - if hasattr(m, - 'weight') and (not isinstance(m, - (nn.BatchNorm, nn.BatchNorm2D)) - ) and (not isinstance(m, ResidualBlockNoBN)): + if hasattr(m, 'weight') and (not isinstance( + m, (nn.BatchNorm, nn.BatchNorm2D))) and ( + not isinstance(m, ResidualBlockNoBN) and + (not isinstance(m, DCNPack))): reset_parameters(m) net.apply(reset_func) diff --git a/ppgan/models/generators/edvr.py b/ppgan/models/generators/edvr.py index edb23a6b3cef63a8a0f10f9fcf69796077894cb7..be3932a0f945f90f197ae301f0f4238352f882c7 100644 --- a/ppgan/models/generators/edvr.py +++ b/ppgan/models/generators/edvr.py @@ -17,7 +17,7 @@ import paddle import numpy as np import paddle.nn as nn -from ...modules.init import kaiming_normal_, constant_ +from ...modules.init import kaiming_normal_, constant_, constant_init from ...modules.dcn import DeformableConv_dygraph # from paddle.vision.ops import DeformConv2D #to be compiled @@ -382,6 +382,8 @@ class DCNPack(nn.Layer): deformable_groups=self.deformable_groups) # self.dcn = DeformConv2D(in_channels=self.num_filters,out_channels=self.num_filters,kernel_size=self.kernel_size,stride=stride,padding=padding,dilation=dilation,deformable_groups=self.deformable_groups,groups=1) # to be compiled self.sigmoid = nn.Sigmoid() + # init conv offset + constant_init(self.conv_offset_mask, 0., 0.) def forward(self, fea_and_offset): out = None @@ -686,6 +688,7 @@ class EDVRNet(nn.Layer): kernel_size=3, stride=1, padding=1) + self.pixel_shuffle = nn.PixelShuffle(2) self.upconv2 = nn.Conv2D(in_channels=self.nf, out_channels=4 * 64, @@ -702,10 +705,11 @@ class EDVRNet(nn.Layer): kernel_size=3, stride=1, padding=1) - self.upsample = nn.Upsample(scale_factor=self.scale_factor, - mode="bilinear", - align_corners=False, - align_mode=0) + if self.scale_factor == 4: + self.upsample = nn.Upsample(scale_factor=self.scale_factor, + mode="bilinear", + align_corners=False, + align_mode=0) def forward(self, x): """ @@ -722,7 +726,7 @@ class EDVRNet(nn.Layer): L1_fea = self.pre_deblur(L1_fea) L1_fea = self.cov_1(L1_fea) if self.HR_in: - H, W = H // self.scale_factor, W // self.scale_factor + H, W = H // 4, W // 4 else: L1_fea = self.conv_first(L1_fea) L1_fea = self.Leaky_relu(L1_fea) @@ -782,5 +786,6 @@ class EDVRNet(nn.Layer): base = x_center else: base = self.upsample(x_center) + out += base return out diff --git a/ppgan/models/generators/generater_lapstyle.py b/ppgan/models/generators/generater_lapstyle.py index 640e9c6f578c457d01754d77e4d04e4bc03c129a..20108d7cd06804602140e16782bb90af5489093e 100644 --- a/ppgan/models/generators/generater_lapstyle.py +++ b/ppgan/models/generators/generater_lapstyle.py @@ -233,7 +233,6 @@ class Encoder(nn.Layer): nn.Conv2D(512, 512, (3, 3)), nn.ReLU() # relu5-4 ) - weight_path = get_path_from_url( 'https://paddlegan.bj.bcebos.com/models/vgg_normalised.pdparams') vgg_net.set_dict(paddle.load(weight_path)) diff --git a/ppgan/modules/dcn.py b/ppgan/modules/dcn.py index f5d60e8523650af0353deba2eb852e432878d2a3..62c6493c654b3cd394494b942656466f9d3df48a 100644 --- a/ppgan/modules/dcn.py +++ b/ppgan/modules/dcn.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import math import numpy as np import paddle from paddle.fluid.layer_helper import LayerHelper @@ -22,10 +23,10 @@ from paddle.fluid.layers import nn, utils from paddle.nn import Layer from paddle.fluid.initializer import Normal from paddle.common_ops_import import * +from .init import uniform_, constant_ class DeformConv2D(Layer): - def __init__(self, in_channels, out_channels, @@ -69,8 +70,21 @@ class DeformConv2D(Layer): shape=filter_shape, attr=self._weight_attr, default_initializer=_get_default_param_initializer()) - self.bias = self.create_parameter( - attr=self._bias_attr, shape=[self._out_channels], is_bias=True) + + self.bias = self.create_parameter(attr=self._bias_attr, + shape=[self._out_channels], + is_bias=True) + + self.init_weight() + + def init_weight(self): + n = self._in_channels + for k in self._kernel_size: + n *= k + stdv = 1. / math.sqrt(n) + uniform_(self.weight, -stdv, stdv) + if hasattr(self, 'bias') and self.bias is not None: + constant_(self.bias, 0.) def forward(self, x, offset, mask): out = deform_conv2d( @@ -84,7 +98,7 @@ class DeformConv2D(Layer): dilation=self._dilation, deformable_groups=self._deformable_groups, groups=self._groups, - ) + ) return out @@ -99,7 +113,7 @@ def deform_conv2d(x, deformable_groups=1, groups=1, name=None): - + stride = utils.convert_to_list(stride, 2, 'stride') padding = utils.convert_to_list(padding, 2, 'padding') dilation = utils.convert_to_list(dilation, 2, 'dilation') @@ -107,8 +121,9 @@ def deform_conv2d(x, use_deform_conv2d_v1 = True if mask is None else False if in_dygraph_mode(): - attrs = ('strides', stride, 'paddings', padding, 'dilations', dilation, 'deformable_groups',deformable_groups, - 'groups', groups, 'im2col_step', 1) + attrs = ('strides', stride, 'paddings', padding, 'dilations', dilation, + 'deformable_groups', deformable_groups, 'groups', groups, + 'im2col_step', 1) if use_deform_conv2d_v1: op_type = 'deformable_conv_v1' pre_bias = getattr(core.ops, op_type)(x, offset, weight, *attrs) @@ -144,8 +159,14 @@ def deform_conv2d(x, class DeformableConv_dygraph(Layer): - def __init__(self,num_filters,filter_size,dilation, - stride,padding,deformable_groups=1,groups=1): + def __init__(self, + num_filters, + filter_size, + dilation, + stride, + padding, + deformable_groups=1, + groups=1): super(DeformableConv_dygraph, self).__init__() self.num_filters = num_filters self.filter_size = filter_size @@ -154,12 +175,18 @@ class DeformableConv_dygraph(Layer): self.padding = padding self.deformable_groups = deformable_groups self.groups = groups - self.defor_conv = DeformConv2D(in_channels=self.num_filters, out_channels=self.num_filters, - kernel_size=self.filter_size, stride=self.stride, padding=self.padding, - dilation=self.dilation, deformable_groups=self.deformable_groups, groups=self.groups, weight_attr=None, bias_attr=None) - - - def forward(self,*input): + self.defor_conv = DeformConv2D(in_channels=self.num_filters, + out_channels=self.num_filters, + kernel_size=self.filter_size, + stride=self.stride, + padding=self.padding, + dilation=self.dilation, + deformable_groups=self.deformable_groups, + groups=self.groups, + weight_attr=None, + bias_attr=None) + + def forward(self, *input): x = input[0] offset = input[1] mask = input[2]