Inpainting lbam (#3714)--New model for picture inpainting

This is lbam model for picture inpainting

PaddleCV/Research/inpainting-lbam/ActivationFunction.py

+import numpy as np
+
+# asymmetric gaussian shaped activation function g_A 
+import paddle.fluid as fluid
+
+
+def GaussActivation(input, a, mu, sigma1, sigma2):
+    initializer = fluid.initializer.ConstantInitializer(value=a)
+    a = fluid.layers.create_parameter(
+        shape=[1], dtype='float32', default_initializer=initializer)
+    a = fluid.layers.clip(a, min=1.01, max=6.0)
+
+    initializer = fluid.initializer.ConstantInitializer(value=mu)
+    mu = fluid.layers.create_parameter(
+        shape=[1], dtype='float32', default_initializer=initializer)
+    mu = fluid.layers.clip(mu, min=0.1, max=3.0)
+
+    initializer = fluid.initializer.ConstantInitializer(value=sigma1)
+    sigma1 = fluid.layers.create_parameter(
+        shape=[1], dtype='float32', default_initializer=initializer)
+    sigma1 = fluid.layers.clip(sigma1, min=1.0, max=2.0)
+
+    initializer = fluid.initializer.ConstantInitializer(value=sigma2)
+    sigma2 = fluid.layers.create_parameter(
+        shape=[1], dtype='float32', default_initializer=initializer)
+    sigma2 = fluid.layers.clip(sigma2, min=1.0, max=2.0)
+
+    lowerThanMu = fluid.layers.less_than(input, mu)
+    largerThanMu = fluid.layers.logical_not(lowerThanMu)
+
+    diff_mu = (input - mu)
+    leftValuesActiv = fluid.layers.exp(-1 * fluid.layers.square(diff_mu) *
+                                       sigma1) * a
+    leftValuesActiv = leftValuesActiv * lowerThanMu
+
+    rightValueActiv = 1 + fluid.layers.exp(-1 * fluid.layers.square(diff_mu) *
+                                           sigma2) * (a - 1)
+    rightValueActiv = rightValueActiv * largerThanMu
+
+    output = leftValuesActiv + rightValueActiv
+
+    return output
+
+
+def MaskUpdate(input, alpha):
+    initializer = fluid.initializer.ConstantInitializer(value=alpha)
+    alpha_t = fluid.layers.create_parameter(
+        shape=[1], dtype='float32', default_initializer=initializer)
+
+    alpha_t = fluid.layers.clip(alpha_t, min=0.6, max=0.8)
+    out = fluid.layers.relu(input)
+    out = fluid.layers.elementwise_pow(out, alpha_t)
+    return out

PaddleCV/Research/inpainting-lbam/LBAMModel.py

+import paddle.fluid as fluid
+from forwardAttentionLayer import ForwardAttention
+from reverseAttentionLayer import ReverseAttention, ReverseMaskConv
+
+
+class LBAMModel():
+    def __init__(self, num_filters):
+        self.num_filters = num_filters
+
+    def net(self, inputImgs, masks):
+        ef1, mu1, skipConnect1, forwardMap1 = ForwardAttention(
+            inputImgs, masks, 64, bn=False)
+        ef2, mu2, skipConnect2, forwardMap2 = ForwardAttention(ef1, mu1, 128)
+        ef3, mu3, skipConnect3, forwardMap3 = ForwardAttention(ef2, mu2, 256)
+        ef4, mu4, skipConnect4, forwardMap4 = ForwardAttention(ef3, mu3, 512)
+
+        ef5, mu5, skipConnect5, forwardMap5 = ForwardAttention(ef4, mu4, 512)
+        ef6, mu6, skipConnect6, forwardMap6 = ForwardAttention(ef5, mu5, 512)
+        ef7, _, _, _ = ForwardAttention(ef6, mu6, 512)
+
+        reverseMap1, revMu1 = ReverseMaskConv(1 - masks, 64)
+        reverseMap2, revMu2 = ReverseMaskConv(revMu1, 128)
+        reverseMap3, revMu3 = ReverseMaskConv(revMu2, 256)
+        reverseMap4, revMu4 = ReverseMaskConv(revMu3, 512)
+        reverseMap5, revMu5 = ReverseMaskConv(revMu4, 512)
+        reverseMap6, _ = ReverseMaskConv(revMu5, 512)
+
+        concatMap6 = fluid.layers.concat([forwardMap6, reverseMap6], axis=1)
+        dcFeatures1 = ReverseAttention(skipConnect6, ef7, concatMap6, 512)
+
+        concatMap5 = fluid.layers.concat([forwardMap5, reverseMap5], axis=1)
+        dcFeatures2 = ReverseAttention(skipConnect5, dcFeatures1, concatMap5,
+                                       512)
+
+        concatMap4 = fluid.layers.concat([forwardMap4, reverseMap4], axis=1)
+        dcFeatures3 = ReverseAttention(skipConnect4, dcFeatures2, concatMap4,
+                                       512)
+
+        concatMap3 = fluid.layers.concat([forwardMap3, reverseMap3], axis=1)
+        dcFeatures4 = ReverseAttention(skipConnect3, dcFeatures3, concatMap3,
+                                       256)
+
+        concatMap2 = fluid.layers.concat([forwardMap2, reverseMap2], axis=1)
+        dcFeatures5 = ReverseAttention(skipConnect2, dcFeatures4, concatMap2,
+                                       128)
+
+        concatMap1 = fluid.layers.concat([forwardMap1, reverseMap1], axis=1)
+        dcFeatures6 = ReverseAttention(skipConnect1, dcFeatures5, concatMap1,
+                                       64)
+
+        dcFeatures7 = fluid.layers.conv2d_transpose(
+            input=dcFeatures6,
+            num_filters=self.num_filters,
+            filter_size=4,
+            stride=2,
+            padding=1,
+            act=None,
+            bias_attr=False)
+
+        output = fluid.layers.abs(fluid.layers.tanh(dcFeatures7))
+
+        return output

PaddleCV/Research/inpainting-lbam/README.md

+# Image Inpainting with Learnable Bidirectional Attention Maps.
+The PaddlePaddle implementation of Image Inpainting with Learnable Bidirectional Attention Maps in ICCV 2019, by Chaohao Xie, Shaohui Liu, Chao Li, Ming-Ming Cheng, Wangmeng Zuo, Xiao Liu, Shilei Wen, Errui Ding.\
+<https://arxiv.org/abs/1909.00968>
+
+## 1. Requirements.
+
+PaddlePaddle version == 1.6.\
+Python version == 3.6.\
+NCCL for multiple GPUs.
+
+## 2. Usage.
+
+Download the pretrained models by <https://pan.baidu.com/s/1Xpgj6pcBTYYYxsAlJrFXgg>, password: apfo.\
+Run the test script.
+```
+sh test.sh
+```
+
+```
+mkdir -p results/paris
+
+FLAGS_fraction_of_gpu_memory_to_use=0.1 \
+CUDA_VISIBLE_DEVICES=0 \
+FLAGS_eager_delete_tensor_gb=0.0 \
+FLAGS_fast_eager_deletion_mode=1 \
+python -u test.py \
+--pretrained_model 'pretrained_models/LBAM_ParisStreetView' \  # path to the pretrained model
+--imgfn 'imgs/paris/pic.png' \                                 # input picture.
+--maskfn 'imgs/paris/mask.png' \                               # mask.
+--resultfn 'results/paris'                                     # folder for the result.
+```
+Input picture:\
+![avatar](imgs/paris/pic.png)
+
+Input mask:\
+![avatar](imgs/paris/mask.png)
+
+Inpainting result:\
+![avatar](results/paris/pic.png)

PaddleCV/Research/inpainting-lbam/forwardAttentionLayer.py

+import paddle.fluid as fluid
+from ActivationFunction import GaussActivation
+from ActivationFunction import MaskUpdate
+
+# learnable forward attention conv layer
+
+
+def ForwardAttentionLayer(inputFeatures,
+                          inputMasks,
+                          num_filters,
+                          kSize,
+                          stride,
+                          padding,
+                          bias=False):
+    convFeatures = fluid.layers.conv2d(
+        input=inputFeatures,
+        num_filters=num_filters,
+        filter_size=kSize,
+        stride=stride,
+        padding=padding,
+        act=None,
+        bias_attr=bias)
+    maskFeatures = fluid.layers.conv2d(
+        input=inputMasks,
+        num_filters=num_filters,
+        filter_size=kSize,
+        stride=stride,
+        padding=padding,
+        act=None,
+        bias_attr=bias)
+
+    maskActiv = GaussActivation(maskFeatures, 1.1, 2.0, 1.0, 1.0)
+    convOut = convFeatures * maskActiv
+
+    maskUpdate = MaskUpdate(maskFeatures, 0.8)
+
+    return convOut, maskUpdate, convFeatures, maskActiv
+
+
+def ForwardAttention(inputFeatures,
+                     inputMasks,
+                     num_filters,
+                     bn=True,
+                     sample='down-4',
+                     activ='leaky',
+                     convBias=False):
+    if sample == 'down-4':
+        kSize = 4
+        stride = 2
+        padding = 1
+    elif sample == 'down-5':
+        kSize = 5
+        stride = 2
+        padding = 2
+
+    elif sample == 'down-7':
+        kSize = 7
+        stride = 2
+        padding = 3
+    elif sample == 'down-3':
+        kSize = 3
+        stride = 2
+        padding = 1
+    else:
+        kSize = 3
+        stride = 1
+        padding = 1
+    features, maskUpdated, convPreF, maskActiv = ForwardAttentionLayer(
+        inputFeatures,
+        inputMasks,
+        num_filters,
+        kSize,
+        stride,
+        padding,
+        bias=convBias)
+
+    if bn:
+        features = fluid.layers.batch_norm(input=features)
+
+    if activ == 'leaky':
+        features = fluid.layers.leaky_relu(features, alpha=0.2)
+
+    elif activ == 'relu':
+        features = fluid.layers.relu(features)
+
+    elif activ == 'sigmoid':
+        features = fluid.layers.sigmoid(features)
+
+    elif activ == 'tanh':
+        features = fluid.layers.tanh(features)
+
+    elif activ == 'prelu':
+        features = fluid.layers.prelu(features, 'all')
+
+    else:
+        pass
+
+    return features, maskUpdated, convPreF, maskActiv

PaddleCV/Research/inpainting-lbam/reverseAttentionLayer.py

+import paddle.fluid as fluid
+from ActivationFunction import GaussActivation
+from ActivationFunction import MaskUpdate
+
+
+def ReverseMaskConv(inputMasks,
+                    num_filters,
+                    kSize=4,
+                    stride=2,
+                    padding=1,
+                    convBias=False):
+    maskFeatures = fluid.layers.conv2d(
+        input=inputMasks,
+        num_filters=num_filters,
+        filter_size=kSize,
+        stride=stride,
+        padding=padding,
+        act=None,
+        bias_attr=convBias)
+
+    maskActiv = GaussActivation(maskFeatures, 1.1, 2.0, 1.0, 1.0)
+
+    maskUpdate = MaskUpdate(maskFeatures, 0.8)
+
+    return maskActiv, maskUpdate
+
+
+
+
+def ReverseAttention(ecFeaturesSkip, dcFeatures, maskFeaturesForAttention, num_filters, bn=True, activ='leaky', \
+        kSize=4, stride=2, padding=1, outPadding=0,convBias=False):
+
+    nextDcFeatures = fluid.layers.conv2d_transpose(
+        input=dcFeatures,
+        num_filters=num_filters,
+        filter_size=kSize,
+        stride=stride,
+        padding=padding,
+        act=None,
+        bias_attr=convBias)
+    concatFeatures = fluid.layers.concat(
+        [ecFeaturesSkip, nextDcFeatures], axis=1)
+    outputFeatures = concatFeatures * maskFeaturesForAttention
+
+    if bn:
+        outputFeatures = fluid.layers.batch_norm(input=outputFeatures)
+
+    if activ == 'leaky':
+        outputFeatures = fluid.layers.leaky_relu(outputFeatures, alpha=0.2)
+
+    elif activ == 'relu':
+        outputFeatures = fluid.layers.relu(outputFeatures)
+
+    elif activ == 'sigmoid':
+        outputFeatures = fluid.layers.sigmoid(outputFeatures)
+
+    elif activ == 'tanh':
+        outputFeatures = fluid.layers.tanh(outputFeatures)
+
+    elif activ == 'prelu':
+        outputFeatures = fluid.layers.prelu(outputFeatures, 'all')
+
+    else:
+        pass
+
+    return outputFeatures

PaddleCV/Research/inpainting-lbam/test.py

+import os
+import sys
+import paddle
+import paddle.fluid as fluid
+import cv2
+import numpy as np
+import glob
+from paddle.fluid.framework import Parameter
+
+from LBAMModel import LBAMModel
+
+import functools
+import argparse
+from utility import add_arguments, print_arguments
+parser = argparse.ArgumentParser(description=__doc__)
+add_arg = functools.partial(add_arguments, argparser=parser)
+# yapf: disable
+add_arg('imgfn',             str,    None,        "image file name.")
+add_arg('maskfn',            str,    None,        "mask file name.")
+add_arg('resultfn',           str,    None,         "result file name.")
+add_arg('pretrained_model',  str,    None,        "pretrained_model")
+
+def test():
+    args = parser.parse_args()
+    print_arguments(args)
+
+    pretrained_model = args.pretrained_model
+
+    place = fluid.CUDAPlace(0)
+    exe = fluid.Executor(place)
+    exe.run(fluid.default_startup_program())
+
+    [inference_program, feed_target_names, fetch_targets] = fluid.io.load_inference_model(dirname=pretrained_model, executor=exe, model_filename='model', params_filename='params')
+
+    imgfn  = args.imgfn
+    maskfn = args.maskfn
+    resultfn = args.resultfn
+    if not os.path.exists(args.resultfn):
+        os.makedirs(args.resultfn)
+
+    imglist  = sorted(glob.glob(imgfn))
+    masklist = sorted(glob.glob(maskfn))
+
+    for imgfn_,maskfn_ in (list(zip(imglist,masklist))):
+        print(imgfn_)
+        print(maskfn_)
+        print('')
+
+        img = cv2.imread(imgfn_)
+        mask = cv2.imread(maskfn_)
+
+        img  = img.transpose(2, 0, 1)[::-1]
+        img  = img.astype(np.float32)/255.0
+        mask = mask.transpose(2, 0, 1)
+        mask = mask.astype(np.float32)/255.0
+
+        threshhold = 0.5
+        mask = (mask >= threshhold).astype(np.float32)
+
+        # CHW RGB
+        mask = 1 - mask
+        img = img * mask
+
+        img0 = img
+        img  = np.concatenate((img, mask[0:1]), axis=0)
+
+        result = exe.run(inference_program,feed={feed_target_names[0]: img[np.newaxis,:], feed_target_names[1]: mask[np.newaxis,:]}, fetch_list=fetch_targets)
+
+        outimg = result[0][0]
+        outimg = outimg * (1-mask) +  img0 * mask
+
+        # BGR HWC
+        outimg = outimg[::-1].transpose(1, 2, 0)*255.0
+
+
+        outfn = os.path.join(args.resultfn, os.path.basename(imgfn_))
+        cv2.imwrite(outfn,outimg)
+
+
+if __name__ == '__main__':
+    test()

PaddleCV/Research/inpainting-lbam/test.sh

+mkdir -p results/paris
+
+FLAGS_fraction_of_gpu_memory_to_use=0.1 \
+CUDA_VISIBLE_DEVICES=0 \
+FLAGS_eager_delete_tensor_gb=0.0 \
+FLAGS_fast_eager_deletion_mode=1 \
+python -u test.py \
+--pretrained_model 'pretrained_models/LBAM_ParisStreetView' \  # path to the pretrained model
+--imgfn 'imgs/paris/pic.png' \                                 # input picture.
+--maskfn 'imgs/paris/mask.png' \                               # mask.
+--resultfn 'results/paris'                                     # folder for the result.
\ No newline at end of file

PaddleCV/Research/inpainting-lbam/utility.py

+"""Contains common utility functions."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+import distutils.util
+import numpy as np
+import six
+from paddle.fluid import core
+
+
+def print_arguments(args):
+    """Print argparse's arguments.
+
+    Usage:
+
+    .. code-block:: python
+
+        parser = argparse.ArgumentParser()
+        parser.add_argument("name", default="Jonh", type=str, help="User name.")
+        args = parser.parse_args()
+        print_arguments(args)
+
+    :param args: Input argparse.Namespace for printing.
+    :type args: argparse.Namespace
+    """
+    print("-----------  Configuration Arguments -----------")
+    for arg, value in sorted(six.iteritems(vars(args))):
+        print("%s: %s" % (arg, value))
+    print("------------------------------------------------")
+
+
+def add_arguments(argname, type, default, help, argparser, **kwargs):
+    """Add argparse's argument.
+
+    Usage:
+
+    .. code-block:: python
+
+        parser = argparse.ArgumentParser()
+        add_argument("name", str, "Jonh", "User name.", parser)
+        args = parser.parse_args()
+    """
+    type = distutils.util.strtobool if type == bool else type
+    argparser.add_argument(
+        "--" + argname,
+        default=default,
+        type=type,
+        help=help + ' Default: %(default)s.',
+        **kwargs)