update tensorflow module

x2paddle/__init__.py

@@ -3,15 +3,3 @@ __version__ = "0.8.5"
 from .core.program import PaddleGraph
 
 program = PaddleGraph()
-
-name_counter = dict()
-
-
-def gen_name(op_name, var_name):
-    name = "{}.{}".format(op_name, var_name)
-    if name not in name_counter:
-        name_counter[name] = 0
-    else:
-        name_counter[name] += 1
-    name = name + "." + str(name_counter[name])
-    return name

x2paddle/convert.py

@@ -98,7 +98,7 @@ def arg_parser():
 
 def tf2paddle(model_path,
               save_dir,
-              without_data_format_optimization,
+              without_data_format_optimization=False,
               define_input_shape=False,
               params_merge=False):
     # check tensorflow installation and version
@@ -117,37 +117,24 @@ def tf2paddle(model_path,
             "[ERROR] Tensorflow is not installed, use \"pip install tensorflow\"."
         )
         return
-
+    from x2paddle import program
     from x2paddle.decoder.tf_decoder import TFDecoder
     from x2paddle.op_mapper.tf_op_mapper import TFOpMapper
-    from x2paddle.op_mapper.tf_op_mapper_nhwc import TFOpMapperNHWC
-    from x2paddle.optimizer.tf_optimizer import TFOptimizer
+    from x2paddle.optimizer.tensorflow.bias import BiasOpt
+    from x2paddle.optimizer.tensorflow.transpose import TransposeOpt
+    from x2paddle.optimizer.tensorflow.batch_norm import BatchNormOpt
 
     print("Now translating model from tensorflow to paddle.")
     model = TFDecoder(model_path, define_input_shape=define_input_shape)
-    if not without_data_format_optimization:
     mapper = TFOpMapper(model)
-        optimizer = TFOptimizer(mapper)
-        # neccesary optimization
-        optimizer.delete_redundance_code()
-        # optimizer below is experimental
-        optimizer.optimize_elementwise_op()
-        optimizer.merge_activation()
-        optimizer.merge_bias()
-        optimizer.optimize_sub_graph()
-
-#        optimizer.merge_batch_norm()
-#        optimizer.merge_prelu()
-    else:
-        mapper = TFOpMapperNHWC(model)
-        optimizer = TFOptimizer(mapper)
-        optimizer.delete_redundance_code()
-        optimizer.strip_graph()
-        optimizer.merge_activation()
-        optimizer.merge_bias()
-        optimizer.make_nchw_input_output()
-        optimizer.remove_transpose()
-    mapper.save_inference_model(save_dir, params_merge)
+    program.build()
+    bias_opt = BiasOpt()
+    transpose_opt = TransposeOpt()
+    batch_norm_opt = BatchNormOpt()
+    bias_opt.run(program)
+    batch_norm_opt.run(program)
+    transpose_opt.run(program)
+    program.gen_model(save_dir)
 
 
 def caffe2paddle(proto, weight, save_dir, caffe_proto, params_merge=False):

x2paddle/core/program.py

@@ -99,6 +99,53 @@ class PaddleGraph(object):
         self.layers[layer_id] = layer
         return layer_id
 
+    def del_layer(self, layer_id):
+        layer = self.layers[layer_id]
+        outputs = self.edges_out.get(layer_id, [])
+        inputs = self.edges_in.get(layer_id, [])
+
+        assert len(
+            inputs) <= 1, "There should be 0 or 1 input for deleted layer."
+
+        if len(inputs) == 0:
+            for out in outputs:
+                while layer_id in self.edges_in[out]:
+                    index = self.edges_in[out].index(layer_id)
+                    del self.edges_in[out][index]
+
+                input_keys = list(self.layers[out].inputs.keys())
+                for k in input_keys:
+                    if self.layers[out].inputs[k] == layer.outputs[0]:
+                        del self.layers[out].inputs[k]
+
+            del self.layers[layer_id]
+            if layer_id in self.edges_in:
+                del self.edges_in[layer_id]
+            if layer_id in self.edges_out:
+                del self.edges_out[layer_id]
+            return
+
+        # 将所有输出layer的输入layer进行替换
+        for out in outputs:
+            for i in range(len(self.edges_in[out])):
+                if self.edges_in[out][i] == layer_id:
+                    self.edges_in[out][i] = inputs[0]
+
+        # 将输出layer赋给输入layer的输出
+        replace_index = self.edges_out[inputs[0]].index(layer_id)
+        del self.edges_out[inputs[0]][replace_index]
+        for i, out in enumerate(outputs):
+            self.edges_out[inputs[0]].insert(replace_index + i, out)
+            for k, v in self.layers[out].inputs.items():
+                if v == layer.outputs[0]:
+                    self.layers[out].inputs[k] = list(layer.inputs.values())[0]
+
+        del self.layers[layer_id]
+        if layer_id in self.edges_out:
+            del self.edges_out[layer_id]
+        if layer_id in self.edges_in:
+            del self.edges_in[layer_id]
+
     def build(self, inputs=None, outputs=None):
         self.clear_edges()
         outputs_from_nodes = dict()

x2paddle/decoder/tf_decoder.py

@@ -89,6 +89,12 @@ class TFGraphNode(GraphNode):
         field = getattr(attr, attr.WhichOneof('value'))
         return tensor_util.MakeNdarray(field)
 
+    @property
+    def name(self):
+        if hasattr(self, 'index'):
+            return self.layer_name + "_p{}".format(self.index)
+        return self.layer_name
+
     def get_attr(self, name):
         if name not in self.layer.attr:
             return None

x2paddle/op_mapper/tf_op_mapper.py

@@ -15,10 +15,25 @@
 from x2paddle.decoder.tf_decoder import TFGraph
 from x2paddle.core.op_mapper import OpMapper
 from x2paddle.core.util import *
+from x2paddle import program
+import traceback
+import math
 import inspect
 import numpy
 import sys
 
+name_counter = dict()
+
+
+def gen_name(op_name, var_name):
+    name = "{}_{}".format(op_name, var_name)
+    if name not in name_counter:
+        name_counter[name] = 0
+    else:
+        name_counter[name] += 1
+    name = name + '_' + str(name_counter[name])
+    return name
+
 
 # compute padding size for SAME mode
 def get_same_padding(in_size, kernel_size, stride):
@@ -31,45 +46,34 @@ def get_same_padding(in_size, kernel_size, stride):
     return [pad0, pad1]
 
 
-def nhwc_dim_to_nchw(node, dim):
-    tf_data_format = list(node.tf_data_format)
-    pd_data_format = list(node.pd_data_format)
-    if isinstance(dim, list):
-        for i in range(len(dim)):
-            char = tf_data_format[dim[i]]
-            dim[i] = pd_data_format.index(char)
-    else:
-        char = tf_data_format[dim]
-        dim = pd_data_format.index(char)
-    return dim
-
-    if dim < 0:
-        dim += 4
-    if dim > 0:
-        dim = (dim + 1) % 4 + int((dim + 1) / 4)
-    return dim
-
-
 class TFOpMapper(OpMapper):
     directly_map_ops = {
         'Relu': ['relu'],
         'Relu6': ['relu6'],
-        'Shape': ['shape'],
         'Abs': ['abs'],
         'Sigmoid': ['sigmoid'],
         'Exp': ['exp'],
         'Rsqrt': ['rsqrt'],
+        'Sqrt': ['sqrt'],
         'swish_f32': ['swish'],
         'Tanh': ['tanh'],
+        'Softplus': ['softplus'],
         'LeakyRelu': ['leaky_relu', {
             'alpha': 'alpha'
-        }]
+        }],
+        'Floor': ['floor'],
+        'Erf': ['erf'],
+        'Square': ['square']
     }
     elementwise_ops = {
         'Add': 'elementwise_add',
+        'AddV2': 'elementwise_add',
         'RealDiv': 'elementwise_div',
         'Sub': 'elementwise_sub',
         'Maximum': 'elementwise_max',
+        'Minimum': 'elementwise_min',
+        'LessEqual': 'less_equal',
+        'GreaterEqual': 'greater_equal',
         'Mul': 'elementwise_mul',
         'FloorDiv': 'elementwise_floordiv'
     }
@@ -78,22 +82,28 @@ class TFOpMapper(OpMapper):
         super(TFOpMapper, self).__init__()
         self.decoder = decoder
         self.graph = decoder.tf_graph
-        self.batch_node = None
         self.weights = dict()
         self.omit_nodes = list()
         self.used_custom_layers = dict()
+        program.clear()
 
         not_placeholder = list()
         for name in self.graph.input_nodes:
-            if self.graph.get_node(name).layer_type != "Placeholder" \
-               and self.graph.get_node(name).layer_type != "OneShotIterator":
+            if self.graph.get_node(
+                    name).layer_type != "Placeholder" and self.graph.get_node(
+                        name
+                    ).layer_type != "OneShotIterator" and self.graph.get_node(
+                        name).layer_type != "IteratorV2":
                 not_placeholder.append(name)
         for name in not_placeholder:
             idx = self.graph.input_nodes.index(name)
             del self.graph.input_nodes[idx]
 
-        sys.stderr.write("Total nodes: {}\n".format(len(self.graph.topo_sort)))
+        program.inputs = self.graph.input_nodes
+        program.outputs = self.graph.output_nodes
+
         unsupported_ops = set()
+        sys.stderr.write("Total nodes: {}\n".format(len(self.graph.topo_sort)))
         for i, node_name in enumerate(self.graph.topo_sort):
             sys.stderr.write("\rConverting node {} ...     ".format(i + 1))
             node = self.graph.get_node(node_name)
@@ -110,173 +120,74 @@ class TFOpMapper(OpMapper):
                 if len(unsupported_ops) > 0:
                     continue
                 func = getattr(self, op)
+                try:
                     func(node)
+                except Exception as e:
+                    unsupported_ops.add(op)
+                    print("\n{}\n".format(traceback.format_exc()))
             else:
                 unsupported_ops.add(op)
         if len(unsupported_ops) > 0:
-            sys.stderr.write("=========={} Ops are not supported yet======\n".
-                             format(len(unsupported_ops)))
+            print("\n========= {} OPs are not supported yet ===========".format(
+                len(unsupported_ops)))
             for op in unsupported_ops:
-                sys.stderr.write("========== {} ==========\n".format(op))
+                print("========== {} ============".format(op))
             sys.exit(-1)
-        sys.stderr.write('\nDone!\n')
-
-    def add_omit_nodes(self, in_node_name, out_node_name):
-        in_node = self.graph.get_node(in_node_name)
-        out_node = self.graph.get_node(out_node_name)
-        index = in_node.outputs.index(out_node_name)
-        #        del in_node.outputs[index]
-        index = out_node.inputs.index(in_node_name)
-        #        del out_node.inputs[index]
-        self.omit_nodes.append(in_node.layer_name)
+        sys.stderr.write("\nDone!\n")
 
     def directly_map(self, node):
         assert node.layer_type in self.directly_map_ops
         op_info = self.directly_map_ops[node.layer_type]
-        input = self.graph.get_node(node.layer.input[0], copy=True)
+        input = self.graph.get_node(node.layer.input[0])
         attr = dict()
         for param in op_info[1:]:
             tf_param_name = list(param.keys())[0]
             pd_param_name = list(param.values())[0]
             tf_param = node.get_attr(tf_param_name)
             attr[pd_param_name] = tf_param
-        node.fluid_code.add_layer(
-            op_info[0], inputs=input, output=node, param_attr=attr)
+
+        program.add_layer(
+            kernel="fluid.layers.{}".format(op_info[0]),
+            inputs={"x": input.name},
+            outputs=[node.name],
+            **attr)
 
     def elementwise_map(self, node):
         assert node.layer_type in self.elementwise_ops
         op_type = self.elementwise_ops[node.layer_type]
-        x = self.graph.get_node(node.layer.input[0], copy=True)
-        y = self.graph.get_node(node.layer.input[1], copy=True)
+        x = self.graph.get_node(node.layer.input[0])
+        y = self.graph.get_node(node.layer.input[1])
         x_shape = x.out_shapes[0]
         y_shape = y.out_shapes[0]
-        if len(x_shape) == 0:
-            x_shape = [1]
-        if len(y_shape) == 0:
-            y_shape = [1]
-        # incomplement broadcasting support for paddle
-        x_input = x
-        y_input = y
-        if len(x_shape) < len(y_shape):
-            unrevertable_ops = [
-                "elementwise_sub", "elementwise_div", "elementwise_floordiv",
-                "elementwise_mod", "elementwise_pow"
-            ]
-            if op_type not in unrevertable_ops:
-                x_input = y
-                y_input = x
-                x_shape = y.out_shapes[0]
-                if len(x_shape) == 0:
-                    x_shape = [1]
-                y_shape = x.out_shapes[0]
-                if len(y_shape) == 0:
-                    y_shape = [1]
-            else:
-                if len(x_shape) == 1 and len(y_shape) == 4 and x_shape[
-                        0] == y_shape[-1] and y_shape.count(-1) < 1:
-                    shape = [1, x_shape[0], 1, 1]
-                    attr = {"shape": shape}
-                    node.fluid_code.add_layer(
-                        "reshape",
-                        inputs=x_input,
-                        output="reshape_x",
-                        param_attr=attr)
-                    if y_shape[0] != 1:
-                        attr = {"expand_times": [y_shape[0], 1, 1, 1]}
-                        node.fluid_code.add_layer(
-                            "expand",
-                            inputs="reshape_x",
-                            output="reshape_x",
-                            param_attr=attr)
-                    inputs = {"x": "reshape_x", "y": y_input}
-                    node.fluid_code.add_layer(
-                        op_type, inputs=inputs, output=node, param_attr=None)
-                    return
-                else:
-                    raise Exception("Unexpected situation happend")
-
-        if len(x_shape) == 4 and len(y_shape) == 1:
-            if x_input.tf_data_format == "NHWC":
-                axis = 1
-            else:
-                axis = -1
-            attr = {"axis": axis}
-            inputs = {"x": x_input, "y": y_input}
-            node.fluid_code.add_layer(
-                op_type, inputs=inputs, output=node, param_attr=attr)
-            return
-
-        is_sub_seq = True
-        for i in range(len(y_shape)):
-            index = -1 * i - 1
-            if y_shape[index] != x_shape[index]:
-                is_sub_seq = False
-        if not is_sub_seq:
-            if x_shape.count(-1) > 2:
-                x_shape = self.decoder.infer_tensor_shape(x_input)
-            if y_shape.count(-1) > 2:
-                y_shape = self.decoder.infer_tensor_shape(y_input)
-            x_expand_times = [1] * len(x_shape)
-            y_expand_times = [1] * len(y_shape)
-            x_need_expand = False
-            y_need_expand = False
-            for i in range(len(y_shape)):
-                index = -1 * i - 1
-                if y_shape[index] != x_shape[index]:
-                    if y_shape[index] == 1:
-                        y_expand_times[index] = x_shape[index]
-                        y_need_expand = True
-                    elif x_shape[index] == 1:
-                        x_expand_times[index] = y_shape[index]
-                        x_need_expand = True
-                    else:
-                        raise Exception("Unexpected situation happend")
-            if x_need_expand:
-                if len(x_expand_times) == 3 and x.tf_data_format == "NHWC":
-                    x_expand_times = [x_expand_times[i] for i in [2, 0, 1]]
-                if len(x_expand_times) == 4 and x.tf_data_format == "NHWC":
-                    x_expand_times = [x_expand_times[i] for i in [0, 3, 1, 2]]
-                attr = {"expand_times": x_expand_times}
-                node.fluid_code.add_layer(
-                    "expand", inputs=x_input, output="x_tmp", param_attr=attr)
-                x_input = "x_tmp"
-            if y_need_expand:
-                if len(y_expand_times) == 3 and y.tf_data_format == "NHWC":
-                    y_expand_times = [y_expand_times[i] for i in [2, 0, 1]]
-                if len(y_expand_times) == 4 and y.tf_data_format == "NHWC":
-                    y_expand_times = [y_expand_times[i] for i in [0, 3, 1, 2]]
-                attr = {"expand_times": y_expand_times}
-                node.fluid_code.add_layer(
-                    "expand", inputs=y_input, output="y_tmp", param_attr=attr)
-                y_input = "y_tmp"
-        inputs = {"x": x_input, "y": y_input}
-        node.fluid_code.add_layer(
-            op_type, inputs=inputs, output=node, param_attr=None)
+        layer_id = program.add_layer(
+            kernel="fluid.layers.{}".format(op_type),
+            inputs={"x": x.name,
+                    "y": y.name},
+            outputs=[node.name])
+        program.layers[layer_id].input_shapes = {"x": x_shape, "y": y_shape}
+
+    def NotEqual(self, node):
+        x = self.graph.get_node(node.layer.input[0])
+        y = self.graph.get_node(node.layer.input[1])
+
+        program.add_layer(
+            kernel="fluid.layers.not_equal",
+            inputs={"x": x.name,
+                    "y": y.name},
+            outputs=[node.name])
 
     def Placeholder(self, node):
         shape = node.out_shapes[0]
         assert len(shape) != 0, "Unknown shape of input nodes[{}].".format(
             node.layer_name)
-        if node.tf_data_format == "NHWC" and len(shape) == 4:
-            shape = [shape[i] for i in [0, 3, 1, 2]]
-        elif node.tf_data_format == "NCHW" and len(shape) == 4:
-            self.graph.data_format_propagation(node)
         dtype = node.dtype
-        attr = {
-            'dtype': string(dtype),
-            'shape': shape,
-            'name': string(node.layer_name),
-            'append_batch_size': False
-        }
-
-        if shape[0] < 0:
-            self.batch_node = node
-
-        node.fluid_code.add_layer(
-            "data", inputs=None, output=node, param_attr=attr)
-
-    def OneShotIterator(self, node):
-        return self.Placeholder(node)
+        program.add_layer(
+            kernel="fluid.data",
+            inputs={},
+            outputs=[node.name],
+            dtype=string(dtype),
+            shape=shape,
+            name=string(node.name))
 
     def Const(self, node):
         shape = node.out_shapes[0]
@@ -286,615 +197,864 @@ class TFOpMapper(OpMapper):
         if len(shape) == 0:
             assert value.size == 1, "Unexpected situation happend"
             shape = [1]
+            if value == float('inf'):
+                value = "float('inf')"
             initializer = "Constant({})".format(value)
 
-        self.weights[node.layer_name] = node.value
-
-        if node.tf_data_format == "NHWC":
-            if len(shape) == 4:
-                shape = [shape[i] for i in [0, 3, 1, 2]]
-            if len(shape) == 3:
-                shape = [shape[i] for i in [2, 0, 1]]
-                self.weights[node.layer_name] = numpy.transpose(node.value,
-                                                                (2, 0, 1))
-        elif node.tf_data_format == "NCHW":
-            if len(shape) == 4:
-                self.graph.data_format_propagation(node)
-
-        attr = {
-            'dtype': string(dtype),
-            'shape': shape,
-            'name': string(node.layer_name),
-            'default_initializer': initializer
-        }
-        node.fluid_code.add_layer(
-            "create_parameter", inputs=None, output=node, param_attr=attr)
+        program.parameters[node.name] = node.value
+        program.add_layer(
+            kernel="fluid.layers.create_parameter",
+            inputs={},
+            outputs=[node.name],
+            dtype=string(dtype),
+            shape=shape,
+            name=string(node.name),
+            default_initializer=initializer)
 
     def Transpose(self, node):
-        input = self.graph.get_node(node.layer.input[0], copy=True)
-        perm = self.graph.get_node(node.layer.input[1], copy=True)
+        input = self.graph.get_node(node.layer.input[0])
+        perm = self.graph.get_node(node.layer.input[1])
         assert perm.layer_type == "Const", "Perm of transpose OP should be Const"
-        del self.weights[perm.layer_name.replace('/', '_')]
-        perm.fluid_code.clear()
         perm = perm.value.tolist()
 
-        if perm == [0, 3, 1, 2] and input.data_format == "NHWC":
-            input_name = input.layer_name
-            if hasattr(input, "index"):
-                input_name = input_name + "[{}]".format(input.index)
-            node.fluid_code.add_layer("{} = {}").format(node.layer_name,
-                                                        input_name)
-            node.tf_data_format = "NCHW"
-            self.graph.data_format_propagation(node)
-        elif perm == [0, 2, 3, 1] and input.tf_data_format == "NCHW":
-            input_name = input.layer_name
-            if hasattr(input, "index"):
-                input_name = input_name + "[{}]".format(input.index)
-            node.fluid_code.add_layer("{} = {}").format(node.layer_name,
-                                                        input_name)
-            node.tf_data_format = "NHWC"
-            self.graph.data_format_propagation(node)
-        elif len(input.out_shapes[0]) > 4:
-            tf_data_format = list(input.tf_data_format)
-            pd_data_format = list(input.pd_data_format)
-            new_perm = [i for i in range(len(perm))]
-            for i in range(len(perm)):
-                char0 = tf_data_format[i]
-                char1 = tf_data_format[perm[i]]
-                index0 = pd_data_format.index(char0)
-                index1 = pd_data_format.index(char1)
-                new_perm[index0] = index1
-            node.tf_data_format = [tf_data_format[i] for i in perm]
-            node.pd_data_format = [pd_data_format[i] for i in perm]
-            attr = {'perm': new_perm}
-            node.fluid_code.add_layer(
-                "transpose", inputs=input, output=node, param_attr=attr)
-        elif len(node.out_shapes[0]) != 4:
-            attr = {'perm': perm}
-            node.fluid_code.add_layer(
-                "transpose", inputs=input, output=node, param_attr=attr)
+        program.add_layer(
+            kernel="fluid.layers.transpose",
+            inputs={"x": input.name},
+            outputs=[node.name],
+            perm=perm)
+
+    def Fill(self, node):
+        dims = self.graph.get_node(node.layer.input[0])
+        input_value = self.graph.get_node(node.layer.input[1])
+        inputs = dict()
+        attr = dict()
+        assert input_value.layer_type == "Const", "Value of fill OP should be Const"
+        if dims.layer_type == "Const":
+            attr["shape"] = dims.value.tolist()
         else:
-            raise Exception("Unexpected situation happend in Transpose OP")
+            inputs["shape"] = dims.name
+        attr["dtype"] = string(input_value.dtype)
+        attr["value"] = input_value.value
 
-    def MaxPool(self, node):
-        input = self.graph.get_node(node.layer.input[0], copy=True)
+        program.add_layer(
+            "fluid.layers.fill_constant",
+            inputs=inputs,
+            outputs=[node.name],
+            **attr)
 
-        in_shape = input.out_shapes[0]
-        if in_shape.count(-1) > 2:
-            in_shape = self.decoder.infer_tensor(input).shape
+    def DepthToSpace(self, node):
+        input = self.graph.get_node(node.layer.input[0])
+
+        block_size = node.get_attr("block_size")
+        data_format = node.get_attr("data_format").decode()
+        if data_format == "NHWC":
+            n, h, w, c = input.out_shapes[0]
+        else:
+            n, c, h, w = input.out_shapes[0]
+
+        input_name = input.name
+        if data_format == "NHWC":
+            transpose_name = gen_name("depth_to_space", "transpose")
+            program.add_layer(
+                kernel="fluid.layers.transpose",
+                inputs={"x": input.name},
+                outputs=[transpose_name],
+                perm=[0, 3, 1, 2])
+            input_name = transpose_name
+
+        shape = [0, block_size * block_size, -1, h, w]
+        reshape_name = gen_name("depth_to_space", "reshape")
+        program.add_layer(
+            kernel="fluid.layers.reshape",
+            inputs={"x": input_name},
+            outputs=[reshape_name],
+            shape=shape)
+
+        transpose_name = gen_name("depth_to_space", "transpose")
+        program.add_layer(
+            kernel="fluid.layers.transpose",
+            inputs={"x": reshape_name},
+            outputs=[transpose_name],
+            perm=[0, 2, 1, 3, 4])
+
+        reshape_name = gen_name("depth_to_space", "reshape")
+        program.add_layer(
+            kernel="fluid.layers.reshape",
+            inputs={"x": transpose_name},
+            outputs=[reshape_name],
+            shape=[0, c, h, w])
+
+        program.add_layer(
+            kernel="fluid.layers.pixel_shuffle",
+            inputs={"x": reshape_name},
+            outputs=[node.name],
+            upscale_factor=block_size)
+
+        if data_format == "NHWC":
+            program.add_layer(
+                kernel="fluid.layers.transpose",
+                inputs={"x": node.name},
+                outputs=[node.name],
+                perm=[0, 2, 3, 1])
+
+    def MaxPool(self, node):
+        input = self.graph.get_node(node.layer.input[0])
 
         k_size = node.get_attr("ksize")
         strides = node.get_attr("strides")
         data_format = node.get_attr("data_format").decode()
         pad_mode = node.get_attr("padding").decode()
-        channel_first = data_format == "NCHW"
 
-        if not channel_first:
-            in_shape = [in_shape[i] for i in [0, 3, 1, 2]]
+        input_name = input.name
+        if data_format == "NHWC":
+            transpose_name = gen_name("max_pool", "transpose")
+            program.add_layer(
+                kernel="fluid.layers.transpose",
+                inputs={"x": input.name},
+                outputs=[transpose_name],
+                perm=[0, 3, 1, 2])
             strides = [strides[i] for i in [0, 3, 1, 2]]
             k_size = [k_size[i] for i in [0, 3, 1, 2]]
-        else:
-            self.graph.data_format_propagation(node)
-
-        attr = {
-            "pool_size": k_size[2:4],
-            "pool_type": string("max"),
-            "pool_padding": string(pad_mode),
-            "pool_stride": strides[2:4]
-        }
-        node.fluid_code.add_layer(
-            "pool2d", inputs=input, output=node, param_attr=attr)
+            input_name = transpose_name
+
+        program.add_layer(
+            kernel="fluid.layers.pool2d",
+            inputs={"input": input_name},
+            outputs=[node.name],
+            pool_size=k_size[2:4],
+            pool_type=string("max"),
+            pool_stride=strides[2:4],
+            pool_padding=string(pad_mode))
+
+        if data_format == "NHWC":
+            program.add_layer(
+                kernel="fluid.layers.transpose",
+                inputs={"x": node.name},
+                outputs=[node.name],
+                perm=[0, 2, 3, 1])
 
     def Conv2D(self, node):
-        input = self.graph.get_node(node.layer.input[0], copy=True)
-        kernel = self.graph.get_node(node.layer.input[1], copy=True)
-        assert kernel.layer_type == "Const", "Kernel of Conv2D should be Const"
-        self.add_omit_nodes(kernel.layer_name, node.layer_name)
+        input = self.graph.get_node(node.layer.input[0])
+        kernel = self.graph.get_node(node.layer.input[1])
 
-        in_shape = input.out_shapes[0]
-        if in_shape.count(-1) > 2:
-            in_shape = self.decoder.infer_tensor(input).shape
         k_size = kernel.out_shapes[0]
-        if k_size.count(-1) > 2:
-            k_size = self.decoder.infer_tensor(kernel).shape
-
         strides = node.get_attr("strides")
         dilations = node.get_attr("dilations")
         data_format = node.get_attr("data_format").decode()
         pad_mode = node.get_attr("padding").decode()
-        channel_first = data_format == "NCHW"
+        if data_format == "NHWC":
+            n, h, w, c = input.out_shapes[0]
+        else:
+            n, c, h, w = input.out_shapes[0]
 
-        self.weights[kernel.layer_name.replace('/', '_')] = numpy.transpose(
-            kernel.value, (3, 2, 0, 1))
+        if kernel.layer_type == 'Const':
+            kernel_value = kernel.value
+            kernel_weight_name = kernel.name.replace('/', '_')
+        else:
+            kernel_value = self.decoder.infer_tensor(kernel)
+            if kernel.layer_type == 'Split':
+                kernel_weight_name = "{}_{}_kernel".format(node.name,
+                                                           kernel.name)
+            else:
+                kernel_weight_name = kernel.name.replace('/', '_')
+        program.parameters[kernel_weight_name] = numpy.transpose(kernel_value,
+                                                                 (3, 2, 0, 1))
 
-        if not channel_first:
-            in_shape = [in_shape[i] for i in [0, 3, 1, 2]]
+        input_name = input.name
+        if data_format == "NHWC":
             strides = [strides[i] for i in [0, 3, 1, 2]]
             dilations = [dilations[i] for i in [0, 3, 1, 2]]
-        else:
-            self.graph.data_format_propagation(node)
-
-        attr = {
-            "bias_attr": False,
-            "param_attr": string(kernel.layer_name),
-            "num_filters": k_size[3],
-            "filter_size": k_size[0:2],
-            "stride": strides[2:4],
-            "dilation": dilations[2:4],
-            "padding": string(pad_mode)
-        }
+            transpose_name = gen_name("conv2d", "transpose")
+            program.add_layer(
+                kernel="fluid.layers.transpose",
+                inputs={"x": input.name},
+                outputs=[transpose_name],
+                perm=[0, 3, 1, 2])
+            input_name = transpose_name
+
+        if c == -1:
+            attr = {"shape": [0, k_size[2], 0, 0]}
             node.fluid_code.add_layer(
-            "conv2d", inputs=input, output=node, param_attr=attr)
+                "reshape", inputs=input, output=input, param_attr=attr)
+            program.add_layer(
+                kernel="fluid.layers.reshape",
+                inputs={"x": input_name},
+                outputs=[input_name],
+                shape=[0, k_size[2], 0, 0])
+
+        program.add_layer(
+            kernel="fluid.layers.conv2d",
+            inputs={"input": input_name},
+            outputs=[node.name],
+            bias_attr=False,
+            param_attr=string(kernel_weight_name),
+            num_filters=k_size[3],
+            filter_size=k_size[0:2],
+            stride=strides[2:4],
+            dilation=dilations[2:4],
+            padding=string(pad_mode))
+
+        if data_format == "NHWC":
+            program.add_layer(
+                kernel="fluid.layers.transpose",
+                inputs={"x": node.name},
+                outputs=[node.name],
+                perm=[0, 2, 3, 1])
 
     def BiasAdd(self, node):
-        input = self.graph.get_node(node.layer.input[0], copy=True)
-        bias = self.graph.get_node(node.layer.input[1], copy=True)
-        axis = -1
-        if input.tf_data_format == "NHWC" and len(input.out_shapes[0]) == 4:
-            axis = 1
-        inputs = {"x": input, "y": bias}
-        attr = {"axis": axis}
-        node.fluid_code.add_layer(
-            "elementwise_add", inputs=inputs, output=node, param_attr=attr)
+        input = self.graph.get_node(node.layer.input[0])
+        bias = self.graph.get_node(node.layer.input[1])
+        program.add_layer(
+            kernel="fluid.layers.elementwise_add",
+            inputs={"x": input.name,
+                    "y": bias.name},
+            outputs=[node.name])
 
     def FusedBatchNorm(self, node):
-        input = self.graph.get_node(node.layer.input[0], copy=True)
-        gamma = self.graph.get_node(node.layer.input[1], copy=True)
-        beta = self.graph.get_node(node.layer.input[2], copy=True)
-        moving_mean = self.graph.get_node(node.layer.input[3], copy=True)
-        moving_var = self.graph.get_node(node.layer.input[4], copy=True)
+        input = self.graph.get_node(node.layer.input[0])
+        gamma = self.graph.get_node(node.layer.input[1])
+        beta = self.graph.get_node(node.layer.input[2])
+        moving_mean = self.graph.get_node(node.layer.input[3])
+        moving_var = self.graph.get_node(node.layer.input[4])
         data_format = node.get_attr("data_format").decode()
-        channel_first = data_format == "NCHW"
 
         assert gamma.layer_type == "Const"
         assert beta.layer_type == "Const"
         assert moving_mean.layer_type == "Const"
         assert moving_var.layer_type == "Const"
-        self.add_omit_nodes(gamma.layer_name, node.layer_name)
-        self.add_omit_nodes(beta.layer_name, node.layer_name)
-        self.add_omit_nodes(moving_mean.layer_name, node.layer_name)
-        self.add_omit_nodes(moving_var.layer_name, node.layer_name)
-        if channel_first:
-            self.data_format_propagation(node)
-
-        attr = {
-            "epsilon": node.get_attr("epsilon"),
-            "param_attr": string(gamma.layer_name),
-            "bias_attr": string(beta.layer_name),
-            "moving_mean_name": string(moving_mean.layer_name),
-            "moving_variance_name": string(moving_var.layer_name),
-            "is_test": True
-        }
 
-        node.fluid_code.add_layer(
-            "batch_norm", inputs=input, output=node, param_attr=attr)
+        input_name = input.name
+        if data_format == "NHWC":
+            transpose_name = gen_name("batch_norm", "transpose")
+            program.add_layer(
+                kernel="fluid.layers.transpose",
+                inputs={"x": input.name},
+                outputs=[transpose_name],
+                perm=[0, 3, 1, 2])
+            input_name = transpose_name
+
+        program.add_layer(
+            kernel="fluid.layers.batch_norm",
+            inputs={"input": input_name},
+            outputs=[node.name],
+            epsilon=node.get_attr("epsilon"),
+            param_attr=string(gamma.name),
+            bias_attr=string(beta.name),
+            moving_mean_name=string(moving_mean.name),
+            moving_variance_name=string(moving_var.name),
+            is_test=True)
+
+        if data_format == "NHWC":
+            program.add_layer(
+                kernel="fluid.layers.transpose",
+                inputs={"x": node.name},
+                outputs=[node.name],
+                perm=[0, 2, 3, 1])
+
+    def Mean(self, node):
+        input = self.graph.get_node(node.layer.input[0])
+        reduce_idx = self.graph.get_node(node.layer.input[1])
+        assert reduce_idx.layer_type == "Const", "Only support Const parameter[reduce_idx]"
+        dims = reduce_idx.value.tolist()
+        keep_dims = node.get_attr("keep_dims")
+
+        program.add_layer(
+            kernel="fluid.layers.reduce_mean",
+            inputs={"input": input.name},
+            outputs=[node.name],
+            dim=dims,
+            keep_dim=keep_dims)
+
+    def Reshape(self, node):
+        input = self.graph.get_node(node.layer.input[0])
+        param = self.graph.get_node(node.layer.input[1])
+
+        input_name = input.name
+        if input.dtype == 'bool':
+            cast_name = gen_name('reshape', 'cast')
+            program.add_layer(
+                kernel="fluid.layers.cast",
+                inputs={"x": input_name},
+                outputs=[cast_name],
+                dtype="'int32'")
+            input_name = cast_name
+
+        if param.layer_type == "Const":
+            shape = param.value.tolist()
+            program.add_layer(
+                kernel="fluid.layers.reshape",
+                inputs={"x": input_name},
+                outputs=[node.name],
+                shape=shape)
+        else:
+            program.add_layer(
+                kernel="fluid.layers.reshape",
+                inputs={"x": input_name,
+                        "shape": param.name},
+                outputs=[node.name])
+        if param.layer_type != "Const":
+            out_shape = numpy.array(node.out_shapes[0])
+            if (out_shape > 0).any():
+                out_shape[out_shape < 0] = 0
+                program.add_layer(
+                    kernel="fluid.layers.reshape",
+                    inputs={"x": node.name},
+                    outputs=[node.name],
+                    shape=out_shape.tolist())
+
+        if input.dtype == 'bool':
+            program.add_layer(
+                kernel="fluid.layers.cast",
+                inputs={"x": node.name},
+                outputs=[node.name],
+                dtype="'bool'")
+
+    def Pad(self, node):
+        input = self.graph.get_node(node.layer.input[0])
+        paddings = self.graph.get_node(node.layer.input[1])
+        assert paddings.layer_type == "Const", "Padding should be Const"
+        paddings = paddings.value.flatten().tolist()
 
-    def FusedBatchNormV3(self, node):
-        return self.FusedBatchNorm(node)
+        if len(input.out_shapes[0]) == 4:
+            if paddings[0] + paddings[1] + paddings[6] + paddings[7] == 0:
+                new_padding = paddings[2:6]
+                transpose_name = gen_name("pad", "transpose")
+                program.add_layer(
+                    kernel="fluid.layers.transpose",
+                    inputs={"x": input.name},
+                    outputs=[transpose_name],
+                    perm=[0, 3, 1, 2])
+                program.add_layer(
+                    kernel="fluid.layers.pad2d",
+                    inputs={"input": transpose_name},
+                    outputs=[node.name],
+                    paddings=new_padding)
+                program.add_layer(
+                    kernel="fluid.layers.transpose",
+                    inputs={"x": node.name},
+                    outputs=[node.name],
+                    perm=[0, 2, 3, 1])
+                return
+
+        program.add_layer(
+            kernel="fluid.layers.pad",
+            inputs={"input": input.name},
+            outputs=[node.name],
+            paddings=paddings)
+
+    def Squeeze(self, node):
+        input = self.graph.get_node(node.layer.input[0])
+        squeeze_dims = node.get_attr('squeeze_dims')
+        program.add_layer(
+            kernel="fluid.layers.squeeze",
+            inputs={"input": input.name},
+            outputs=[node.name],
+            axes=squeeze_dims)
+
+    def Softmax(self, node):
+        input = self.graph.get_node(node.layer.input[0])
+        axis = node.get_attr("axis")
+        program.add_layer(
+            kernel="fluid.layers.softmax",
+            inputs={"input": input.name},
+            outputs=[node.name],
+            axis=axis)
+
+    def Shape(self, node):
+        input = self.graph.get_node(node.layer.input[0])
+        input_name = input.name
+        if input.dtype == 'bool':
+            cast_name = gen_name('shape', 'cast')
+            program.add_layer(
+                kernel="fluid.layers.cast",
+                inputs={"x": input.name},
+                outputs=[cast_name],
+                dtype="'int32'")
+            input_name = cast_name
+        program.add_layer(
+            kernel="fluid.layers.shape",
+            inputs={"input": input_name},
+            outputs=[node.name])
+
+    def ArgMax(self, node):
+        input = self.graph.get_node(node.layer.input[0])
+        axis = self.graph.get_node(node.layer.input[1])
+        assert axis.layer_type == "Const", "ArgMax only support Const parameter"
+        axis = axis.value
+        program.add_layer(
+            kernel="fluid.layers.argmax",
+            inputs={"x": input.name},
+            outputs=[node.name],
+            axis=axis)
+
+    def MatMul(self, node):
+        x = self.graph.get_node(node.layer.input[0])
+        y = self.graph.get_node(node.layer.input[1])
+        transpose_a = node.get_attr('transpose_a')
+        transpose_b = node.get_attr('transpose_b')
+        if transpose_a is None:
+            transpose_a = node.get_attr('adj_x')
+        if transpose_b is None:
+            transpose_b = node.get_attr('adj_y')
+        program.add_layer(
+            kernel="fluid.layers.matmul",
+            inputs={"x": x.name,
+                    "y": y.name},
+            outputs=[node.name],
+            transpose_x=transpose_a,
+            transpose_y=transpose_b)
+
+    def BatchMatMul(self, node):
+        return self.MatMul(node)
+
+    def BatchMatMulV2(self, node):
+        return self.MatMul(node)
 
     def DepthwiseConv2dNative(self, node):
-        input = self.graph.get_node(node.layer.input[0], copy=True)
-        kernel = self.graph.get_node(node.layer.input[1], copy=True)
+        input = self.graph.get_node(node.layer.input[0])
+        kernel = self.graph.get_node(node.layer.input[1])
         assert kernel.layer_type == "Const", "Kernel of DepthwiseConv2DNative should be Const"
-        self.add_omit_nodes(kernel.layer_name, node.layer_name)
 
         in_shape = input.out_shapes[0]
-        if in_shape.count(-1) > 2:
-            in_shape = self.decoder.infer_tensor(input).shape
         k_size = kernel.out_shapes[0]
-        if k_size.count(-1) > 2:
-            k_size = self.decoder.infer_tensor(kernel).shape
-
         strides = node.get_attr("strides")
         dilations = node.get_attr("dilations")
         data_format = node.get_attr("data_format").decode()
         pad_mode = node.get_attr("padding").decode()
-        channel_first = data_format == "NCHW"
 
-        self.weights[kernel.layer_name.replace('/', '_')] = numpy.transpose(
-            kernel.value, (2, 3, 0, 1))
+        program.parameters[kernel.layer_name.replace(
+            '/', '_')] = numpy.transpose(kernel.value, (2, 3, 0, 1))
 
-        if not channel_first:
+        input_name = input.name
+        if data_format == "NHWC":
             in_shape = [in_shape[i] for i in [0, 3, 1, 2]]
             strides = [strides[i] for i in [0, 3, 1, 2]]
             dilations = [dilations[i] for i in [0, 3, 1, 2]]
-        else:
-            self.data_format_propagation(node)
-
-        attr = {
-            "bias_attr": False,
-            "param_attr": string(kernel.layer_name),
-            "num_filters": in_shape[1],
-            "filter_size": k_size[0:2],
-            "stride": strides[2:4],
-            "dilation": dilations[2:4],
-            "groups": k_size[3] * in_shape[1],
-            "use_cudnn": False,
-            "padding": string(pad_mode)
-        }
-        node.fluid_code.add_layer(
-            "conv2d", inputs=input, output=node, param_attr=attr)
-
-    def Reshape(self, node):
-        input = self.graph.get_node(node.layer.input[0], copy=True)
-        param = self.graph.get_node(node.layer.input[1], copy=True)
-        is_variable = False
-        if param.layer_type == "Const":
-            attr = {"shape": param.value.tolist()}
-            self.add_omit_nodes(param.layer_name, node.layer_name)
-        else:
-            # Here is a trick method to solove tensor parameter in tensorflow
-            shape = self.decoder.infer_shape_tensor(param, node.out_shapes[0])
-            if shape.count(-1) <= 1:
-                attr = {"shape": shape}
-                self.add_omit_nodes(param.layer_name, node.layer_name)
-            elif shape.count(-1) == 2 and shape[0] == -1:
-                shape[0] = 0
-                attr = {"shape": shape}
-                self.add_omit_nodes(param.layer_name, node.layer_name)
-            else:
-                assert len(param.out_shapes[
-                    0]) == 1, "Unexpected situation of shape parameter"
-                attr = {"shape": [-1]}
-                node.fluid_code.add_layer(
-                    "reshape",
-                    inputs=param,
-                    output="shape_param",
-                    param_attr=attr)
-                attr = {"num_or_sections": param.out_shapes[0][0], "dim": 0}
-                node.fluid_code.add_layer(
-                    "split", inputs="shape_param", output=node, param_attr=attr)
-                new_param = "["
-                for i in range(param.out_shapes[0][0]):
-                    new_param += (node.layer_name + "[{}]".format(i) + ", ")
-                new_param = new_param.strip(", ") + "]"
-                attr = {"shape": new_param}
-                is_variable = True
-
-        # to change [192, -1]->[-1, 192], allways put -1 in the first dimension
-        # optimization for Paddle-Lite
-        in_shape = input.out_shapes[0]
-        if not is_variable and in_shape.count(-1) < 1:
-            total_size = 1
-            for i in range(len(in_shape)):
-                total_size *= in_shape[i]
-            for i in range(len(attr["shape"])):
-                if attr["shape"][i] == 0:
-                    attr["shape"][i] = in_shape[i]
-                if attr["shape"][i] != -1:
-                    total_size /= attr["shape"][i]
-            if attr["shape"].count(-1) > 0:
-                index = attr["shape"].index(-1)
-                attr["shape"][index] = int(total_size)
-                attr["shape"][0] = -1
-
-        if len(input.out_shapes[0]) == 4 and node.tf_data_format == "NHWC":
-            if len(attr["shape"]) < 3:
-                perm = {"perm": [0, 2, 3, 1]}
-                node.fluid_code.add_layer(
-                    "transpose", inputs=input, output=node, param_attr=perm)
-                node.fluid_code.add_layer(
-                    "reshape", inputs=node, output=node, param_attr=attr)
-                return
-
-        if len(attr["shape"]) == 4 and node.tf_data_format == "NHWC":
-            input_shape = self.decoder.infer_tensor(input).shape
-            if input_shape[1] == attr["shape"][1]:
-                attr["shape"] = [attr["shape"][i] for i in [0, 3, 1, 2]]
-            else:
-                perm = {"perm": [0, 2, 3, 1]}
-                node.fluid_code.add_layer(
-                    "transpose", inputs=input, output=node, param_attr=perm)
-                node.fluid_code.add_layer(
-                    "reshape", inputs=node, output=node, param_attr=attr)
-                perm = {"perm": [0, 3, 1, 2]}
-                node.fluid_code.add_layer(
-                    "transpose", inputs=node, output=node, param_attr=perm)
-                return
-        if len(attr["shape"]) == 5:
-            attr["shape"] = [attr["shape"][i] for i in [0, 1, 4, 2, 3]]
-
-        node.fluid_code.add_layer(
-            "reshape", inputs=input, output=node, param_attr=attr)
+            transpose_name = gen_name('depthwise_conv2d', 'transpose')
+            program.add_layer(
+                kernel="fluid.layers.transpose",
+                inputs={"x": input.name},
+                outputs=[transpose_name],
+                perm=[0, 3, 1, 2])
+            input_name = transpose_name
+
+        program.add_layer(
+            kernel="fluid.layers.conv2d",
+            inputs={"input": input_name},
+            outputs=[node.name],
+            num_filters=in_shape[1],
+            filter_size=k_size[0:2],
+            stride=strides[2:4],
+            dilation=dilations[2:4],
+            groups=k_size[3] * in_shape[1],
+            padding=string(pad_mode),
+            param_attr=string(kernel.layer_name),
+            bias_attr=False)
+
+        if data_format == "NHWC":
+            program.add_layer(
+                kernel="fluid.layers.transpose",
+                inputs={"x": node.name},
+                outputs=[node.name],
+                perm=[0, 2, 3, 1])
 
     def AvgPool(self, node):
-        input = self.graph.get_node(node.layer.input[0], copy=True)
-
-        in_shape = input.out_shapes[0]
-        if in_shape.count(-1) > 2:
-            in_shape = self.decoder.infer_tensor(input).shape
+        input = self.graph.get_node(node.layer.input[0])
 
         k_size = node.get_attr("ksize")
         strides = node.get_attr("strides")
         data_format = node.get_attr("data_format").decode()
         pad_mode = node.get_attr("padding").decode()
-        channel_first = data_format == "NCHW"
 
-        if not channel_first:
-            in_shape = [in_shape[i] for i in [0, 3, 1, 2]]
+        input_name = input.name
+        if data_format == "NHWC":
+            transpose_name = gen_name("avg_pool", "transpose")
+            program.add_layer(
+                kernel="fluid.layers.transpose",
+                inputs={"x": input.name},
+                outputs=[transpose_name],
+                perm=[0, 3, 1, 2])
             strides = [strides[i] for i in [0, 3, 1, 2]]
             k_size = [k_size[i] for i in [0, 3, 1, 2]]
-        else:
-            self.graph.data_format_propagation(node)
-
-        attr = {
-            "pool_size": k_size[2:4],
-            "pool_type": string("avg"),
-            "pool_stride": strides[2:4],
-            "pool_padding": string(pad_mode)
-        }
-        node.fluid_code.add_layer(
-            "pool2d", inputs=input, output=node, param_attr=attr)
-
-    def SplitV(self, node):
-        input = self.graph.get_node(node.layer.input[0], copy=True)
-        num_sections = self.graph.get_node(node.layer.input[1], copy=True)
-        dim = self.graph.get_node(node.layer.input[2], copy=True)
-        assert num_sections.layer_type == "Const"
-        assert dim.layer_type == "Const"
-        self.add_omit_nodes(num_sections.layer_name, node.layer_name)
-        self.add_omit_nodes(dim.layer_name, node.layer_name)
-        dim = dim.value
-        if input.tf_data_format == "NHWC" and len(input.out_shapes[0]) == 4:
-            dim = nhwc_dim_to_nchw(input, dim)
-        attr = {
-            "num_or_sections": num_sections.value.tolist(),
-            "dim": dim.value
-        }
-        node.fluid_code.add_layer(
-            "split", inputs=input, output=node, param_attr=attr)
-
-    def ConcatV2(self, node):
-        inputs = [
-            self.graph.get_node(
-                name, copy=True) for name in node.layer.input[:-1]
-        ]
-        axis = self.graph.get_node(node.layer.input[-1], copy=True)
-        assert axis.layer_type == "Const"
-        self.add_omit_nodes(axis.layer_name, node.layer_name)
-        axis = axis.value
-        if inputs[0].tf_data_format == "NHWC" and len(inputs[0].out_shapes[
-                0]) == 4:
-            axis = nhwc_dim_to_nchw(inputs[0], axis)
-        attr = {"axis": axis}
-        node.fluid_code.add_layer(
-            "concat", inputs=inputs, output=node, param_attr=attr)
-
-    def Tile(self, node):
-        input = self.graph.get_node(node.layer.input[0], copy=True)
-        expand_times = self.graph.get_node(node.layer.input[1], copy=True)
-        self.add_omit_nodes(expand_times.layer_name, node.layer_name)
-        if expand_times.layer_type == "Const":
-            expand_times = expand_times.value.tolist()
-        else:
-            expand_times = self.decoder.infer_shape_tensor(expand_times)
-        if input.tf_data_format == "NHWC":
-            if len(input.out_shapes[0]) == 4:
-                expand_times = [expand_times[i] for i in [0, 3, 1, 2]]
-            elif len(input.out_shapes[0]) == 3:
-                expand_times = [expand_times[i] for i in [2, 0, 1]]
-        for i in range(len(expand_times)):
-            if expand_times[i] < 0:
-                expand_times[i] = 1
-
-        attr = {"expand_times": expand_times}
-        node.fluid_code.add_layer(
-            "expand", inputs=input, output=node, param_attr=attr)
+            input_name = transpose_name
+
+        program.add_layer(
+            kernel="fluid.layers.pool2d",
+            inputs={"input": input_name},
+            outputs=[node.name],
+            pool_size=k_size[2:4],
+            pool_type=string("avg"),
+            pool_stride=strides[2:4],
+            pool_padding=string(pad_mode))
+
+        if data_format == "NHWC":
+            program.add_layer(
+                kernel="fluid.layers.transpose",
+                inputs={"x": node.name},
+                outputs=[node.name],
+                perm=[0, 2, 3, 1])
 
     def Pack(self, node):
-        inputs = [
-            self.graph.get_node(
-                name, copy=True) for name in node.layer.input
-        ]
+        inputs = [self.graph.get_node(name) for name in node.layer.input]
+        input_names = [i.name for i in inputs]
         axis = node.get_attr("axis")
-        if inputs[0].tf_data_format == "NHWC" and len(inputs[0].out_shapes[
-                0]) == 4:
-            tf_data_format = list(inputs[0].tf_data_format)
-            tf_data_format.insert(axis, str(len(tf_data_format)))
-            axis = nhwc_dim_to_nchw(inputs[0], axis)
-            pd_data_format = list(inputs[0].pd_data_format)
-            pd_data_format.insert(axis, str(len(pd_data_format)))
-            node.tf_data_format = "".join(tf_data_format)
-            node.pd_data_format = "".join(pd_data_format)
-
-        attr = {"axis": axis}
-        node.fluid_code.add_layer(
-            "stack", inputs=inputs, output=node, param_attr=attr)
-
-    def Pad(self, node):
-        input = self.graph.get_node(node.layer.input[0], copy=True)
-        paddings = self.graph.get_node(node.layer.input[1], copy=True)
-        assert paddings.layer_type == "Const", "Padding should be Const"
-        self.add_omit_nodes(paddings.layer_name, node.layer_name)
-        paddings = paddings.value.flatten().tolist()
-        if input.tf_data_format == "NHWC" and len(input.out_shapes[0]) == 4:
-            paddings = [paddings[i] for i in [0, 1, 6, 7, 2, 3, 4, 5]]
-
-        pad_op = "pad"
-        if len(input.out_shapes[0]) == 4:
-            if paddings[0] + paddings[1] + paddings[2] + paddings[3] == 0:
-                paddings = paddings[4:]
-                pad_op = "pad2d"
-        attr = {"paddings": paddings}
-        node.fluid_code.add_layer(
-            pad_op, inputs=input, output=node, param_attr=attr)
+        program.add_layer(
+            kernel="fluid.layers.stack",
+            inputs={"x": input_names},
+            outputs=[node.name],
+            axis=axis)
+        if len(node.out_shapes[0]) == 1:
+            program.add_layer(
+                kernel="fluid.layers.reshape",
+                inputs={"x": node.name},
+                outputs=[node.name],
+                shape=[-1])
+
+    def Unpack(self, node):
+        input = self.graph.get_node(node.layer.input[0])
+        axis = node.get_attr("axis")
+        num = node.get_attr("num")
+        shape = input.out_shapes[0]
+        input_name = input.name
+        if len(shape) == 1:
+            if shape[0] > 0 and num == shape[0]:
+                program.add_layer(
+                    kernel="fluid.layers.unsqueeze",
+                    inputs={"input": input.name},
+                    outputs=[node.name],
+                    axes=[0])
+                input_name = node.name
+                axis = 1
+            else:
+                raise Exception("Unexpected situation happend in Unpack OP")
+        program.add_layer(
+            kernel="fluid.layers.unstack",
+            inputs={"x": input_name},
+            outputs=["{}_p{}".format(node.layer_name, i) for i in range(num)],
+            axis=axis,
+            num=num)
 
-    def MirrorPad(self, node):
-        input = self.graph.get_node(node.layer.input[0], copy=True)
-        paddings = self.graph.get_node(node.layer.input[1], copy=True)
-        assert paddings.layer_type == "Const", "Padding should be Const"
-        self.add_omit_nodes(paddings.layer_name, node.layer_name)
-        paddings = paddings.value.flatten().tolist()
-        mode = node.get_attr("mode").decode()
-        assert mode == "REFLECT", "Only support 'REFLECT` mode in MirrorPad"
-        if input.tf_data_format == "NHWC" and len(input.out_shapes[0]) == 4:
-            paddings = [paddings[i] for i in [0, 1, 6, 7, 2, 3, 4, 5]]
+    def ConcatV2(self, node):
+        inputs = [self.graph.get_node(name) for name in node.layer.input[:-1]]
+        axis = self.graph.get_node(node.layer.input[-1])
+        assert axis.layer_type == "Const", "axis for ConcatV2 must be type Const"
+        axis = axis.value
+        if axis < 0:
+            axis += len(inputs[0].out_shapes[0])
+
+        input_names = [i.name for i in inputs]
+        for i, ipt in enumerate(inputs):
+            if node.dtype == 'bool':
+                cast_name = gen_name('concat', 'cast')
+                program.add_layer(
+                    kernel="fluid.layers.cast",
+                    inputs={"x": ipt.name},
+                    outputs=[cast_name],
+                    dtype="'int32'")
+                input_names[i] = cast_name
+        program.add_layer(
+            kernel="fluid.layers.concat",
+            inputs={"input": input_names},
+            outputs=[node.name],
+            axis=axis)
+        if node.dtype == 'bool':
+            program.add_layer(
+                kernel="fluid.layers.cast",
+                inputs={"x": node.name},
+                outputs=[node.name],
+                dtype="'bool'")
 
-        pad_op = "pad"
-        if len(input.out_shapes[0]) == 4:
-            if paddings[0] + paddings[1] + paddings[2] + paddings[3] == 0:
-                paddings = paddings[4:]
-                pad_op = "pad2d"
-        attr = {"paddings": paddings, "mode": string("reflect")}
-        node.fluid_code.add_layer(
-            pad_op, inputs=input, output=node, param_attr=attr)
+    def StridedSlice(self, node):
+        input = self.graph.get_node(node.layer.input[0])
+        begin = self.graph.get_node(node.layer.input[1])
+        end = self.graph.get_node(node.layer.input[2])
+        strides = self.graph.get_node(node.layer.input[3])
 
-    def Range(self, node):
-        start = self.graph.get_node(node.layer.input[0], copy=True)
-        limit = self.graph.get_node(node.layer.input[1], copy=True)
-        delta = self.graph.get_node(node.layer.input[2], copy=True)
-        self.add_omit_nodes(start.layer_name, node.layer_name)
-        self.add_omit_nodes(limit.layer_name, node.layer_name)
-        self.add_omit_nodes(delta.layer_name, node.layer_name)
-        if start.layer_type == "Const":
-            start = start.value
+        if strides.layer_type == "Const":
+            strides = strides.value.tolist()
         else:
-            start = self.decoder.infer_tensor(start)
-        if limit.layer_type == "Const":
-            limit = limit.value
+            strides = self.decoder.infer_shape_tensor(strides)
+        if begin.layer_type == "Const":
+            begin = begin.value.tolist()
         else:
-            limit = self.decoder.infer_tensor(limit)
-        if delta.layer_type == "Const":
-            delta = delta.value
+            begin = self.decoder.infer_shape_tensor(begin)
+        if end.layer_type == "Const":
+            end = end.value.tolist()
         else:
-            delta = self.decoder.infer_tensor(delta)
+            end = self.decoder.infer_shape_tensor(end)
 
-        inputs = {"start": start, "end": limit, "step": delta}
-        attr = {"dtype": string(node.dtype)}
-        node.fluid_code.add_layer(
-            "range", inputs=inputs, output=node, param_attr=attr)
-
-    def Mean(self, node):
-        input = self.graph.get_node(node.layer.input[0], copy=True)
-        reduce_idx = self.graph.get_node(node.layer.input[1], copy=True)
-        assert reduce_idx.layer_type == "Const", "Only support Const parameter[reduce_idx]"
-        dims = reduce_idx.value.tolist()
-        keep_dims = node.get_attr("keep_dims")
+        assert len(set(strides)) == 1 and strides[
+            0] == 1, "Only support strides be 1 in StridedSlice OP"
 
-        if input.tf_data_format == "NHWC" and len(input.out_shapes[0]) == 4:
-            for i in range(len(dims)):
-                dims[i] = nhwc_dim_to_nchw(input, dims[i])
+        if len(begin) < len(input.out_shapes[0]):
+            begin = begin + [0] * (len(input.out_shapes[0]) - len(begin))
+        if len(end) < len(input.out_shapes[0]):
+            end = end + [0] * (len(input.out_shapes[0]) - len(end))
+        for i in range(len(end)):
+            if end[i] == 0:
+                end[i] = 999999
 
-        attr = {"dim": dims, "keep_dim": keep_dims}
-        node.fluid_code.add_layer(
-            "reduce_mean", inputs=input, output=node, param_attr=attr)
+        begin_mask = node.get_attr('begin_mask')
+        end_mask = node.get_attr('end_mask')
+        ellipsis_mask = node.get_attr('ellipsis_mask')
+        new_axis_mask = node.get_attr('new_axis_mask')
+        shrink_axis_mask = node.get_attr('shrink_axis_mask')
 
-    def MatMul(self, node):
-        x = self.graph.get_node(node.layer.input[0], copy=True)
-        y = self.graph.get_node(node.layer.input[1], copy=True)
-        transpose_a = node.get_attr('transpose_a')
-        transpose_b = node.get_attr('transpose_b')
-        inputs = {"x": x, "y": y}
-        # fix paddle shape infer problem
-        # should be removed after paddle 1.6
-        if x.out_shapes[0][-1] < 0 and y.out_shapes[0][0] > 0:
-            shape = x.out_shapes[0]
-            shape[-1] = y.out_shapes[0][0]
-            attr = {"shape": shape}
-            node.fluid_code.add_layer(
-                "reshape", inputs=x, output=x, param_attr=attr)
-        attr = {"transpose_x": transpose_a, "transpose_y": transpose_b}
-        node.fluid_code.add_layer(
-            "matmul", inputs=inputs, output=node, param_attr=attr)
+        assert ellipsis_mask == 0, "(OP:{} Name:{})Only support ellipsis_mask be 0[now: {}] n StridedSlice OP".format(
+            node.layer_type, node.layer.name, ellipsis_mask)
+
+        # TODO codes without validation
+        # Use it carefully
+        new_begin = list()
+        new_end = list()
+        new_axes = list()
+        shrink_axes = list()
+        for i, item in enumerate(begin):
+            mask = (new_axis_mask >> i) & 1
+            if mask != 0:
+                new_axes.append(i)
+                continue
 
-    def ArgMax(self, node):
-        input = self.graph.get_node(node.layer.input[0], copy=True)
-        axis = self.graph.get_node(node.layer.input[1], copy=True)
-        assert axis.layer_type == "Const", "ArgMax only support Const parameter"
-        self.add_omit_nodes(axis.layer_name, node.layer_name)
-        axis = axis.value
-        if input.tf_data_format == "NHWC" and len(input.out_shapes[0]) == 4:
-            axis = nhwc_dim_to_nchw(input, axis)
-        attr = {"axis": axis}
-        node.fluid_code.add_layer(
-            "argmax", inputs=input, output=node, param_attr=attr)
+            mask = (shrink_axis_mask >> i) & 1
+            if mask != 0:
+                shrink_axes.append(i)
 
-    def StridedSlice(self, node):
-        input = self.graph.get_node(node.layer.input[0], copy=True)
-        begin = self.graph.get_node(node.layer.input[1], copy=True)
-        end = self.graph.get_node(node.layer.input[2], copy=True)
-        strides = self.graph.get_node(node.layer.input[3], copy=True)
-        assert begin.layer_type == "Const"
-        assert end.layer_type == "Const"
-        assert strides.layer_type == "Const"
-        self.add_omit_nodes(begin.layer_name, node.layer_name)
-        self.add_omit_nodes(end.layer_name, node.layer_name)
-        self.add_omit_nodes(strides.layer_name, node.layer_name)
-        strides = strides.value.tolist()
-        assert len(set(strides)) == 1 and strides[0] == 1
-
-        begin = begin.value.tolist()
-        end = end.value.tolist()
-        if input.tf_data_format == "NHWC" and len(input.out_shapes[0]) == 4:
-            begin = [begin[i] for i in [0, 3, 1, 2]]
-            end = [end[i] for i in [0, 3, 1, 2]]
+            mask = (begin_mask >> i) & 1
+            if mask != 0:
+                new_begin.append(0)
+            else:
+                new_begin.append(item)
 
-        for i in range(len(end)):
-            if end[i] == 0:
-                end[i] = 999999
+            mask = (end_mask >> i) & 1
+            if mask != 0:
+                new_end.append(999999)
+            else:
+                new_end.append(end[i])
+
+        program.add_layer(
+            kernel="fluid.layers.slice",
+            inputs={"input": input.name},
+            outputs=[node.name],
+            axes=[i for i in range(len(new_begin))],
+            starts=new_begin,
+            ends=new_end)
+        if len(new_axes) > 0:
+            program.add_layer(
+                kernel="fluid.layers.unsqueeze",
+                inputs={"input": node.name},
+                outputs=[node.name],
+                axes=new_axes)
+        if len(shrink_axes) > 0:
+            if len(input.out_shapes[0]) + len(new_axes) <= 1:
+                pass
+            else:
+                program.add_layer(
+                    kernel="fluid.layers.squeeze",
+                    inputs={"input": node.name},
+                    outputs=[node.name],
+                    axes=shrink_axes)
 
-        attr = {
-            "axes": [i for i in range(len(strides))],
-            "starts": begin,
-            "ends": end
-        }
+    def Split(self, node):
+        dim = self.graph.get_node(node.layer.input[0])
+        input = self.graph.get_node(node.layer.input[1])
+        assert dim.layer_type == "Const"
+        num_split = node.get_attr('num_split')
+        dim = dim.value
 
-        shrink_axis_mask = node.get_attr('shrink_axis_mask')
-        squeeze_dims = list()
-        for i in range(len(begin)):
-            x = shrink_axis_mask >> i & 1
-            if x == 1:
-                squeeze_dims.append(i)
-        node.fluid_code.add_layer(
-            "slice", inputs=input, output=node, param_attr=attr)
-        if shrink_axis_mask > 0 and len(input.out_shapes[0]) == 5:
-            attr = {"axes": squeeze_dims}
-            node.fluid_code.add_layer(
-                "squeeze", inputs=node, output=node, param_attr=attr)
+        program.add_layer(
+            kernel="fluid.layers.split",
+            inputs={"input": input.name},
+            outputs=[
+                "{}_p{}".format(node.layer_name, i) for i in range(num_split)
+            ],
+            num_or_sections=num_split,
+            dim=dim)
 
     def Slice(self, node):
-        input = self.graph.get_node(node.layer.input[0], copy=True)
-        begin = self.graph.get_node(node.layer.input[1], copy=True)
-        size = self.graph.get_node(node.layer.input[2], copy=True)
-        self.add_omit_nodes(begin.layer_name, node.layer_name)
-        self.add_omit_nodes(size.layer_name, node.layer_name)
+        input = self.graph.get_node(node.layer.input[0])
+        begin = self.graph.get_node(node.layer.input[1])
+        size = self.graph.get_node(node.layer.input[2])
+
+        inputs = {"x": input.name}
+        attrs = {}
         if begin.layer_type == "Const":
             begin = begin.value.tolist()
+            attrs['offsets'] = begin
         else:
+            #             shape = begin.out_shapes[0]
+            #             reshape_name = gen_name("slice", "reshape")
+            #             program.add_layer(
+            #                 kernel="fluid.layers.reshape",
+            #                 inputs={"x": begin.name},
+            #                 outputs=[reshape_name],
+            #                 shape=shape)
+            #             inputs['offsets'] = reshape_name
             begin = self.decoder.infer_tensor(begin).tolist()
-        if size.layer_type == "const":
+            attrs['offsets'] = begin
+        if size.layer_type == "Const":
             size = size.value.tolist()
+            attrs['shape'] = size
         else:
-            size = self.decoder.infer_tensor(size).tolist()
+            shape = size.out_shapes[0]
+            reshape_name = gen_name("slice", "reshape")
+            program.add_layer(
+                kernel="fluid.layers.reshape",
+                inputs={"x": size.name},
+                outputs=[reshape_name],
+                shape=shape)
+            inputs['shape'] = reshape_name
+        program.add_layer(
+            kernel="fluid.layers.crop_tensor",
+            inputs=inputs,
+            outputs=[node.name],
+            **attrs)
 
-        if input.tf_data_format == "NHWC" and len(input.out_shapes[0]) == 4:
-            size = [size[i] for i in [0, 3, 1, 2]]
-            begin = [begin[i] for i in [0, 3, 1, 2]]
+    def ResizeNearestNeighbor(self, node):
+        input = self.graph.get_node(node.layer.input[0])
+        resize_shape = self.graph.get_node(node.layer.input[1])
+        data_format = "NHWC"
+        inputs = {"input": input.name}
+        attrs = {"align_corners": node.get_attr("align_corners")}
 
-        for i in range(len(size)):
-            if size[i] < 0:
-                size[i] = 99999999
+        if resize_shape.layer_type == "Const":
+            resize_shape = resize_shape.value.tolist()
+            attrs["out_shape"] = resize_shape
         else:
-                size[i] = size[i] + begin[i]
+            shape = resize_shape.out_shapes[0]
+            reshape_name = gen_name("resize_nearest", "reshape")
+            program.add_layer(
+                kernel="fluid.layers.reshape",
+                inputs={"x": resize_shape.name},
+                outputs=[reshape_name],
+                shape=shape)
+            inputs["out_shape"] = reshape_name
+
+        if data_format == "NHWC":
+            transpose_name = gen_name("resize_nearest", "reshape")
+            program.add_layer(
+                kernel="fluid.layers.transpose",
+                inputs={"x": input.name},
+                outputs=[transpose_name],
+                perm=[0, 3, 1, 2])
+            inputs["input"] = transpose_name
+
+        program.add_layer(
+            kernel="fluid.layers.resize_nearest",
+            inputs=inputs,
+            outputs=[node.name],
+            **attrs)
+
+        if data_format == "NHWC":
+            program.add_layer(
+                kernel="fluid.layers.transpose",
+                inputs={"x": node.name},
+                outputs=[node.name],
+                perm=[0, 2, 3, 1])
 
-        attr = {
-            "axes": [i for i in range(len(size))],
-            "starts": begin,
-            "ends": size
-        }
-        node.fluid_code.add_layer(
-            "slice", inputs=input, output=node, param_attr=attr)
+    def ResizeBilinear(self, node):
+        input = self.graph.get_node(node.layer.input[0])
+        resize_shape = self.graph.get_node(node.layer.input[1])
+        data_format = "NHWC"
+        inputs = {"input": input.name}
+        attrs = {"align_corners": node.get_attr("align_corners")}
+
+        if resize_shape.layer_type == "Const":
+            resize_shape = resize_shape.value.tolist()
+            attrs["out_shape"] = resize_shape
+        else:
+            shape = resize_shape.out_shapes[0]
+            reshape_name = gen_name("resize_bilinear", "reshape")
+            program.add_layer(
+                kernel="fluid.layers.reshape",
+                inputs={"x": resize_shape.name},
+                outputs=[reshape_name],
+                shape=shape)
+            inputs["out_shape"] = reshape_name
+
+        if data_format == "NHWC":
+            transpose_name = gen_name("resize_bilinear", "reshape")
+            program.add_layer(
+                kernel="fluid.layers.transpose",
+                inputs={"x": input.name},
+                outputs=[transpose_name],
+                perm=[0, 3, 1, 2])
+            inputs["input"] = transpose_name
+
+        program.add_layer(
+            kernel="fluid.layers.resize_bilinear",
+            inputs=inputs,
+            outputs=[node.name],
+            **attrs)
+
+        if data_format == "NHWC":
+            program.add_layer(
+                kernel="fluid.layers.transpose",
+                inputs={"x": node.name},
+                outputs=[node.name],
+                perm=[0, 2, 3, 1])
+
+    def Cast(self, node):
+        input = self.graph.get_node(node.layer.input[0])
+        dtype = node.dtype
+        program.add_layer(
+            kernel="fluid.layers.cast",
+            inputs={"x": input.name},
+            outputs=[node.name],
+            dtype=string(dtype))
+
+    def Sum(self, node):
+        input = self.graph.get_node(node.layer.input[0])
+        reduce_idx = self.graph.get_node(node.layer.input[1])
+        assert reduce_idx.layer_type == "Const", "Only support Const parameter[reduce_idx]"
+        keep_dims = node.get_attr("keep_dims")
+        dim = reduce_idx.value.tolist()
+
+        program.add_layer(
+            kernel="fluid.layers.reduce_sum",
+            inputs={"input": input.name},
+            outputs=[node.name],
+            dim=dim,
+            keep_dim=keep_dims)
+
+    def Max(self, node):
+        input = self.graph.get_node(node.layer.input[0])
+        reduce_idx = self.graph.get_node(node.layer.input[1])
+        assert reduce_idx.layer_type == "Const", "Only support Const parameter[reduce_idx]"
+        keep_dims = node.get_attr("keep_dims")
+        dim = reduce_idx.value.tolist()
+        program.add_layer(
+            kernel="fluid.layers.reduce_max",
+            inputs={"input": input.name},
+            outputs=[node.name],
+            dim=dim,
+            keep_dim=keep_dims)
+
+    def RandomUniform(self, node):
+        shape = self.graph.get_node(node.layer.input[0])
+        if shape.layer_type == "Const":
+            shape = shape.value.tolist()
+            program.add_layer(
+                kernel="fluid.layers.uniform_random",
+                inputs={},
+                outputs=[node.name],
+                shape=shape,
+                min=0.0,
+                max=0.9999)
+        else:
+            program.add_layer(
+                kernel="fluid.layers.uniform_random",
+                inputs={'shape': shape.name},
+                outputs=[node.name],
+                min=0.0,
+                max=0.9999)
 
     def Conv2DBackpropInput(self, node):
-        out_shape = self.graph.get_node(node.layer.input[0], copy=True)
-        kernel = self.graph.get_node(node.layer.input[1], copy=True)
-        input = self.graph.get_node(node.layer.input[2], copy=True)
+        out_shape = self.graph.get_node(node.layer.input[0])
+        kernel = self.graph.get_node(node.layer.input[1])
+        input = self.graph.get_node(node.layer.input[2])
 
         assert kernel.layer_type == "Const", "Kernel of Conv2DBackpropInput should be Const"
 
-        self.add_omit_nodes(kernel.layer_name, node.layer_name)
-        self.add_omit_nodes(out_shape.layer_name, node.layer_name)
-
         if out_shape.layer_type == "Const":
             out_shape = out_shape.value.tolist()
         else:
@@ -912,163 +1072,199 @@ class TFOpMapper(OpMapper):
         strides = node.get_attr("strides")
         dilations = node.get_attr("dilations")
         data_format = node.get_attr("data_format").decode()
-        channel_first = data_format == "NCHW"
 
-        self.weights[kernel.layer_name.replace('/', '_')] = numpy.transpose(
-            kernel.value, (3, 2, 0, 1))
-        if not channel_first:
+        program.parameters[kernel.layer_name.replace(
+            '/', '_')] = numpy.transpose(kernel.value, (3, 2, 0, 1))
+
+        input_name = input.name
+        if data_format == "NHWC":
             in_shape = [in_shape[i] for i in [0, 3, 1, 2]]
             strides = [strides[i] for i in [0, 3, 1, 2]]
             dilations = [dilations[i] for i in [0, 3, 1, 2]]
-        else:
-            self.data_format_propagation(node)
-
-        attr = {
-            "bias_attr": False,
-            "param_attr": string(kernel.layer_name),
-            "num_filters": k_size[2],
-            "filter_size": k_size[0:2],
-            "stride": strides[2:4],
-            "dilation": dilations[2:4],
-            "padding": string(pad_mode),
-            "output_size": out_shape[1:3]
-        }
-        node.fluid_code.add_layer(
-            "conv2d_transpose", inputs=input, output=node, param_attr=attr)
-
-    def Max(self, node):
-        input = self.graph.get_node(node.layer.input[0], copy=True)
-        reduce_idx = self.graph.get_node(node.layer.input[1], copy=True)
-        assert reduce_idx.layer_type == "Const", "Only support Const parameter[reduce_idx]"
-        keep_dims = node.get_attr("keep_dims")
-        dim = reduce_idx.value.tolist()
-        if input.tf_data_format == "NHWC" and len(input.out_shapes[0]) == 4:
-            dim = nhwc_dim_to_nchw(input, dim)
-
-        attr = {"dim": dim, "keep_dim": keep_dims}
-        node.fluid_code.add_layer(
-            "reduce_max", inputs=input, output=node, param_attr=attr)
-
-    def Sum(self, node):
-        input = self.graph.get_node(node.layer.input[0], copy=True)
-        reduce_idx = self.graph.get_node(node.layer.input[1], copy=True)
-        assert reduce_idx.layer_type == "Const", "Only support Const parameter[reduce_idx]"
-        keep_dims = node.get_attr("keep_dims")
-        dim = reduce_idx.value.tolist()
-        if input.tf_data_format == "NHWC" and len(input.out_shapes[0]) == 4:
-            dim = nhwc_dim_to_nchw(input, dim)
+            transpose_name = gen_name("conv2dbackpropinput", "transpose")
+            program.add_layer(
+                kernel="fluid.layers.transpose",
+                inputs={"x": input.name},
+                outputs=[transpose_name],
+                perm=[0, 3, 1, 2])
+            input_name = transpose_name
+
+        program.add_layer(
+            kernel="fluid.layers.conv2d_transpose",
+            inputs={"input": input_name},
+            outputs=[node.name],
+            bias_attr=False,
+            param_attr=string(kernel.layer_name),
+            num_filters=k_size[2],
+            filter_size=k_size[0:2],
+            stride=strides[2:4],
+            dilation=dilations[2:4],
+            padding=string(pad_mode),
+            output_size=out_shape[1:3])
+
+        if data_format == "NHWC":
+            program.add_layer(
+                kernel="fluid.layers.transpose",
+                inputs={"x": node.name},
+                outputs=[node.name],
+                perm=[0, 2, 3, 1])
 
-        attr = {"dim": dim, "keep_dim": keep_dims}
-        node.fluid_code.add_layer(
-            "reduce_sum", inputs=input, output=node, param_attr=attr)
-
-    def Cast(self, node):
-        input = self.graph.get_node(node.layer.input[0], copy=True)
-        dtype = node.dtype_map[node.get_attr('DstT')]
-        attr = {"dtype": string(dtype)}
-        node.fluid_code.add_layer(
-            "cast", inputs=input, output=node, param_attr=attr)
-
-    def Split(self, node):
-        dim = self.graph.get_node(node.layer.input[0], copy=True)
-        input = self.graph.get_node(node.layer.input[1], copy=True)
-        self.add_omit_nodes(dim.layer_name, node.layer_name)
-        num_split = node.get_attr('num_split')
-        dim = dim.value
-        if input.tf_data_format == "NHWC" and len(input.out_shapes[0]) == 4:
-            dim = nhwc_dim_to_nchw(input, dim)
-
-        attr = {"num_or_sections": num_split, "dim": dim}
-        node.fluid_code.add_layer(
-            "split", inputs=input, output=node, param_attr=attr)
+    def Tile(self, node):
+        input = self.graph.get_node(node.layer.input[0])
+        expand_times = self.graph.get_node(node.layer.input[1])
+        inputs = {"x": input.name}
+        attr = dict()
+        if expand_times.layer_type == "Const":
+            expand_times = expand_times.value.tolist()
+            attr["expand_times"] = expand_times
+        else:
+            inputs["expand_times"] = expand_times.name
 
-    def Squeeze(self, node):
-        input = self.graph.get_node(node.layer.input[0], copy=True)
-        squeeze_dims = node.get_attr('squeeze_dims')
-        if input.tf_data_format == "NHWC" and len(input.out_shapes[0]) == 4:
-            for i in range(len(squeeze_dims)):
-                squeeze_dims[i] = nhwc_dim_to_nchw(input, squeeze_dims[i])
-        attr = {"axes": squeeze_dims}
-        node.fluid_code.add_layer(
-            "squeeze", inputs=input, output=node, param_attr=attr)
+        program.add_layer(
+            kernel="fluid.layers.expand",
+            inputs=inputs,
+            outputs=[node.name],
+            **attr)
 
-    def Softmax(self, node):
-        input = self.graph.get_node(node.layer.input[0], copy=True)
-        axis = node.get_attr("axis")
-        if axis is None:
-            axis = -1 + len(input.out_shapes[0])
-        if input.tf_data_format == "NHWC" and len(input.out_shapes[0]) == 4:
-            axis = nhwc_dim_to_nchw(input, axis)
-        attr = {"axis": axis}
-        node.fluid_code.add_layer(
-            "softmax", inputs=input, output=node, param_attr=attr)
+    def Range(self, node):
+        start = self.graph.get_node(node.layer.input[0])
+        limit = self.graph.get_node(node.layer.input[1])
+        delta = self.graph.get_node(node.layer.input[2])
+        inputs = dict()
+        attr = dict()
 
-    def ResizeNearestNeighbor(self, node):
-        input = self.graph.get_node(node.layer.input[0], copy=True)
-        resize_shape = self.graph.get_node(node.layer.input[1], copy=True)
-        self.add_omit_nodes(resize_shape.layer_name, node.layer_name)
-        if resize_shape.layer_type == "Const":
-            resize_shape = resize_shape.value.tolist()
+        if start.layer_type == "Const":
+            attr["start"] = start.value
         else:
-            resize_shape = self.decoder.infer_shape_tensor(resize_shape,
-                                                           node.out_shapes[0])
-        align_corners = node.get_attr("align_corners")
-        attr = {"align_corners": align_corners, "out_shape": resize_shape}
-        node.fluid_code.add_layer(
-            "resize_nearest", inputs=input, output=node, param_attr=attr)
-
-    def ResizeBilinear(self, node):
-        input = self.graph.get_node(node.layer.input[0], copy=True)
-        resize_shape = self.graph.get_node(node.layer.input[1], copy=True)
-        self.add_omit_nodes(resize_shape.layer_name, node.layer_name)
-        if resize_shape.layer_type == "Const":
-            resize_shape = resize_shape.value.tolist()
+            inputs["start"] = start.name
+        if limit.layer_type == "Const":
+            attr["end"] = limit.value
         else:
-            resize_shape = self.decoder.infer_shape_tensor(resize_shape,
-                                                           node.out_shapes[0])
-        align_corners = node.get_attr("align_corners")
-        attr = {
-            "align_corners": align_corners,
-            "out_shape": resize_shape,
-            "align_mode": 1
-        }
-        node.fluid_code.add_layer(
-            "resize_bilinear", inputs=input, output=node, param_attr=attr)
+            inputs["end"] = limit.name
+        if delta.layer_type == "Const":
+            attr["step"] = delta.value
+        else:
+            inputs["step"] = delta.name
+        attr["dtype"] = string(node.dtype)
 
-    def GreaterEqual(self, node):
-        x = self.graph.get_node(node.layer.input[0], copy=True)
-        y = self.graph.get_node(node.layer.input[1], copy=True)
-        inputs = {"x": x, "y": y}
-        node.fluid_code.add_layer(
-            "greater_equal", inputs=inputs, output=node, param_attr=None)
+        program.add_layer(
+            kernel="fluid.layers.range",
+            inputs=inputs,
+            outputs=[node.name],
+            **attr)
 
-    def RandomUniform(self, node):
-        shape = self.graph.get_node(node.layer.input[0], copy=True)
-        self.add_omit_nodes(shape.layer_name, node.layer_name)
-        if shape.layer_type == "Const":
-            shape = shape.value.tolist()
-        else:
-            shape = self.decoder.infer_shape_tensor(shape)
-        if len(shape) == 4 and node.tf_data_format == "NHWC":
-            shape = [shape[i] for i in [0, 3, 1, 2]]
-        attr = {"shape": shape, "min": 0.0, "max": 0.9999}
-        if shape[0] < 0:
-            input = self.batch_node
-            node.fluid_code.add_layer(
-                "uniform_random_batch_size_like",
-                inputs=input,
-                output=node,
-                param_attr=attr)
+    def SquaredDifference(self, node):
+        x = self.graph.get_node(node.layer.input[0])
+        y = self.graph.get_node(node.layer.input[1])
+        inputs = {"x": x.name, "y": y.name}
+        x_shape = x.out_shapes[0]
+        y_shape = y.out_shapes[0]
+        layer_id = program.add_layer(
+            "fluid.layers.elementwise_sub", inputs=inputs, outputs=[node.name])
+        program.layers[layer_id].input_shapes = {"x": x_shape, "y": y_shape}
+
+        inputs = {"x": node.name, "y": node.name}
+        x_shape = node.out_shapes[0]
+        y_shape = node.out_shapes[0]
+        layer_id = program.add_layer(
+            "fluid.layers.elementwise_mul", inputs=inputs, outputs=[node.name])
+        program.layers[layer_id].input_shapes = {"x": x_shape, "y": y_shape}
+
+    def OneHot(self, node):
+        input = self.graph.get_node(node.layer.input[0])
+        depth = self.graph.get_node(node.layer.input[1])
+        on_value = self.graph.get_node(node.layer.input[2])
+        off_value = self.graph.get_node(node.layer.input[3])
+        assert depth.layer_type == 'Const', 'Parameter depth should be Const in OneHot'
+        assert on_value.layer_type == 'Const', 'Parameter on_value should be Const in OneHot'
+        assert off_value.layer_type == 'Const', 'Parameter off_value should be Const in OneHot'
+
+        attr = {'depth': depth.value}
+        on_value = on_value.value
+        off_value = off_value.value
+        assert math.fabs(on_value -
+                         1.0) < 1e-06, "on_value should be 1 in OneHot"
+        assert math.fabs(off_value -
+                         0.0) < 1e-06, "off_value should be 0 in OneHot"
+
+        program.add_layer(
+            "fluid.one_hot",
+            inputs={"input": input.name},
+            outputs=[node.name],
+            depth=depth.value)
+
+    def Pow(self, node):
+        x = self.graph.get_node(node.layer.input[0])
+        factor = self.graph.get_node(node.layer.input[1])
+        inputs = {"x": x.name}
+        attr = dict()
+        if factor.layer_type == 'Const':
+            attr["factor"] = factor.value.tolist()
         else:
-            node.fluid_code.add_layer(
-                "uniform_random", inputs=None, output=node, param_attr=attr)
+            inputs["factor"] = factor.name
+        program.add_layer(
+            "fluid.layers.pow", inputs=inputs, outputs=[node.name], **attr)
 
-    def SquaredDifference(self, node):
+    def All(self, node):
+        input = self.graph.get_node(node.layer.input[0])
+        reduce_idx = self.graph.get_node(node.layer.input[1])
+        assert reduce_idx.layer_type == "Const", "Only support Const parameter[reduce_idx]"
+        attr = dict()
+        attr["dim"] = reduce_idx.value.tolist()
+        attr["keep_dim"] = node.get_attr("keep_dims")
+
+        program.add_layer(
+            "fluid.layers.reduce_all",
+            inputs={"input": input.name},
+            outputs=[node.name],
+            **attr)
+
+        node.layer.attr['dtype'].type = 10
+
+    def GatherV2(self, node):
+        embeddings = self.graph.get_node(node.layer.input[0])
+        index = self.graph.get_node(node.layer.input[1])
+        axis = self.graph.get_node(node.layer.input[2])
+        assert axis.layer_type == 'Const', "Only support Const parameter[axis]"
+        axis = axis.value.tolist()
+        assert axis == 0, "Only support axis=0 in GatherV2 OP"
+        index_name = index.name
+        if len(index.out_shapes[0]) != 1:
+            reshape_name = gen_name("gather", "reshape")
+            index_name = reshape_name
+            program.add_layer(
+                "fluid.layers.reshape",
+                inputs={"x": index.name},
+                outputs=[reshape_name],
+                shape=[-1])
+        inputs = {'input': embeddings.name, 'index': index_name}
+        program.add_layer(
+            "fluid.layers.gather",
+            inputs=inputs,
+            outputs=[node.name],
+            overwrite=False)
+        if len(index.out_shapes[0]) != 1:
+            out_shape = node.out_shapes[0]
+            program.add_layer(
+                kernel="fluid.layers.reshape",
+                inputs={"x": node.name},
+                outputs=[node.name],
+                shape=out_shape)
+
+    def ExpandDims(self, node):
         x = self.graph.get_node(node.layer.input[0], copy=True)
         y = self.graph.get_node(node.layer.input[1], copy=True)
-        inputs = {"x": x, "y": y}
-        node.fluid_code.add_layer(
-            "elementwise_sub", inputs=inputs, output=node, param_attr=None)
-        inputs = {"x": node, "y": node}
-        node.fluid_code.add_layer(
-            "elementwise_mul", inputs=inputs, output=node, param_attr=None)
+        inputs = {"input": x.name}
+        attr = dict()
+        if y.layer_type == 'Const':
+            dim = y.value.tolist()
+            if not isinstance(dim, list):
+                dim = [dim]
+            attr['axes'] = dim
+        else:
+            inputs['axes'] = y.name
+        program.add_layer(
+            "fluid.layers.unsqueeze",
+            inputs=inputs,
+            outputs=[node.name],
+            **attr)

x2paddle/op_mapper/tf_op_mapper_nhwc.py

-#   Copyright (c) 2019  PaddlePaddle Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License"
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from x2paddle.decoder.tf_decoder import TFGraph
-from x2paddle.core.op_mapper import OpMapper
-from x2paddle.core.util import *
-import math
-import inspect
-import numpy
-import sys
-
-
-# compute padding size for SAME mode
-def get_same_padding(in_size, kernel_size, stride):
-    new_size = int(math.ceil(in_size * 1.0 / stride))
-    pad_size = (new_size - 1) * stride + kernel_size - in_size
-    if pad_size < 0:
-        pad_size = 0
-    pad0 = int(pad_size / 2)
-    pad1 = pad_size - pad0
-    return [pad0, pad1]
-
-
-class TFOpMapperNHWC(OpMapper):
-    directly_map_ops = {
-        'Relu': ['relu'],
-        'Relu6': ['relu6'],
-        'Shape': ['shape'],
-        'Abs': ['abs'],
-        'Sigmoid': ['sigmoid'],
-        'Exp': ['exp'],
-        'Rsqrt': ['rsqrt'],
-        'Sqrt': ['sqrt'],
-        'swish_f32': ['swish'],
-        'Tanh': ['tanh'],
-        'Softplus': ['softplus'],
-        'LeakyRelu': ['leaky_relu', {
-            'alpha': 'alpha'
-        }],
-        'Floor': ['floor'],
-        'Erf': ['erf']
-    }
-    elementwise_ops = {
-        'Add': 'elementwise_add',
-        'AddV2': 'elementwise_add',
-        'RealDiv': 'elementwise_div',
-        'Sub': 'elementwise_sub',
-        'Maximum': 'elementwise_max',
-        'Minimum': 'elementwise_min',
-        'LessEqual': 'less_equal',
-        'Mul': 'elementwise_mul',
-        'FloorDiv': 'elementwise_floordiv'
-    }
-
-    def __init__(self, decoder):
-        super(TFOpMapperNHWC, self).__init__()
-        self.decoder = decoder
-        self.graph = decoder.tf_graph
-        self.weights = dict()
-        self.batch_node = None
-        self.omit_nodes = list()
-        self.used_custom_layers = dict()
-
-        not_placeholder = list()
-        for name in self.graph.input_nodes:
-            if self.graph.get_node(
-                    name).layer_type != "Placeholder" and self.graph.get_node(
-                        name
-                    ).layer_type != "OneShotIterator" and self.graph.get_node(
-                        name).layer_type != "IteratorV2":
-                not_placeholder.append(name)
-        for name in not_placeholder:
-            idx = self.graph.input_nodes.index(name)
-            del self.graph.input_nodes[idx]
-
-        unsupported_ops = set()
-        sys.stderr.write("Total nodes: {}\n".format(len(self.graph.topo_sort)))
-        for i, node_name in enumerate(self.graph.topo_sort):
-            sys.stderr.write("\rConverting node {} ...     ".format(i + 1))
-            node = self.graph.get_node(node_name)
-            op = node.layer_type
-            if op in self.directly_map_ops:
-                if len(unsupported_ops) > 0:
-                    continue
-                self.directly_map(node)
-            elif op in self.elementwise_ops:
-                if len(unsupported_ops) > 0:
-                    continue
-                self.elementwise_map(node)
-            elif hasattr(self, op):
-                if len(unsupported_ops) > 0:
-                    continue
-                func = getattr(self, op)
-                try:
-                    func(node)
-                except Exception as e:
-                    unsupported_ops.add(op)
-                    print(e)
-            else:
-                unsupported_ops.add(op)
-        if len(unsupported_ops) > 0:
-            print("========= {} OPs are not supported yet ===========".format(
-                len(unsupported_ops)))
-            for op in unsupported_ops:
-                print("========== {} ============".format(op))
-            sys.exit(-1)
-        sys.stderr.write("\nDone!\n")
-
-    def add_omit_nodes(self, in_node_name, out_node_name):
-        in_node = self.graph.get_node(in_node_name)
-        out_node = self.graph.get_node(out_node_name)
-        index = in_node.outputs.index(out_node_name)
-        del in_node.outputs[index]
-        index = out_node.inputs.index(in_node_name)
-        del out_node.inputs[index]
-        self.omit_nodes.append(in_node.layer_name)
-
-    def directly_map(self, node):
-        assert node.layer_type in self.directly_map_ops
-        op_info = self.directly_map_ops[node.layer_type]
-        input = self.graph.get_node(node.layer.input[0], copy=True)
-        attr = dict()
-        for param in op_info[1:]:
-            tf_param_name = list(param.keys())[0]
-            pd_param_name = list(param.values())[0]
-            tf_param = node.get_attr(tf_param_name)
-            attr[pd_param_name] = tf_param
-
-        if len(input.out_shapes[0]) == 4 and op_info[0] != 'shape':
-            attr1 = {"perm": [0, 3, 1, 2]}
-            node.fluid_code.add_layer(
-                'transpose', inputs=input, output=node, param_attr=attr1)
-            input = node
-            node.fluid_code.add_layer(
-                op_info[0], inputs=input, output=node, param_attr=attr)
-            input = node
-            attr2 = {"perm": [0, 2, 3, 1]}
-            node.fluid_code.add_layer(
-                'transpose', inputs=input, output=node, param_attr=attr2)
-        else:
-            node.fluid_code.add_layer(
-                op_info[0], inputs=input, output=node, param_attr=attr)
-
-    def elementwise_map(self, node):
-        assert node.layer_type in self.elementwise_ops
-        op_type = self.elementwise_ops[node.layer_type]
-        x = self.graph.get_node(node.layer.input[0], copy=True)
-        y = self.graph.get_node(node.layer.input[1], copy=True)
-        inputs = {"x": x, "y": y}
-        node.fluid_code.add_layer(
-            op_type, inputs=inputs, output=node, param_attr=None)
-
-    def Placeholder(self, node):
-        shape = node.out_shapes[0]
-        assert len(shape) != 0, "Unknown shape of input nodes[{}].".format(
-            node.layer_name)
-        dtype = node.dtype
-        if shape[0] < 0:
-            self.batch_node = node
-        attr = {
-            'dtype': string(dtype),
-            'shape': shape,
-            'name': string(node.layer_name),
-            'append_batch_size': False
-        }
-
-        node.fluid_code.add_layer(
-            "data", inputs=None, output=node, param_attr=attr)
-
-    def Const(self, node):
-        shape = node.out_shapes[0]
-        dtype = node.dtype
-        value = node.value
-        initializer = "Constant(0.0)"
-        if len(shape) == 0:
-            assert value.size == 1, "Unexpected situation happend"
-            shape = [1]
-            initializer = "Constant({})".format(value)
-
-        self.weights[node.layer_name] = node.value
-
-        attr = {
-            'dtype': string(dtype),
-            'shape': shape,
-            'name': string(node.layer_name),
-            'default_initializer': initializer
-        }
-        node.fluid_code.add_layer(
-            "create_parameter", inputs=None, output=node, param_attr=attr)
-
-    def Transpose(self, node):
-        input = self.graph.get_node(node.layer.input[0], copy=True)
-        perm = self.graph.get_node(node.layer.input[1], copy=True)
-        assert perm.layer_type == "Const", "Perm of transpose OP should be Const"
-        del self.weights[perm.layer_name.replace('/', '_')]
-        perm.fluid_code.clear()
-        perm = perm.value.tolist()
-
-        attr = {'perm': perm}
-        node.fluid_code.add_layer(
-            "transpose", inputs=input, output=node, param_attr=attr)
-
-    def Fill(self, node):
-        dims = self.graph.get_node(node.layer.input[0], copy=True)
-        input_value = self.graph.get_node(node.layer.input[1], copy=True)
-
-        assert input_value.layer_type == "Const", "Value of fill OP should be Const"
-
-        self.add_omit_nodes(input_value.layer_name, node.layer_name)
-        input_value = input_value.value
-        input_dtype = string(input_value.dtype)
-        attr = {'value': input_value, 'dtype': input_dtype}
-
-        node.fluid_code.add_layer(
-            "fill_constant", inputs=dims, output=node, param_attr=attr)
-
-    def DepthToSpace(self, node):
-        input = self.graph.get_node(node.layer.input[0], copy=True)
-
-        block_size = node.get_attr("block_size")
-        data_format = node.get_attr("data_format").decode()
-
-        if data_format == "NHWC":
-            attr = {"perm": [0, 3, 1, 2]}
-            node.fluid_code.add_layer(
-                "transpose", inputs=input, output=input, param_attr=attr)
-        n, h, w, c = input.out_shapes[0]
-
-        attr = {'shape': [0, block_size * block_size, -1, h, w]}
-        node.fluid_code.add_layer(
-            "reshape", inputs=input, output=input, param_attr=attr)
-
-        attr = {'perm': [0, 2, 1, 3, 4]}
-        node.fluid_code.add_layer(
-            "transpose", inputs=input, output=input, param_attr=attr)
-        attr = {'shape': [0, c, h, w]}
-        node.fluid_code.add_layer(
-            "reshape", inputs=input, output=input, param_attr=attr)
-
-        attr = {'upscale_factor': block_size}
-        node.fluid_code.add_layer(
-            "pixel_shuffle", inputs=input, output=node, param_attr=attr)
-
-        if data_format == "NHWC":
-            attr = {"perm": [0, 2, 3, 1]}
-            node.fluid_code.add_layer(
-                "transpose", inputs=node, output=node, param_attr=attr)
-
-    def MaxPool(self, node):
-        input = self.graph.get_node(node.layer.input[0], copy=True)
-
-        k_size = node.get_attr("ksize")
-        strides = node.get_attr("strides")
-        data_format = node.get_attr("data_format").decode()
-        pad_mode = node.get_attr("padding").decode()
-        channel_first = data_format == "NCHW"
-
-        if not channel_first:
-            attr = {"perm": [0, 3, 1, 2]}
-            node.fluid_code.add_layer(
-                "transpose", inputs=input, output=node, param_attr=attr)
-            strides = [strides[i] for i in [0, 3, 1, 2]]
-            k_size = [k_size[i] for i in [0, 3, 1, 2]]
-            input = node
-
-        attr = {
-            "pool_size": k_size[2:4],
-            "pool_type": string("max"),
-            "pool_stride": strides[2:4],
-            "pool_padding": string(pad_mode)
-        }
-        node.fluid_code.add_layer(
-            "pool2d", inputs=input, output=node, param_attr=attr)
-
-        if not channel_first:
-            attr = {"perm": [0, 2, 3, 1]}
-            node.fluid_code.add_layer(
-                "transpose", inputs=node, output=node, param_attr=attr)
-
-    def Conv2D(self, node):
-        input = self.graph.get_node(node.layer.input[0], copy=True)
-        kernel = self.graph.get_node(node.layer.input[1], copy=True)
-        self.add_omit_nodes(kernel.layer_name, node.layer_name)
-
-        k_size = kernel.out_shapes[0]
-        strides = node.get_attr("strides")
-        dilations = node.get_attr("dilations")
-        data_format = node.get_attr("data_format").decode()
-        pad_mode = node.get_attr("padding").decode()
-        channel_first = data_format == "NCHW"
-        if data_format == "NHWC":
-            n, h, w, c = input.out_shapes[0]
-        else:
-            n, c, h, w = input.out_shapes[0]
-
-        if kernel.layer_type == 'Const':
-            kernel_value = kernel.value
-            kernel_weight_name = kernel.layer_name.replace('/', '_')
-        else:
-            kernel_value = self.decoder.infer_tensor(kernel)
-            if kernel.layer_type == 'Split':
-                kernel_weight_name = "{}_{}_kernel".format(node.layer_name,
-                                                           kernel.layer_name)
-            else:
-                kernel_weight_name = kernel.layer_name.replace('/', '_')
-        self.weights[kernel_weight_name] = numpy.transpose(kernel_value,
-                                                           (3, 2, 0, 1))
-
-        if not channel_first:
-            strides = [strides[i] for i in [0, 3, 1, 2]]
-            dilations = [dilations[i] for i in [0, 3, 1, 2]]
-            attr = {"perm": [0, 3, 1, 2]}
-            node.fluid_code.add_layer(
-                "transpose", inputs=input, output=node, param_attr=attr)
-            input = node
-        attr = {
-            "bias_attr": False,
-            "param_attr": string(kernel_weight_name),
-            "num_filters": k_size[3],
-            "filter_size": k_size[0:2],
-            "stride": strides[2:4],
-            "dilation": dilations[2:4],
-            "padding": string(pad_mode)
-        }
-        if hasattr(node, 'dilation') and attr['dilation'] == [1, 1]:
-            if len(node.dilation) == 1:
-                attr['dilation'] = [1, node.dilation[0]]
-
-        if c == -1:
-            reshape_attr = {"shape": [0, k_size[2], 0, 0]}
-            node.fluid_code.add_layer(
-                "reshape", inputs=input, output=input, param_attr=reshape_attr)
-
-        node.fluid_code.add_layer(
-            "conv2d", inputs=input, output=node, param_attr=attr)
-        if not channel_first:
-            attr = {"perm": [0, 2, 3, 1]}
-            node.fluid_code.add_layer(
-                "transpose", inputs=node, output=node, param_attr=attr)
-
-    def BiasAdd(self, node):
-        input = self.graph.get_node(node.layer.input[0], copy=True)
-        bias = self.graph.get_node(node.layer.input[1], copy=True)
-        inputs = {"x": input, "y": bias}
-        node.fluid_code.add_layer(
-            "elementwise_add", inputs=inputs, output=node, param_attr=None)
-
-    def FusedBatchNorm(self, node):
-        input = self.graph.get_node(node.layer.input[0], copy=True)
-        gamma = self.graph.get_node(node.layer.input[1], copy=True)
-        beta = self.graph.get_node(node.layer.input[2], copy=True)
-        moving_mean = self.graph.get_node(node.layer.input[3], copy=True)
-        moving_var = self.graph.get_node(node.layer.input[4], copy=True)
-        data_format = node.get_attr("data_format").decode()
-        channel_first = data_format == "NCHW"
-
-        assert gamma.layer_type == "Const"
-        assert beta.layer_type == "Const"
-        assert moving_mean.layer_type == "Const"
-        assert moving_var.layer_type == "Const"
-        self.add_omit_nodes(gamma.layer_name, node.layer_name)
-        self.add_omit_nodes(beta.layer_name, node.layer_name)
-        self.add_omit_nodes(moving_mean.layer_name, node.layer_name)
-        self.add_omit_nodes(moving_var.layer_name, node.layer_name)
-
-        if not channel_first:
-            attr = {"perm": [0, 3, 1, 2]}
-            node.fluid_code.add_layer(
-                "transpose", inputs=input, output=node, param_attr=attr)
-            input = node
-
-        attr = {
-            "epsilon": node.get_attr("epsilon"),
-            "param_attr": string(gamma.layer_name),
-            "bias_attr": string(beta.layer_name),
-            "moving_mean_name": string(moving_mean.layer_name),
-            "moving_variance_name": string(moving_var.layer_name),
-            "is_test": True
-        }
-
-        node.fluid_code.add_layer(
-            "batch_norm", inputs=input, output=node, param_attr=attr)
-
-        if not channel_first:
-            attr = {"perm": [0, 2, 3, 1]}
-            node.fluid_code.add_layer(
-                "transpose", inputs=node, output=node, param_attr=attr)
-
-    def DepthwiseConv2dNative(self, node):
-        input = self.graph.get_node(node.layer.input[0], copy=True)
-        kernel = self.graph.get_node(node.layer.input[1], copy=True)
-        assert kernel.layer_type == "Const", "Kernel of DepthwiseConv2DNative should be Const"
-        self.add_omit_nodes(kernel.layer_name, node.layer_name)
-
-        in_shape = input.out_shapes[0]
-        k_size = kernel.out_shapes[0]
-        strides = node.get_attr("strides")
-        dilations = node.get_attr("dilations")
-        data_format = node.get_attr("data_format").decode()
-        pad_mode = node.get_attr("padding").decode()
-        channel_first = data_format == "NCHW"
-
-        self.weights[kernel.layer_name.replace('/', '_')] = numpy.transpose(
-            kernel.value, (2, 3, 0, 1))
-
-        if not channel_first:
-            in_shape = [in_shape[i] for i in [0, 3, 1, 2]]
-            strides = [strides[i] for i in [0, 3, 1, 2]]
-            dilations = [dilations[i] for i in [0, 3, 1, 2]]
-            attr = {"perm": [0, 3, 1, 2]}
-            node.fluid_code.add_layer(
-                "transpose", inputs=input, output=node, param_attr=attr)
-            input = node
-
-        attr = {
-            "bias_attr": False,
-            "param_attr": string(kernel.layer_name),
-            "num_filters": in_shape[1],
-            "filter_size": k_size[0:2],
-            "stride": strides[2:4],
-            "dilation": dilations[2:4],
-            "groups": k_size[3] * in_shape[1],
-            "use_cudnn": False,
-            "padding": string(pad_mode)
-        }
-        node.fluid_code.add_layer(
-            "conv2d", inputs=input, output=node, param_attr=attr)
-
-        if not channel_first:
-            attr = {"perm": [0, 2, 3, 1]}
-            node.fluid_code.add_layer(
-                "transpose", inputs=node, output=node, param_attr=attr)
-
-    def Reshape(self, node):
-        input = self.graph.get_node(node.layer.input[0], copy=True)
-        param = self.graph.get_node(node.layer.input[1], copy=True)
-        if param.layer_type == "Const":
-            self.add_omit_nodes(param.layer_name, node.layer_name)
-            shape = param.value.tolist()
-        else:
-            shape = param
-        inputs = {"x": input, "shape": shape}
-        node.fluid_code.add_layer(
-            "reshape", inputs=inputs, output=node, param_attr=None)
-        if param.layer_type != "Const":
-            out_shape = numpy.array(node.out_shapes[0])
-            if (out_shape > 0).any():
-                out_shape[out_shape < 0] = 0
-                attr = {'shape': out_shape.tolist()}
-                node.fluid_code.add_layer(
-                    "reshape", inputs=node, output=node, param_attr=attr)
-
-    def AvgPool(self, node):
-        input = self.graph.get_node(node.layer.input[0], copy=True)
-
-        k_size = node.get_attr("ksize")
-        strides = node.get_attr("strides")
-        data_format = node.get_attr("data_format").decode()
-        pad_mode = node.get_attr("padding").decode()
-        channel_first = data_format == "NCHW"
-
-        if not channel_first:
-            strides = [strides[i] for i in [0, 3, 1, 2]]
-            k_size = [k_size[i] for i in [0, 3, 1, 2]]
-            attr = {"perm": [0, 3, 1, 2]}
-            node.fluid_code.add_layer(
-                "transpose", inputs=input, output=node, param_attr=attr)
-            input = node
-
-        attr = {
-            "pool_size": k_size[2:4],
-            "pool_type": string("avg"),
-            "pool_stride": strides[2:4],
-            "pool_padding": string(pad_mode)
-        }
-        node.fluid_code.add_layer(
-            "pool2d", inputs=input, output=node, param_attr=attr)
-
-        if not channel_first:
-            attr = {"perm": [0, 2, 3, 1]}
-            node.fluid_code.add_layer(
-                "transpose", inputs=node, output=node, param_attr=attr)
-
-    def SplitV(self, node):
-        input = self.graph.get_node(node.layer.input[0], copy=True)
-        num_sections = self.graph.get_node(node.layer.input[1], copy=True)
-        dim = self.graph.get_node(node.layer.input[2], copy=True)
-        assert num_sections.layer_type == "Const"
-        assert dim.layer_type == "Const"
-        self.add_omit_nodes(num_sections.layer_name, node.layer_name)
-        self.add_omit_nodes(dim.layer_name, node.layer_name)
-        dim = dim.value
-        attr = {
-            "num_or_sections": num_sections.value.tolist(),
-            "dim": dim.value
-        }
-        node.fluid_code.add_layer(
-            "split", inputs=input, output=node, param_attr=attr)
-
-    def ConcatV2(self, node):
-        inputs = [
-            self.graph.get_node(
-                name, copy=True) for name in node.layer.input[:-1]
-        ]
-        axis = self.graph.get_node(node.layer.input[-1], copy=True)
-        assert axis.layer_type == "Const"
-        self.add_omit_nodes(axis.layer_name, node.layer_name)
-        axis = axis.value
-        if axis < 0:
-            axis += len(inputs[0].out_shapes[0])
-        attr = {"axis": axis}
-        node.fluid_code.add_layer(
-            "concat", inputs=inputs, output=node, param_attr=attr)
-
-    def Tile(self, node):
-        input = self.graph.get_node(node.layer.input[0], copy=True)
-        expand_times = self.graph.get_node(node.layer.input[1], copy=True)
-        if expand_times.layer_type == "Const":
-            self.add_omit_nodes(expand_times.layer_name, node.layer_name)
-            expand_times = expand_times.value.tolist()
-        else:
-            expand_times = expand_times
-        inputs = {"x": input, "expand_times": expand_times}
-        node.fluid_code.add_layer(
-            "expand", inputs=inputs, output=node, param_attr=None)
-
-    def Pack(self, node):
-        inputs = [
-            self.graph.get_node(
-                name, copy=True) for name in node.layer.input
-        ]
-        reshape_shape = list()
-        for input_node in inputs:
-            k_size = input_node.out_shapes[0]
-            if len(k_size) and k_size[-1] != -1:
-                reshape_shape = [0] * len(k_size)
-                reshape_shape[-1] = k_size[-1]
-                break
-        if len(reshape_shape):
-            for i, input_node in enumerate(inputs):
-                node.fluid_code.add_layer(
-                    "reshape",
-                    inputs=input_node,
-                    output='tmp_{}'.format(i),
-                    param_attr={"shape": reshape_shape})
-        axis = node.get_attr("axis")
-        attr = {"axis": axis}
-        if len(reshape_shape):
-            inputs = ['tmp_{}'.format(i) for i in range(len(inputs))]
-        node.fluid_code.add_layer(
-            "stack", inputs=inputs, output=node, param_attr=attr)
-
-    def Pad(self, node):
-        input = self.graph.get_node(node.layer.input[0], copy=True)
-        paddings = self.graph.get_node(node.layer.input[1], copy=True)
-        assert paddings.layer_type == "Const", "Padding should be Const"
-        self.add_omit_nodes(paddings.layer_name, node.layer_name)
-        paddings = paddings.value.flatten().tolist()
-        data_format = input.tf_data_format
-
-        if len(input.out_shapes[0]) == 4:
-            new_padding = None
-            if input.tf_data_format == "NHWC":
-                if paddings[0] + paddings[1] + paddings[6] + paddings[7] == 0:
-                    new_padding = paddings[2:6]
-            else:
-                if paddings[0] + paddings[1] + paddings[2] + paddings[3] == 0:
-                    new_padding = paddings[4:]
-            if new_padding is not None:
-                if input.tf_data_format == "NHWC":
-                    attr = {"perm": [0, 3, 1, 2]}
-                    node.fluid_code.add_layer(
-                        "transpose", inputs=input, output=node, param_attr=attr)
-                    input = node
-                attr = {"paddings": new_padding}
-                node.fluid_code.add_layer(
-                    "pad2d", inputs=input, output=node, param_attr=attr)
-                if input.tf_data_format == "NHWC":
-                    attr = {"perm": [0, 2, 3, 1]}
-                    node.fluid_code.add_layer(
-                        "transpose", inputs=node, output=node, param_attr=attr)
-
-                return
-
-        attr = {"paddings": paddings}
-        node.fluid_code.add_layer(
-            "pad", inputs=input, output=node, param_attr=attr)
-
-    def Range(self, node):
-        start = self.graph.get_node(node.layer.input[0], copy=True)
-        limit = self.graph.get_node(node.layer.input[1], copy=True)
-        delta = self.graph.get_node(node.layer.input[2], copy=True)
-
-        if start.layer_type == "Const":
-            self.add_omit_nodes(start.layer_name, node.layer_name)
-            start = start.value
-        if limit.layer_type == "Const":
-            self.add_omit_nodes(limit.layer_name, node.layer_name)
-            limit = limit.value
-        if delta.layer_type == "Const":
-            self.add_omit_nodes(delta.layer_name, node.layer_name)
-            delta = delta.value
-
-        dtype = node.dtype
-        inputs = {
-            "start": start,
-            "end": limit,
-            "step": delta,
-        }
-        attr = {"dtype": string(node.dtype)}
-        node.fluid_code.add_layer(
-            "range", inputs=inputs, output=node, param_attr=attr)
-
-    def Mean(self, node):
-        input = self.graph.get_node(node.layer.input[0], copy=True)
-        reduce_idx = self.graph.get_node(node.layer.input[1], copy=True)
-        assert reduce_idx.layer_type == "Const", "Only support Const parameter[reduce_idx]"
-        dims = reduce_idx.value.tolist()
-        keep_dims = node.get_attr("keep_dims")
-
-        attr = {"dim": dims, "keep_dim": keep_dims}
-        node.fluid_code.add_layer(
-            "reduce_mean", inputs=input, output=node, param_attr=attr)
-
-    def MatMul(self, node):
-        x = self.graph.get_node(node.layer.input[0], copy=True)
-        y = self.graph.get_node(node.layer.input[1], copy=True)
-        transpose_a = node.get_attr('transpose_a')
-        transpose_b = node.get_attr('transpose_b')
-        inputs = {"x": x, "y": y}
-        # fix paddle shape infer problem
-        # should be removed after paddle 1.6
-        if x.out_shapes[0][-1] < 0 and y.out_shapes[0][0] > 0:
-            shape = x.out_shapes[0]
-            shape[-1] = y.out_shapes[0][0]
-            attr = {"shape": shape}
-            node.fluid_code.add_layer(
-                "reshape", inputs=x, output=x, param_attr=attr)
-        if transpose_a is None:
-            transpose_a = node.get_attr('adj_x')
-        if transpose_b is None:
-            transpose_b = node.get_attr('adj_y')
-        attr = {"transpose_x": transpose_a, "transpose_y": transpose_b}
-        node.fluid_code.add_layer(
-            "matmul", inputs=inputs, output=node, param_attr=attr)
-
-    def BatchMatMul(self, node):
-        return self.MatMul(node)
-
-    def BatchMatMulV2(self, node):
-        return self.MatMul(node)
-
-    def ArgMax(self, node):
-        input = self.graph.get_node(node.layer.input[0], copy=True)
-        axis = self.graph.get_node(node.layer.input[1], copy=True)
-        assert axis.layer_type == "Const", "ArgMax only support Const parameter"
-        self.add_omit_nodes(axis.layer_name, node.layer_name)
-        axis = axis.value
-        attr = {"axis": axis}
-        node.fluid_code.add_layer(
-            "argmax", inputs=input, output=node, param_attr=attr)
-
-    def StridedSlice(self, node):
-        input = self.graph.get_node(node.layer.input[0], copy=True)
-        begin = self.graph.get_node(node.layer.input[1], copy=True)
-        end = self.graph.get_node(node.layer.input[2], copy=True)
-        strides = self.graph.get_node(node.layer.input[3], copy=True)
-        assert begin.layer_type == "Const"
-        assert end.layer_type == "Const"
-        assert strides.layer_type == "Const"
-        self.add_omit_nodes(begin.layer_name, node.layer_name)
-        self.add_omit_nodes(end.layer_name, node.layer_name)
-        self.add_omit_nodes(strides.layer_name, node.layer_name)
-        strides = strides.value.tolist()
-        assert len(set(strides)) == 1 and strides[
-            0] == 1, "Only support strides be 1 in StridedSlice OP"
-
-        begin = begin.value.tolist()
-        end = end.value.tolist()
-
-        for i in range(len(end)):
-            if end[i] == 0:
-                end[i] = 999999
-
-        begin_mask = node.get_attr('begin_mask')
-        end_mask = node.get_attr('end_mask')
-        ellipsis_mask = node.get_attr('ellipsis_mask')
-        new_axis_mask = node.get_attr('new_axis_mask')
-        shrink_axis_mask = node.get_attr('shrink_axis_mask')
-
-        assert ellipsis_mask == 0, "(OP:{} Name:{})Only support ellipsis_mask be 0[now: {}] n StridedSlice OP".format(
-            node.layer_type, node.layer.name, ellipsis_mask)
-
-        # TODO codes without validation
-        # Use it carefully
-        new_begin = list()
-        new_end = list()
-        new_axes = list()
-        shrink_axes = list()
-        for i, item in enumerate(begin):
-            mask = (new_axis_mask >> i) & 1
-            if mask != 0:
-                new_axes.append(i)
-                continue
-
-            mask = (shrink_axis_mask >> i) & 1
-            if mask != 0:
-                shrink_axes.append(i)
-
-            mask = (begin_mask >> i) & 1
-            if mask != 0:
-                new_begin.append(0)
-            else:
-                new_begin.append(item)
-
-            mask = (end_mask >> i) & 1
-            if mask != 0:
-                new_end.append(999999)
-            else:
-                new_end.append(end[i])
-
-        attr = {
-            "axes": [i for i in range(len(new_begin))],
-            "starts": new_begin,
-            "ends": new_end
-        }
-        node.fluid_code.add_layer(
-            "slice", inputs=input, output=node, param_attr=attr)
-        if len(new_axes) > 0:
-            attr = {"axes": new_axes}
-            node.fluid_code.add_layer(
-                "unsqueeze", inputs=node, output=node, param_attr=attr)
-        if len(shrink_axes) > 0:
-            if len(input.out_shapes[0]) + len(new_axes) <= 1:
-                pass
-            else:
-                attr = {"axes": shrink_axes}
-                node.fluid_code.add_layer(
-                    "squeeze", inputs=node, output=node, param_attr=attr)
-
-    def Slice(self, node):
-        input = self.graph.get_node(node.layer.input[0], copy=True)
-        begin = self.graph.get_node(node.layer.input[1], copy=True)
-        size = self.graph.get_node(node.layer.input[2], copy=True)
-        if begin.layer_type == "Const":
-            self.add_omit_nodes(begin.layer_name, node.layer_name)
-            begin = begin.value.tolist()
-        else:
-            begin = self.decoder.infer_tensor(begin).tolist()
-
-#             shape = begin.out_shapes[0]
-#             attr = {"shape": shape}
-#             node.fluid_code.add_layer(
-#                 "reshape", inputs=begin, output=begin, param_attr=attr)
-        if size.layer_type == "Const":
-            self.add_omit_nodes(size.layer_name, node.layer_name)
-            size = size.value.tolist()
-        else:
-            size = size
-            shape = size.out_shapes[0]
-            attr = {"shape": shape}
-            node.fluid_code.add_layer(
-                "reshape", inputs=size, output=size, param_attr=attr)
-        inputs = {"x": input, "offsets": begin, "shape": size}
-        node.fluid_code.add_layer(
-            "crop_tensor", inputs=inputs, output=node, param_attr=None)
-
-    def Conv2DBackpropInput(self, node):
-        out_shape = self.graph.get_node(node.layer.input[0], copy=True)
-        kernel = self.graph.get_node(node.layer.input[1], copy=True)
-        input = self.graph.get_node(node.layer.input[2], copy=True)
-
-        assert kernel.layer_type == "Const", "Kernel of Conv2DBackpropInput should be Const"
-
-        self.add_omit_nodes(kernel.layer_name, node.layer_name)
-        self.add_omit_nodes(out_shape.layer_name, node.layer_name)
-
-        if out_shape.layer_type == "Const":
-            out_shape = out_shape.value.tolist()
-        else:
-            out_shape = self.decoder.infer_shape_tensor(out_shape,
-                                                        node.out_shapes[0])
-
-        in_shape = input.out_shapes[0]
-        if in_shape.count(-1) > 2:
-            in_shape = self.decoder.infer_tensor(input).shape
-        k_size = kernel.out_shapes[0]
-        if k_size.count(-1) > 2:
-            k_size = self.decoder.infer_tensor(kernel).shape
-
-        pad_mode = node.get_attr("padding").decode()
-        strides = node.get_attr("strides")
-        dilations = node.get_attr("dilations")
-        data_format = node.get_attr("data_format").decode()
-        channel_first = data_format == "NCHW"
-
-        self.weights[kernel.layer_name.replace('/', '_')] = numpy.transpose(
-            kernel.value, (3, 2, 0, 1))
-        if not channel_first:
-            in_shape = [in_shape[i] for i in [0, 3, 1, 2]]
-            strides = [strides[i] for i in [0, 3, 1, 2]]
-            dilations = [dilations[i] for i in [0, 3, 1, 2]]
-            attr = {"perm": [0, 3, 1, 2]}
-            node.fluid_code.add_layer(
-                "transpose", inputs=input, output=node, param_attr=attr)
-            input = node
-        else:
-            self.graph.data_format_propagation(node)
-
-        attr = {
-            "bias_attr": False,
-            "param_attr": string(kernel.layer_name),
-            "num_filters": k_size[2],
-            "filter_size": k_size[0:2],
-            "stride": strides[2:4],
-            "dilation": dilations[2:4],
-            "padding": string(pad_mode),
-            "output_size": out_shape[1:3]
-        }
-        node.fluid_code.add_layer(
-            "conv2d_transpose", inputs=input, output=node, param_attr=attr)
-
-        if not channel_first:
-            attr = {"perm": [0, 2, 3, 1]}
-            node.fluid_code.add_layer(
-                "transpose", inputs=node, output=node, param_attr=attr)
-
-    def Max(self, node):
-        input = self.graph.get_node(node.layer.input[0], copy=True)
-        reduce_idx = self.graph.get_node(node.layer.input[1], copy=True)
-        assert reduce_idx.layer_type == "Const", "Only support Const parameter[reduce_idx]"
-        keep_dims = node.get_attr("keep_dims")
-        dim = reduce_idx.value.tolist()
-
-        attr = {"dim": dim, "keep_dim": keep_dims}
-        node.fluid_code.add_layer(
-            "reduce_max", inputs=input, output=node, param_attr=attr)
-
-    def Sum(self, node):
-        input = self.graph.get_node(node.layer.input[0], copy=True)
-        reduce_idx = self.graph.get_node(node.layer.input[1], copy=True)
-        assert reduce_idx.layer_type == "Const", "Only support Const parameter[reduce_idx]"
-        keep_dims = node.get_attr("keep_dims")
-        dim = reduce_idx.value.tolist()
-
-        attr = {"dim": dim, "keep_dim": keep_dims}
-        node.fluid_code.add_layer(
-            "reduce_sum", inputs=input, output=node, param_attr=attr)
-
-    def Cast(self, node):
-        input = self.graph.get_node(node.layer.input[0], copy=True)
-        dtype = node.dtype_map[node.get_attr('DstT')]
-        attr = {"dtype": string(dtype)}
-        node.fluid_code.add_layer(
-            "cast", inputs=input, output=node, param_attr=attr)
-
-    def Split(self, node):
-        dim = self.graph.get_node(node.layer.input[0], copy=True)
-        input = self.graph.get_node(node.layer.input[1], copy=True)
-        assert dim.layer_type == "Const"
-        self.add_omit_nodes(dim.layer_name, node.layer_name)
-        num_split = node.get_attr('num_split')
-        dim = dim.value
-
-        attr = {"num_or_sections": num_split, "dim": dim}
-        node.fluid_code.add_layer(
-            "split", inputs=input, output=node, param_attr=attr)
-
-    def Squeeze(self, node):
-        input = self.graph.get_node(node.layer.input[0], copy=True)
-        squeeze_dims = node.get_attr('squeeze_dims')
-        attr = {"axes": squeeze_dims}
-        node.fluid_code.add_layer(
-            "squeeze", inputs=input, output=node, param_attr=attr)
-
-    def Softmax(self, node):
-        input = self.graph.get_node(node.layer.input[0], copy=True)
-        axis = node.get_attr("axis")
-        attr = {"axis": axis}
-        node.fluid_code.add_layer(
-            "softmax", inputs=input, output=node, param_attr=attr)
-
-    def ResizeNearestNeighbor(self, node):
-        input = self.graph.get_node(node.layer.input[0], copy=True)
-        resize_shape = self.graph.get_node(node.layer.input[1], copy=True)
-        if resize_shape.layer_type == "Const":
-            self.add_omit_nodes(resize_shape.layer_name, node.layer_name)
-            resize_shape = resize_shape.value.tolist()
-        else:
-            resize_shape = resize_shape
-            shape = resize_shape.out_shapes[0]
-            attr = {"shape": shape}
-            node.fluid_code.add_layer(
-                "reshape",
-                inputs=resize_shape,
-                output=resize_shape,
-                param_attr=attr)
-
-        align_corners = node.get_attr("align_corners")
-        attr = {"perm": [0, 3, 1, 2]}
-        node.fluid_code.add_layer(
-            "transpose", inputs=input, output=node, param_attr=attr)
-        inputs = {"input": node, "out_shape": resize_shape}
-        attr = {"align_corners": align_corners}
-        node.fluid_code.add_layer(
-            "resize_nearest", inputs=inputs, output=node, param_attr=attr)
-        attr = {"perm": [0, 2, 3, 1]}
-        node.fluid_code.add_layer(
-            "transpose", inputs=node, output=node, param_attr=attr)
-
-    def ResizeBilinear(self, node):
-        input = self.graph.get_node(node.layer.input[0], copy=True)
-        resize_shape = self.graph.get_node(node.layer.input[1], copy=True)
-        if resize_shape.layer_type == "Const":
-            self.add_omit_nodes(resize_shape.layer_name, node.layer_name)
-            resize_shape = resize_shape.value.tolist()
-        else:
-            shape = resize_shape.out_shapes[0]
-            attr = {"shape": shape}
-            node.fluid_code.add_layer(
-                "reshape",
-                inputs=resize_shape,
-                output=resize_shape,
-                param_attr=attr)
-        align_corners = node.get_attr("align_corners")
-        attr = {"perm": [0, 3, 1, 2]}
-        node.fluid_code.add_layer(
-            "transpose", inputs=input, output=node, param_attr=attr)
-        inputs = {"input": node, "out_shape": resize_shape}
-        attr = {
-            #"out_shape": resize_shape,
-            "align_corners": align_corners,
-            "align_mode": 1
-        }
-        node.fluid_code.add_layer(
-            "resize_bilinear", inputs=inputs, output=node, param_attr=attr)
-        attr = {"perm": [0, 2, 3, 1]}
-        node.fluid_code.add_layer(
-            "transpose", inputs=node, output=node, param_attr=attr)
-
-    def GreaterEqual(self, node):
-        x = self.graph.get_node(node.layer.input[0], copy=True)
-        y = self.graph.get_node(node.layer.input[1], copy=True)
-        inputs = {"x": x, "y": y}
-        node.fluid_code.add_layer(
-            "greater_equal", inputs=inputs, output=node, param_attr=None)
-
-    def RandomUniform(self, node):
-        shape = self.graph.get_node(node.layer.input[0], copy=True)
-        if shape.layer_type == "Const":
-            self.add_omit_nodes(shape.layer_name, node.layer_name)
-            shape = shape.value.tolist()
-        else:
-            shape = shape
-        attr = {"min": 0.0, "max": 0.9999}
-
-        node.fluid_code.add_layer(
-            "uniform_random", inputs=shape, output=node, param_attr=attr)
-
-    def SquaredDifference(self, node):
-        x = self.graph.get_node(node.layer.input[0], copy=True)
-        y = self.graph.get_node(node.layer.input[1], copy=True)
-        inputs = {"x": x, "y": y}
-        node.fluid_code.add_layer(
-            "elementwise_sub", inputs=inputs, output=node, param_attr=None)
-        inputs = {"x": node, "y": node}
-        node.fluid_code.add_layer(
-            "elementwise_mul", inputs=inputs, output=node, param_attr=None)
-
-    def ExpandDims(self, node):
-        x = self.graph.get_node(node.layer.input[0], copy=True)
-        y = self.graph.get_node(node.layer.input[1], copy=True)
-        if y.layer_type == 'Const':
-            self.add_omit_nodes(y.layer_name, node.layer_name)
-            dim = y.value.tolist()
-            if not isinstance(dim, list):
-                dim = [dim]
-            attr = {'axes': dim}
-        else:
-            attr = {'axes': y}
-        node.fluid_code.add_layer(
-            "unsqueeze", inputs=x, output=node, param_attr=attr)
-
-    def BatchToSpaceND(self, node):
-        x = self.graph.get_node(node.layer.input[0], copy=True)
-        y = self.graph.get_node(node.layer.input[1], copy=True)
-        if hasattr(node, 'skip') and node.skip:
-            node.fluid_code.add_layer(
-                "=", inputs=x, output=node, param_attr=None)
-        else:
-            raise Exception("BatchToSpaceND is not supported")
-
-    def SpaceToBatchND(self, node):
-        x = self.graph.get_node(node.layer.input[0], copy=True)
-        y = self.graph.get_node(node.layer.input[1], copy=True)
-        if hasattr(node, 'skip') and node.skip:
-            node.fluid_code.add_layer(
-                "=", inputs=x, output=node, param_attr=None)
-        else:
-            raise Exception("SpaceToBatchND is not supported")
-
-    def OneHot(self, node):
-        input = self.graph.get_node(node.layer.input[0], copy=True)
-        depth = self.graph.get_node(node.layer.input[1], copy=True)
-        on_value = self.graph.get_node(node.layer.input[2], copy=True)
-        off_value = self.graph.get_node(node.layer.input[3], copy=True)
-        assert depth.layer_type == 'Const', 'Parameter depth should be Const in OneHot'
-        assert on_value.layer_type == 'Const', 'Parameter on_value should be Const in OneHot'
-        assert off_value.layer_type == 'Const', 'Parameter off_value should be Const in OneHot'
-        self.add_omit_nodes(depth.layer_name, node.layer_name)
-        self.add_omit_nodes(on_value.layer_name, node.layer_name)
-        self.add_omit_nodes(off_value.layer_name, node.layer_name)
-        depth = depth.value
-        on_value = on_value.value
-        off_value = off_value.value
-        assert math.fabs(on_value -
-                         1.0) < 1e-06, "on_value should be 1 in OneHot"
-        assert math.fabs(off_value -
-                         0.0) < 1e-06, "off_value should be 0 in OneHot"
-        attr = {'depth': depth}
-        node.fluid_code.add_layer(
-            "one_hot",
-            inputs=input,
-            output=node,
-            param_attr=attr,
-            use_fluid=True)
-
-    def Pow(self, node):
-        x = self.graph.get_node(node.layer.input[0], copy=True)
-        factor = self.graph.get_node(node.layer.input[1], copy=True)
-        self.add_omit_nodes(factor.layer_name, node.layer_name)
-        if factor.layer_type == 'Const':
-            factor = factor.value.tolist()
-        else:
-            factor = self.decoder.infer_tensor(factor)
-        attr = {'factor': factor}
-        node.fluid_code.add_layer("pow", inputs=x, output=node, param_attr=attr)
-
-    def All(self, node):
-        input = self.graph.get_node(node.layer.input[0], copy=True)
-        reduce_idx = self.graph.get_node(node.layer.input[1], copy=True)
-        self.add_omit_nodes(reduce_idx.layer_name, node.layer_name)
-        assert reduce_idx.layer_type == "Const", "Only support Const parameter[reduce_idx]"
-        dims = reduce_idx.value.tolist()
-        keep_dims = node.get_attr("keep_dims")
-
-        attr = {"dim": dims, "keep_dim": keep_dims}
-        node.fluid_code.add_layer(
-            "reduce_all", inputs=input, output=node, param_attr=attr)
-
-    def GatherV2(self, node):
-        embeddings = self.graph.get_node(node.layer.input[0], copy=True)
-        index = self.graph.get_node(node.layer.input[1], copy=True)
-        axis = self.graph.get_node(node.layer.input[2], copy=True)
-        self.add_omit_nodes(axis.layer_name, node.layer_name)
-        assert axis.layer_type == 'Const', "Only support Const parameter[axis]"
-        axis = axis.value.tolist()
-        assert axis == 0, "Only support axis=0 in GatherV2 OP"
-        attr = {'overwrite': False}
-        embeddings_shape = embeddings.out_shapes[0][-1]
-        reshape_list = list()
-        reshape_name = index.layer_name
-        if len(index.out_shapes[0]) != 1:
-            reshape_list = index.out_shapes[0]
-            reshape_attr = {"shape": [-1]}
-            reshape_name = "{}_reshape".format(index.layer_name)
-            node.fluid_code.add_layer(
-                "reshape",
-                inputs=index,
-                output=reshape_name,
-                param_attr=reshape_attr)
-        inputs = {'input': embeddings, 'index': reshape_name}
-        node.fluid_code.add_layer(
-            "gather", inputs=inputs, output=node, param_attr=attr)
-        if len(index.out_shapes[0]) != 1:
-            reshape_attr = {"shape": reshape_list + [embeddings_shape]}
-            node.fluid_code.add_layer(
-                "reshape", inputs=node, output=node, param_attr=reshape_attr)
-
-    def OneShotIterator(self, node):
-        return self.Placeholder(node)
-
-    def IteratorV2(self, node):
-        dtype_map = {
-            1: "float32",
-            3: "int32",
-            4: "uint8",
-            9: "int64",
-            10: "bool"
-        }
-        shapes = node.out_shapes
-        dtypes = node.layer.attr['output_types'].list.type
-        node.fluid_code.add_note("{} = [0] * {}".format(node.layer_name,
-                                                        len(shapes)))
-        for i, shape in enumerate(shapes):
-            attr = {
-                'dtype': string(dtype_map[dtypes[i]]),
-                'shape': shape,
-                'name': string("{}_{}".format(node.layer_name, i)),
-                'append_batch_size': False
-            }
-            output = "{}[{}]".format(node.layer_name, i)
-            node.fluid_code.add_layer(
-                "data", inputs=None, output=output, param_attr=attr)

x2paddle/optimizer/tensorflow/batch_norm.py

+import copy
+from collections import OrderedDict
+from x2paddle.core.program import PaddleLayer
+
+
+class BatchNormOpt:
+    def __init__(self):
+        pass
+
+    def run(self, graph):
+        layers = copy.deepcopy(graph.layers)
+        for layer_id, layer in layers.items():
+            if layer.kernel != "fluid.layers.elementwise_add":
+                continue
+            axis = layer.attrs.get('axis', -1)
+            if axis != -1 and axis != 3:
+                continue
+
+            input_ids0 = graph.edges_in[layer_id]
+            mul_layer0 = graph.layers[input_ids0[0]]
+            sub_layer0 = graph.layers[input_ids0[1]]
+            if mul_layer0.kernel != "fluid.layers.elementwise_mul":
+                continue
+            if sub_layer0.kernel != "fluid.layers.elementwise_sub":
+                continue
+            axis = mul_layer0.attrs.get('axis', -1)
+            if axis != -1 and axis != 3:
+                continue
+            axis = sub_layer0.attrs.get('axis', -1)
+            if axis != -1 and axis != 0:
+                continue
+            if len(graph.edges_out.get(input_ids0[0], [])) != 1:
+                continue
+            if len(graph.edges_out.get(input_ids0[1], [])) != 1:
+                continue
+
+            input_ids1 = graph.edges_in[input_ids0[0]]
+            nhwc_input = graph.layers[input_ids1[0]]
+            mul_layer1 = graph.layers[input_ids1[1]]
+            if mul_layer1.kernel != "fluid.layers.elementwise_mul":
+                continue
+            axis = mul_layer1.attrs.get('axis', -1)
+            if axis != -1 and axis != 0:
+                continue
+            if len(graph.edges_out.get(input_ids1[1], [])) != 2:
+                continue
+
+            input_ids2 = graph.edges_in[input_ids0[1]]
+            beta = graph.layers[input_ids2[0]]
+            mul_layer2 = graph.layers[input_ids2[1]]
+            if beta.kernel != "fluid.layers.create_parameter":
+                continue
+            axis = mul_layer2.attrs.get('axis', -1)
+            if axis != -1 and axis != 0:
+                continue
+            if len(graph.edges_out.get(input_ids2[0], [])) != 1:
+                continue
+            if len(graph.edges_out.get(input_ids2[1], [])) != 1:
+                continue
+            if beta.outputs[0] not in graph.parameters:
+                continue
+            beta_shape = graph.parameters[beta.outputs[0]].shape
+            if len(beta_shape) != 1:
+                continue
+
+            input_ids3 = graph.edges_in[input_ids2[1]]
+            mean = graph.layers[input_ids3[0]]
+            mul_layer3 = graph.layers[input_ids3[1]]
+            if mean.kernel != "fluid.layers.create_parameter":
+                continue
+            axis = mul_layer3.attrs.get('axis', -1)
+            if axis != -1 and axis != 0:
+                continue
+            if len(graph.edges_out.get(input_ids3[0], [])) != 1:
+                continue
+            if len(graph.edges_out.get(input_ids3[1], [])) != 2:
+                continue
+            if mul_layer3.id != mul_layer1.id:
+                continue
+            if mean.outputs[0] not in graph.parameters:
+                continue
+            mean_shape = graph.parameters[mean.outputs[0]].shape
+            if mean_shape != beta_shape:
+                continue
+
+            input_ids4 = graph.edges_in[input_ids3[1]]
+            rsqrt_layer = graph.layers[input_ids4[0]]
+            gamma = graph.layers[input_ids4[1]]
+            if rsqrt_layer.kernel != "fluid.layers.rsqrt":
+                continue
+            if gamma.kernel != "fluid.layers.create_parameter":
+                continue
+            if len(graph.edges_out.get(input_ids4[0], [])) != 1:
+                continue
+            if len(graph.edges_out.get(input_ids4[1], [])) != 1:
+                continue
+            if gamma.outputs[0] not in graph.parameters:
+                continue
+            gamma_shape = graph.parameters[gamma.outputs[0]].shape
+            if gamma_shape != beta_shape:
+                continue
+
+            input_ids5 = graph.edges_in[input_ids4[0]]
+            add_layer = graph.layers[input_ids5[0]]
+            if add_layer.kernel != "fluid.layers.elementwise_add":
+                continue
+            axis = add_layer.attrs.get('axis', -1)
+            if axis != -1 and axis != 0:
+                continue
+            if len(graph.edges_out.get(input_ids5[0], [])) != 1:
+                continue
+
+            input_ids6 = graph.edges_in[input_ids5[0]]
+            variance = graph.layers[input_ids6[0]]
+            other = graph.layers[input_ids6[1]]
+            if variance.kernel != "fluid.layers.create_parameter":
+                continue
+            if other.kernel != "fluid.layers.create_parameter":
+                continue
+            if len(graph.edges_out.get(input_ids6[0], [])) != 1:
+                continue
+            if len(graph.edges_out.get(input_ids6[1], [])) != 1:
+                continue
+            if variance.outputs[0] not in graph.parameters:
+                continue
+            variance_shape = graph.parameters[variance.outputs[0]].shape
+            if variance_shape != beta_shape:
+                continue
+            if other.outputs[0] not in graph.parameters:
+                continue
+            if graph.parameters[other.outputs[0]].size != 1:
+                continue
+
+            ids = set([
+                layer_id, mul_layer0.id, sub_layer0.id, mul_layer1.id, beta.id,
+                mul_layer2.id, mean.id, mul_layer2.id, rsqrt_layer.id, gamma.id,
+                add_layer.id, variance.id, other.id
+            ])
+
+            for id in ids:
+                del graph.layers[id]
+                if id in graph.edges_in:
+                    del graph.edges_in[id]
+                if id in graph.edges_out:
+                    del graph.edges_out[id]
+
+            copy_layers = copy.deepcopy(graph.layers)
+            graph.layers = OrderedDict()
+            for k, v in copy_layers.items():
+                if k != nhwc_input.id:
+                    graph.layers[k] = v
+                    continue
+                graph.layers[k] = v
+                transpose0 = PaddleLayer(
+                    id='{}_1'.format(k),
+                    kernel="fluid.layers.transpose",
+                    inputs={"x": v.outputs[0]},
+                    outputs=["transpose_for_bn"],
+                    perm=[0, 3, 1, 2])
+                bn = PaddleLayer(
+                    id='{}_2'.format(k),
+                    kernel="fluid.layers.batch_norm",
+                    inputs={"input": "transpose_for_bn"},
+                    outputs=layer.outputs,
+                    epsilon=graph.parameters[other.outputs[0]],
+                    param_attr="'{}'".format(gamma.outputs[0]),
+                    bias_attr="'{}'".format(beta.outputs[0]),
+                    moving_mean_name="'{}'".format(mean.outputs[0]),
+                    moving_variance_name="'{}'".format(variance.outputs[0]))
+                transpose1 = PaddleLayer(
+                    id=layer_id,
+                    kernel="fluid.layers.transpose",
+                    inputs={"x": layer.outputs[0]},
+                    outputs=layer.outputs,
+                    perm=[0, 2, 3, 1])
+                graph.layers[transpose0.id] = transpose0
+                graph.layers[bn.id] = bn
+                graph.layers[transpose1.id] = transpose1
+        graph.build()

x2paddle/optimizer/tensorflow/bias.py

+import copy
+
+
+class BiasOpt:
+    def __init__(self):
+        self.conv_layers = [
+            'fluid.layers.conv2d', 'fluid.layers.conv2d_transpose'
+        ]
+        self.act_layers = [
+            'fluid.layers.relu', 'fluid.layers.relu6', 'fluid.layers.sigmoid',
+            'fluid.layers.exp', 'fluid.layers.tanh', 'fluid.layers.softplus',
+            'fluid.layers.leaky_relu'
+        ]
+
+    def run(self, graph):
+        layers = copy.deepcopy(graph.layers)
+        for layer_id, layer in layers.items():
+            if layer.kernel in self.conv_layers or layer.kernel == "fluid.layers.transpose":
+                if len(graph.edges_out.get(layer_id, [])) > 1:
+                    continue
+                if layer.outputs[0] in graph.outputs:
+                    continue
+
+                out_layer_id = graph.edges_out[layer_id][0]
+                if graph.layers[
+                        out_layer_id].kernel != "fluid.layers.elementwise_add":
+                    continue
+                if graph.layers[out_layer_id].attrs.get('axis', -1) != -1:
+                    continue
+
+                in_layer_id = graph.edges_in[out_layer_id]
+                bias_layer_id = in_layer_id[1 - in_layer_id.index(layer_id)]
+                if graph.layers[
+                        bias_layer_id].kernel != "fluid.layers.create_parameter":
+                    continue
+
+                bias_layer = graph.layers[bias_layer_id]
+                if len(bias_layer.attrs['shape']) != 1:
+                    continue
+                if len(graph.edges_out[bias_layer_id]) != 1:
+                    continue
+
+                if layer.kernel == "fluid.layers.transpose":
+                    if layer.attrs['perm'] != [0, 2, 3, 1]:
+                        continue
+                    in_layer_id = graph.edges_in[layer_id][0]
+                    if graph.layers[in_layer_id].kernel not in self.conv_layers:
+                        continue
+                    if graph.layers[in_layer_id].attrs['bias_attr'] != False:
+                        continue
+                    if graph.layers[in_layer_id].outputs[0] in graph.outputs:
+                        continue
+                    if len(graph.edges_out[in_layer_id]) != 1:
+                        continue
+                    graph.layers[in_layer_id].attrs[
+                        'bias_attr'] = bias_layer.attrs['name']
+                else:
+                    graph.layers[layer_id].attrs[
+                        'bias_attr'] = bias_layer.attrs['name']
+                bias_add_outs = graph.edges_out.get(out_layer_id, [])
+                bias_add_output = graph.layers[out_layer_id].outputs[0]
+                graph.del_layer(bias_layer_id)
+                graph.del_layer(out_layer_id)
+
+                for out in bias_add_outs:
+                    for k, v in graph.layers[out].inputs.items():
+                        if v == layer.outputs[0]:
+                            graph.layers[out].inputs[k] = bias_add_output
+                graph.layers[layer_id].outputs[0] = bias_add_output
+
+                if layer.kernel == "fluid.layers.transpose":
+                    in_layer_id = graph.edges_in[layer_id][0]
+                    graph.layers[in_layer_id].outputs[0] = bias_add_output
+                    graph.layers[layer_id].inputs['x'] = bias_add_output

x2paddle/optimizer/tensorflow/transpose.py

+import copy
+import sys
+
+
+class TransposeOpt:
+    def __init__(self):
+        self.image_layers = [
+            'fluid.layers.conv2d', 'fluid.layers.batch_norm',
+            'fluid.layers.conv2d_transpose', 'fluid.layers.resize_nearest',
+            'fluid.layers.resize_bilinear', 'fluid.layers.pool2d',
+            'fluid.layers.pad2d'
+        ]
+        self.direct_layers = [
+            'fluid.layers.relu', 'fluid.layers.relu6', 'fluid.layers.abs',
+            'fluid.layers.sigmoid', 'fluid.layers.exp', 'fluid.layers.rsqrt',
+            'fluid.layers.swish_f32', 'fluid.layers.tanh',
+            'fluid.layers.softplus', 'fluid.layers.leaky_relu',
+            'fluid.layers.floor', 'fluid.layers.erf', 'fluid.layers.swish'
+        ]
+        self.elementwise_layers = [
+            'fluid.layers.elementwise_add', 'fluid.layers.elementwise_sub',
+            'fluid.layers.elementwise_mul', 'fluid.layers.elementwise_div'
+        ]
+        #        self.reduce_layers = []
+        self.reduce_layers = [
+            'fluid.layers.reduce_mean', 'fluid.layers.reduce_all',
+            'fluid.layers.reduce_max', 'fluid.layers.reduce_any',
+            'fluid.layers.reduce_sum', 'fluid.layers.reduce_prod'
+        ]
+
+    def get_transpose_num(self, graph):
+        count = 0
+        for layer_id, layer in graph.layers.items():
+            if layer.kernel == "fluid.layers.transpose":
+                count += 1
+        return count
+
+    def run(self, graph):
+        total_layer_num = len(graph.layers)
+        scanned_layers = set()
+        optimized_transpose_layers = list()
+        optimized_reduce_layers = list()
+        optimized_concat_layers = list()
+        optimized_elementwise_layers = list()
+
+        def strip_transpose(_graph):
+            layers = copy.deepcopy(_graph.layers)
+            for layer_id, layer in layers.items():
+                if layer_id in scanned_layers:
+                    continue
+                scanned_layers.add(layer_id)
+                percent = round(len(scanned_layers) / total_layer_num * 100, 2)
+                sys.stderr.write("\rOptimize Transpose Layers...{}%".format(
+                    percent))
+
+                if layer.kernel != "fluid.layers.transpose":
+                    continue
+                if layer.attrs["perm"] != [0, 2, 3, 1]:
+                    continue
+                transpose_layers = list()
+                propagate_layers = list()
+                reduce_layers = list()
+                concat_layers = list()
+                # 此elementwise_layers专用于存储shape(4) + shape(1)的形式layer
+                elementwise_layers = list()
+                can_be_optimized = True
+                for out in _graph.edges_out.get(layer_id, []):
+                    if _graph.layers[out].kernel == "fluid.layers.transpose":
+                        if _graph.layers[out].attrs["perm"] != [0, 3, 1, 2]:
+                            can_be_optimized = False
+                            break
+                        transpose_layers.append(out)
+                    elif _graph.layers[out].kernel in self.elementwise_layers:
+                        propagate_layers.append(out)
+                    elif _graph.layers[out].kernel in self.direct_layers:
+                        if _graph.layers[out].outputs[0] in _graph.outputs:
+                            can_be_optimized = False
+                            break
+                        propagate_layers.append(out)
+                    elif _graph.layers[out].kernel in self.reduce_layers:
+                        if _graph.layers[out].outputs[0] in _graph.outputs:
+                            can_be_optimized = False
+                            break
+                        if not _graph.layers[out].attrs.get('keep_dim', False):
+                            can_be_optimized = False
+                            break
+                        propagate_layers.append(out)
+                        reduce_layers.append(out)
+                    elif _graph.layers[out].kernel == "fluid.layers.concat":
+                        if _graph.layers[out].outputs[0] in _graph.outputs:
+                            can_be_optimized = False
+                            break
+                        propagate_layers.append(out)
+                        concat_layers.append(out)
+                    else:
+                        can_be_optimized = False
+                        break
+
+                visited_layers = set()
+                while len(propagate_layers) > 0 and can_be_optimized:
+                    current_id = propagate_layers.pop(0)
+                    visited_layers.add(current_id)
+                    for out in _graph.edges_out.get(current_id, []):
+                        if _graph.layers[
+                                out].kernel == "fluid.layers.transpose":
+                            if _graph.layers[out].attrs["perm"] != [0, 3, 1, 2]:
+                                can_be_optimized = False
+                                break
+                            transpose_layers.append(out)
+                        elif _graph.layers[
+                                out].kernel in self.elementwise_layers:
+                            if _graph.layers[out].outputs[0] in _graph.outputs:
+                                can_be_optimized = False
+                                break
+                            if out not in visited_layers:
+                                propagate_layers.append(out)
+                        elif _graph.layers[out].kernel in self.direct_layers:
+                            if _graph.layers[out].outputs[0] in _graph.outputs:
+                                can_be_optimized = False
+                                break
+                            if out not in visited_layers:
+                                propagate_layers.append(out)
+                        elif _graph.layers[out].kernel in self.reduce_layers:
+                            if _graph.layers[out].outputs[0] in _graph.outputs:
+                                can_be_optimized = False
+                                break
+                            if not _graph.layers[out].attrs.get('keep_dim',
+                                                                False):
+                                can_be_optimized = False
+                                break
+                            if out not in visited_layers:
+                                propagate_layers.append(out)
+                                reduce_layers.append(out)
+                        elif _graph.layers[out].kernel == "fluid.layers.concat":
+                            if _graph.layers[out].outputs[0] in _graph.outputs:
+                                can_be_optimized = False
+                                break
+                            if out not in visited_layers:
+                                propagate_layers.append(out)
+                                concat_layers.append(out)
+                        else:
+                            can_be_optimized = False
+                            break
+                    for ipt in _graph.edges_in.get(current_id, []):
+                        if _graph.layers[
+                                current_id].kernel in self.elementwise_layers:
+                            try:
+                                x_shape = _graph.layers[
+                                    current_id].input_shapes['x']
+                                y_shape = _graph.layers[
+                                    current_id].input_shapes['y']
+                                if _graph.layers[ipt].outputs[
+                                        0] == _graph.layers[current_id].inputs[
+                                            'x']:
+                                    if len(x_shape) <= 1:
+                                        elementwise_layers.append(current_id)
+                                        continue
+                                elif _graph.layers[ipt].outputs[
+                                        0] == _graph.layers[current_id].inputs[
+                                            'y']:
+                                    if len(y_shape) <= 1:
+                                        elementwise_layers.append(current_id)
+                                        continue
+                                else:
+                                    raise Exception(
+                                        "Unexcepted situation happend while optimizing transpose"
+                                    )
+                            except Exception as e:
+                                can_be_optimized = False
+                                break
+                        if _graph.layers[
+                                ipt].kernel == "fluid.layers.transpose":
+                            if _graph.layers[ipt].attrs["perm"] != [0, 2, 3, 1]:
+                                can_be_optimized = False
+                                break
+                            if ipt not in visited_layers:
+                                transpose_layers.append(ipt)
+                        elif _graph.layers[
+                                ipt].kernel in self.elementwise_layers:
+                            if _graph.layers[ipt].outputs[0] in _graph.outputs:
+                                can_be_optimized = False
+                                break
+                            if ipt not in visited_layers:
+                                propagate_layers.append(ipt)
+                        elif _graph.layers[ipt].kernel in self.direct_layers:
+                            if _graph.layers[ipt].outputs[0] in _graph.outputs:
+                                can_be_optimized = False
+                                break
+                            if ipt not in visited_layers:
+                                propagate_layers.append(ipt)
+                        elif _graph.layers[ipt].kernel in self.reduce_layers:
+                            if _graph.layers[ipt].outputs[0] in _graph.outputs:
+                                can_be_optimized = False
+                                break
+                            if not _graph.layers[ipt].attrs.get('keep_dim',
+                                                                False):
+                                can_be_optimized = False
+                                break
+                            if ipt not in visited_layers:
+                                propagate_layers.append(ipt)
+                                reduce_layers.append(ipt)
+                        elif _graph.layers[ipt].kernel == "fluid.layers.concat":
+                            if _graph.layers[ipt].outputs[0] in _graph.outputs:
+                                can_be_optimized = False
+                                break
+                            if ipt not in visited_layers:
+                                propagate_layers.append(ipt)
+                                concat_layers.append(ipt)
+                        else:
+                            can_be_optimized = False
+                            break
+                    if not can_be_optimized:
+                        break
+                if not can_be_optimized:
+                    continue
+
+                transpose_layers.append(layer_id)
+                transpose_layers = list(set(transpose_layers))
+                for l in transpose_layers:
+                    if graph.layers[l].outputs[0] in graph.outputs:
+                        can_be_optimized = False
+                        break
+                if not can_be_optimized:
+                    continue
+
+                for l in transpose_layers:
+                    _graph.del_layer(l)
+
+                optimized_transpose_layers.extend(transpose_layers)
+                optimized_reduce_layers.extend(reduce_layers)
+                optimized_concat_layers.extend(concat_layers)
+                optimized_elementwise_layers.extend(elementwise_layers)
+                return True
+            return False
+
+        before_transpose_num = self.get_transpose_num(graph)
+        opt_graph = copy.deepcopy(graph)
+        total_layer_num = len(opt_graph.layers)
+
+        while strip_transpose(opt_graph):
+            pass
+
+        for layer_id in list(set(optimized_transpose_layers)):
+            graph.del_layer(layer_id)
+        for layer_id in list(set(optimized_reduce_layers)):
+            dim = graph.layers[layer_id].attrs.get('dim', None)
+            if dim is not None:
+                for i in range(len(dim)):
+                    dim[i] = [0, 2, 3, 1][dim[i]]
+                graph.layers[layer_id].attrs['dim'] = dim
+        for layer_id in list(set(optimized_concat_layers)):
+            axis = graph.layers[layer_id].attrs.get('axis', 0)
+            graph.layers[layer_id].attrs['axis'] = [0, 2, 3, 1][axis]
+        for layer_id in list(set(optimized_elementwise_layers)):
+            axis = graph.layers[layer_id].attrs.get('axis', -1)
+            graph.layers[layer_id].attrs['axis'] = [0, 2, 3, 1][axis]
+
+        current_transpose_num = self.get_transpose_num(graph)
+        print(
+            "\nTranspose layers optimized, before: transpose_num={}, after: transpose_num={}".
+            format(before_transpose_num, current_transpose_num))

x2paddle/optimizer/tf_optimizer.py

-#   Copyright (c) 2019  PaddlePaddle Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License"
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# TODO useless node remove
-from x2paddle.op_mapper.tf_op_mapper import TFOpMapper
-from x2paddle.core.fluid_code import Layer
-from x2paddle.core.util import *
-import six
-import numpy
-import copy as cp
-
-
-def exist_act(node):
-    for layer in node.fluid_code.layers:
-        if layer.param_attr is not None:
-            act = layer.param_attr.get("act", None)
-            if act is not None:
-                return True
-    return False
-
-
-class TFOptimizer(object):
-    activation_ops = {
-        'Relu': 'relu',
-        'Sigmoid': 'sigmoid',
-        'Relu6': 'relu6',
-        'swish_f32': 'swish'
-    }
-    layers_with_act = [
-        'Conv2D', 'BiasAdd', 'DepthwiseConv2dNative', 'Conv2DBackpropInput',
-        'FusedBatchNorm', 'conv2d', 'elementwise_add', 'conv2d_transpose',
-        'batch_norm'
-    ]
-    layers_with_bias = [
-        'Conv2D', 'DepthwiseConv2dNative', 'Conv2DBackpropInput', 'conv2d',
-        'conv2d_transpose'
-    ]
-
-    def __init__(self, op_mapper):
-        self.op_mapper = op_mapper
-        self.graph = op_mapper.graph
-
-    def delete_redundance_code(self):
-        for node_name in self.graph.topo_sort:
-            if node_name in self.op_mapper.omit_nodes:
-                node = self.graph.get_node(node_name)
-                if node is None:
-                    continue
-                omit_freq = self.op_mapper.omit_nodes.count(node_name)
-                if len(node.outputs) <= omit_freq:
-                    node.fluid_code.clear()
-
-                    # remove node from graph
-                    input_names = node.inputs
-                    output_names = node.outputs
-                    for in_name in input_names:
-                        in_node = self.graph.get_node(in_name)
-                        index = in_node.outputs.index(node_name)
-                        del in_node.outputs[index]
-                    for out_name in output_names:
-                        out_node = self.graph.get_node(out_name)
-                        index = out_node.inputs.index(node_name)
-                        del out_node.inputs[index]
-                    del self.graph.node_map[node_name]
-
-    def strip_graph(self):
-        visited_nodes = set()
-
-        def visit(node_name):
-            if node_name in visited_nodes:
-                return
-            visited_nodes.add(node_name)
-            input_names = self.graph.get_node(node_name).inputs
-            for in_name in input_names:
-                visit(in_name)
-
-        for node_name in self.graph.output_nodes:
-            visit(node_name)
-
-        for i, node_name in enumerate(self.graph.topo_sort):
-            if node_name not in visited_nodes:
-                node = self.graph.get_node(node_name)
-                if node is None:
-                    continue
-                input_names = node.inputs
-                output_names = node.outputs
-                for in_name in input_names:
-                    in_node = self.graph.get_node(in_name)
-                    index = in_node.outputs.index(node_name)
-                    del in_node.outputs[index]
-                for out_name in output_names:
-                    out_node = self.graph.get_node(out_name)
-                    index = out_node.inputs.index(node_name)
-                    del out_node.inputs[index]
-                del self.graph.node_map[node_name]
-
-    def optimize_elementwise_op(self):
-        elementwise_ops = [
-            'Sub', 'Add', 'RealDiv', 'Maximum', 'Mul', 'FloorDiv',
-            'GreaterEqual'
-        ]
-        revertable_ops = ['Add', 'Mul']
-        for node_name in self.graph.topo_sort:
-            node = self.graph.get_node(node_name)
-            if node is None:
-                continue
-            if node.layer_type in elementwise_ops:
-                if len(node.fluid_code.layers) != 2:
-                    continue
-                if node.fluid_code.layers[0].op != "expand":
-                    continue
-                expand_out = node.fluid_code.layers[0].output
-                expand_in = node.fluid_code.layers[0].inputs
-                expand_times = node.fluid_code.layers[0].param_attr[
-                    "expand_times"]
-
-                x = node.fluid_code.layers[1].inputs["x"]
-                y = node.fluid_code.layers[1].inputs["y"]
-                if isinstance(
-                        x,
-                        six.string_types) and node.layer_type in revertable_ops:
-                    node.fluid_code.layers[1].inputs["y"] = x
-                    node.fluid_code.layers[1].inputs["x"] = y
-                    x = node.fluid_code.layers[1].inputs["x"]
-                    y = expand_in
-                elif isinstance(y, six.string_types):
-                    y = expand_in
-                else:
-                    continue
-
-                x_shape = x.out_shapes[0]
-                y_shape = y.out_shapes[0]
-                if len(x_shape) != len(y_shape):
-                    continue
-                if len(x_shape) == 4:
-                    x_shape = [x_shape[i] for i in [0, 3, 1, 2]]
-                    y_shape = [y_shape[i] for i in [0, 3, 1, 2]]
-
-                continue_flag = True
-                for i in range(len(x_shape)):
-                    if y_shape[-1 * (i + 1)] == 1 and continue_flag:
-                        expand_times[-1 * (i + 1)] = 1
-                    else:
-                        continue_flag = False
-
-                if expand_times.count(1) == len(expand_times):
-                    node.fluid_code.layers[1].inputs["y"] = expand_in
-                    del node.fluid_code.layers[0]
-
-    def merge_activation(self):
-        act_nodes = list()
-        for node_name in self.graph.topo_sort:
-            node = self.graph.get_node(node_name)
-            if node is None:
-                continue
-            if node.layer_type in self.activation_ops:
-                act_nodes.append(node_name)
-
-        for act_node_name in act_nodes:
-            node = self.graph.get_node(act_node_name)
-            input = self.graph.get_node(node.inputs[0])
-            if input.layer_type not in self.layers_with_act:
-                continue
-            if len(input.fluid_code.layers) == 0:
-                continue
-            if 'act' in input.fluid_code.layers[
-                    -1].param_attr and input.fluid_code.layers[-1].param_attr[
-                        'act'] is not None:
-                continue
-            if len(input.outputs) != 1:
-                continue
-            index = -1
-            for i in range(len(input.fluid_code.layers)):
-                if input.fluid_code.layers[i].op in self.layers_with_act:
-                    index = i
-                    break
-            input.fluid_code.layers[index].param_attr['act'] = string(
-                self.activation_ops[node.layer_type])
-            input.fluid_code.layers[-1].output = node.fluid_code.layers[
-                0].output
-            self.graph.remove_node(act_node_name)
-
-    def merge_bias(self):
-        for node_name in self.graph.topo_sort:
-            node = self.graph.get_node(node_name)
-            if node is None:
-                continue
-            if node.layer_type == "BiasAdd":
-                input = self.graph.get_node(node.inputs[0])
-                if input.layer_type not in self.layers_with_bias:
-                    continue
-                if len(input.outputs) != 1:
-                    continue
-                if len(input.fluid_code.layers) == 0:
-                    continue
-                bias_with_act = False
-                if 'act' in node.fluid_code.layers[-1].param_attr:
-                    bias_with_act = True
-                layer_with_act = False
-                index = -1
-                for i in range(len(input.fluid_code.layers)):
-                    if input.fluid_code.layers[i].op in self.layers_with_bias:
-                        index = i
-                        break
-                if 'act' in input.fluid_code.layers[
-                        index].param_attr and input.fluid_code.layers[
-                            index].param_attr['act'] is not None:
-                    layer_with_act = True
-
-                if bias_with_act and layer_with_act:
-                    continue
-                if not input.fluid_code.layers[index].param_attr['bias_attr']:
-                    bias_name = node.inputs[1]
-                    input.fluid_code.layers[index].param_attr[
-                        'bias_attr'] = string(bias_name)
-                    input.fluid_code.layers[-1].output = node.fluid_code.layers[
-                        0].output
-                    if bias_with_act:
-                        input.fluid_code.layers[index].param_attr[
-                            'act'] = node.fluid_code.layers[-1].param_attr[
-                                'act']
-                    node.fluid_code.clear()
-                    self.graph.remove_node(node.layer_name)
-                    self.graph.identity_map[node.layer_name] = input.layer_name
-
-    def remove_transpose(self):
-        graph_copy = cp.deepcopy(self.graph)
-        elementwise_ops = [
-            'Sub', 'Add', 'RealDiv', 'Maximum', 'Mul', 'FloorDiv',
-            'GreateerEqual'
-        ]
-        can_be_optimized_ops = [
-            'Conv2D', 'MaxPool', 'FusedBatchNorm', 'DepthwiseConv2dNative',
-            'AvgPool', 'Pad', 'Conv2DBackpropInput', 'ResizeNearestNeighbor',
-            'Placeholder', 'Relu', 'Relu6', 'Abs', 'Sigmoid', 'Exp', 'Rsqrt',
-            'swish_f32', 'LeakyRelu', 'Cast', 'Tanh'
-        ]
-        # These ops may have one more Variable input
-        can_be_optimized_special_ops = ['ResizeBilinear']
-        for node_name in self.graph.topo_sort:
-            node = graph_copy.get_node(node_name)
-            if node is None:
-                continue
-            if node.layer_type in can_be_optimized_ops:
-                if node.fluid_code.layers[
-                        -1].op != "transpose" or node.fluid_code.layers[
-                            -1].param_attr["perm"] != [0, 2, 3, 1]:
-                    continue
-                can_be_removed = True
-                output_names = node.outputs
-                for out_name in output_names:
-                    out_node = graph_copy.get_node(out_name)
-                    if hasattr(out_node, "can_be_removed"):
-                        if not out_node.can_be_removed:
-                            can_be_removed = False
-                            break
-                    elif out_node.fluid_code.layers[
-                            0].op != "transpose" or out_node.fluid_code.layers[
-                                0].param_attr["perm"] != [0, 3, 1, 2]:
-                        can_be_removed = False
-                        break
-                    elif out_node.layer_type in elementwise_ops or out_node.layer_type in can_be_optimized_special_ops:
-                        can_be_removed = False
-                        break
-
-                if can_be_removed and len(node.fluid_code.layers) > 1:
-                    true_node = self.graph.get_node(node_name)
-                    if true_node.layer_type == "Placeholder":
-                        index = self.graph.input_nodes.index(
-                            true_node.fluid_code.layers[-2].output)
-                        if isinstance(true_node.fluid_code.layers[-1].output,
-                                      str):
-                            self.graph.input_nodes[
-                                index] = true_node.fluid_code.layers[-1].output
-                        else:
-                            self.graph.input_nodes[
-                                index] = true_node.fluid_code.layers[
-                                    -1].output.layer_name
-                    true_node.fluid_code.layers[
-                        -2].output = true_node.fluid_code.layers[-1].output
-                    node.removed = True
-                    del true_node.fluid_code.layers[-1]
-
-                    for out_name in output_names:
-                        out_node = self.graph.get_node(out_name)
-                        out_node.fluid_code.layers[
-                            1].inputs = out_node.fluid_code.layers[0].inputs
-                        del out_node.fluid_code.layers[0]
-
-        for node_name in self.graph.topo_sort:
-            node = graph_copy.get_node(node_name)
-            if node is None:
-                continue
-            if node.layer_type in elementwise_ops:
-                can_be_removed = True
-                if node.fluid_code.layers[
-                        -1].op != "transpose" or node.fluid_code.layers[
-                            -1].param_attr["perm"] != [0, 2, 3, 1]:
-                    continue
-                can_be_removed = True
-
-                output_names = node.outputs
-                for out_name in output_names:
-                    out_node = graph_copy.get_node(out_name)
-                    if len(out_node.fluid_code.layers) < 3:
-                        can_be_removed = False
-                        break
-                    if hasattr(out_node, "can_be_removed"):
-                        if not out_node.can_be_removed:
-                            can_be_removed = False
-                            break
-                    if out_node.layer_type in can_be_optimized_ops:
-                        if out_node.fluid_code.layers[
-                                0].op != "transpose" or out_node.fluid_code.layers[
-                                    0].param_attr["perm"] != [0, 3, 1, 2]:
-                            can_be_removed = False
-                            break
-                    elif out_node.layer_type in elementwise_ops:
-                        if out_node.fluid_code.layers[
-                                0].op != "transpose" and out_node.fluid_code.layers[
-                                    1].op != "transpose":
-                            can_be_removed = False
-                            break
-                        if out_node.fluid_code.layers[0].op == "transpose":
-                            if out_node.fluid_code.layers[0].param_attr[
-                                    "perm"] != [0, 3, 1, 2]:
-                                can_be_removed = False
-                                break
-                        if out_node.fluid_code.layers[1].op == "transpose":
-                            if out_node.fluid_code.layers[1].param_attr[
-                                    "perm"] != [0, 3, 1, 2]:
-                                can_be_removed = False
-                                break
-
-                if can_be_removed and len(node.fluid_code.layers) > 1:
-                    true_node = self.graph.get_node(node_name)
-                    true_node.fluid_code.layers[
-                        -2].output = true_node.fluid_code.layers[-1].output
-                    del true_node.fluid_code.layers[-1]
-                    for out_name in output_names:
-                        out_node = self.graph.get_node(out_name)
-                        if out_node.layer_type in can_be_optimized_ops:
-                            out_node.fluid_code.layers[
-                                1].inputs = out_node.fluid_code.layers[0].inputs
-                            del out_node.fluid_code.layers[0]
-                        elif out_node.layer_type in elementwise_ops:
-                            if out_node.inputs[0] in node.layer_name:
-                                if out_node.fluid_code.layers[
-                                        1].op == 'transpose':
-                                    out_node.fluid_code.layers[2].inputs[
-                                        'x'] = out_node.fluid_code.layers[
-                                            0].inputs
-                                    del out_node.fluid_code.layers[0]
-                                else:
-                                    out_node.fluid_code.layers[1].inputs[
-                                        'x'] = out_node.fluid_code.layers[
-                                            0].inputs
-                                    del out_node.fluid_code.layers[0]
-                            elif out_node.inputs[1] in node.layer_name:
-                                if out_node.fluid_code.layers[
-                                        1].op == 'transpose':
-                                    out_node.fluid_code.layers[2].inputs[
-                                        'y'] = out_node.fluid_code.layers[
-                                            1].inputs
-                                    del out_node.fluid_code.layers[1]
-                                else:
-                                    out_node.fluid_code.layers[1].inputs[
-                                        'y'] = out_node.fluid_code.layers[
-                                            0].inputs
-                                    del out_node.fluid_code.layers[0]
-        graph_copy = cp.deepcopy(self.graph)
-        for node_name in self.graph.topo_sort:
-            node = graph_copy.get_node(node_name)
-            if node is None or len(node.fluid_code.layers) < 2:
-                continue
-            if node.layer_type in can_be_optimized_ops and node.layer_type != "Placeholder":
-                if node.fluid_code.layers[
-                        -1].op != "transpose" or node.fluid_code.layers[
-                            -1].param_attr["perm"] != [0, 2, 3, 1]:
-                    continue
-                can_be_removed = True
-                output_names = node.outputs
-                for out_name in output_names:
-                    out_node = graph_copy.get_node(out_name)
-                    if hasattr(out_node, "can_be_removed"):
-                        if not out_node.can_be_removed:
-                            can_be_removed = False
-                            break
-                    if len(out_node.fluid_code.layers) < 2:
-                        can_be_removed = False
-                        break
-                    if out_node.layer_type in can_be_optimized_ops:
-                        if out_node.fluid_code.layers[
-                                0].op != "transpose" or out_node.fluid_code.layers[
-                                    0].param_attr["perm"] != [0, 3, 1, 2]:
-                            can_be_removed = False
-                            break
-                    elif out_node.layer_type in elementwise_ops:
-                        if out_node.fluid_code.layers[
-                                0].op != "transpose" and out_node.fluid_code.layers[
-                                    1].op != "transpose":
-                            can_be_removed = False
-                            break
-                        if out_node.fluid_code.layers[
-                                0].op == "expand" or out_node.fluid_code.layers[
-                                    1].op == "expand":
-                            can_be_removed = False
-                            break
-                        if out_node.fluid_code.layers[0].op == "transpose":
-                            if out_node.fluid_code.layers[0].param_attr[
-                                    "perm"] != [0, 3, 1, 2]:
-                                can_be_removed = False
-                                break
-                        if out_node.fluid_code.layers[1].op == "transpose":
-                            if out_node.fluid_code.layers[1].param_attr[
-                                    "perm"] != [0, 3, 1, 2]:
-                                can_be_removed = False
-                                break
-                    elif out_node.layer_type not in elementwise_ops and out_node.layer_type not in can_be_optimized_ops:
-                        can_be_removed = False
-                        break
-
-                if can_be_removed:
-                    true_node = self.graph.get_node(node_name)
-                    if len(true_node.fluid_code.layers) < 2:
-                        continue
-                    true_node.fluid_code.layers[
-                        -2].output = true_node.fluid_code.layers[-1].output
-                    del true_node.fluid_code.layers[-1]
-                    for out_name in output_names:
-                        out_node = self.graph.get_node(out_name)
-                        if out_node.layer_type in can_be_optimized_ops:
-                            out_node.fluid_code.layers[
-                                1].inputs = out_node.fluid_code.layers[0].inputs
-                            del out_node.fluid_code.layers[0]
-                        elif out_node.layer_type in elementwise_ops:
-                            if out_node.inputs[0] in node.layer_name:
-                                if out_node.fluid_code.layers[
-                                        1].op == 'transpose':
-                                    if out_node.fluid_code.layers[
-                                            2].op == 'transpose':
-                                        out_node.fluid_code.layers[3].inputs[
-                                            'x'] = out_node.fluid_code.layers[
-                                                0].inputs
-                                    else:
-                                        out_node.fluid_code.layers[2].inputs[
-                                            'x'] = out_node.fluid_code.layers[
-                                                0].inputs
-                                    del out_node.fluid_code.layers[0]
-                                else:
-                                    out_node.fluid_code.layers[1].inputs[
-                                        'x'] = out_node.fluid_code.layers[
-                                            0].inputs
-                                    del out_node.fluid_code.layers[0]
-                            elif out_node.inputs[1] in node.layer_name:
-                                if out_node.fluid_code.layers[
-                                        1].op == 'transpose':
-                                    out_node.fluid_code.layers[2].inputs[
-                                        'y'] = out_node.fluid_code.layers[
-                                            1].inputs
-                                    del out_node.fluid_code.layers[1]
-                                else:
-                                    out_node.fluid_code.layers[1].inputs[
-                                        'y'] = out_node.fluid_code.layers[
-                                            0].inputs
-                                    del out_node.fluid_code.layers[0]
-
-        graph_copy = cp.deepcopy(self.graph)
-        for node_name in self.graph.topo_sort:
-            node = graph_copy.get_node(node_name)
-            if node is None:
-                continue
-            if node.layer_type in elementwise_ops:
-                can_be_removed = True
-                if len(node.fluid_code.layers) < 3:
-                    continue
-
-                numTranspose = 0
-                numNotTranspose = 0
-
-                for i in range(len(node.fluid_code.layers)):
-                    if node.fluid_code.layers[i].op == 'transpose':
-                        numTranspose += 1
-                    elif node.fluid_code.layers[i].op != 'expand':
-                        numNotTranspose += 1
-                if numTranspose > numNotTranspose:
-                    if node.fluid_code.layers[0].op == 'expand':
-                        if node.fluid_code.layers[
-                                1].op != 'transpose' or node.fluid_code.layers[
-                                    2].op != 'transpose':
-                            continue
-                        else:
-                            true_node = self.graph.get_node(node_name)
-                            true_node.fluid_code.layers[3].inputs[
-                                'x'] = true_node.fluid_code.layers[1].inputs
-                            true_node.fluid_code.layers[3].inputs[
-                                'y'] = true_node.fluid_code.layers[2].inputs
-
-                            l = Layer()
-                            l.op = 'transpose'
-                            l.inputs = true_node.fluid_code.layers[3].output
-                            l.param_attr = {'perm': [0, 3, 1, 2]}
-                            if isinstance(l.inputs, six.string_types):
-                                l.output = l.inputs
-                            else:
-                                l.output = l.inputs.layer_name
-                            true_node.fluid_code.layers.append(l)
-                            del true_node.fluid_code.layers[1]
-                            del true_node.fluid_code.layers[1]
-                    else:
-                        if node.fluid_code.layers[
-                                0].op != 'transpose' or node.fluid_code.layers[
-                                    1].op != 'transpose':
-                            continue
-                        else:
-                            true_node = self.graph.get_node(node_name)
-                            true_node.fluid_code.layers[2].inputs[
-                                'x'] = true_node.fluid_code.layers[0].inputs
-                            true_node.fluid_code.layers[2].inputs[
-                                'y'] = true_node.fluid_code.layers[1].inputs
-
-                            l = Layer()
-                            l.op = 'transpose'
-                            l.inputs = true_node.fluid_code.layers[2].output
-                            l.param_attr = {'perm': [0, 3, 1, 2]}
-                            l.output = l.inputs.layer_name
-                            true_node.fluid_code.layers.append(l)
-                            del true_node.fluid_code.layers[0]
-                            del true_node.fluid_code.layers[0]
-
-    def make_nchw_input_output(self):
-        for i, name in enumerate(self.graph.input_nodes):
-            node = self.graph.get_node(name)
-            if len(node.out_shapes[0]) == 4 and node.tf_data_format == "NHWC":
-                shape = node.fluid_code.layers[0].param_attr["shape"]
-                shape = [shape[j] for j in [0, 3, 1, 2]]
-                node.fluid_code.layers[0].param_attr["shape"] = shape
-                node.fluid_code.layers[0].output = "nhwc_" + name
-                attr = {"perm": [0, 2, 3, 1]}
-                node.fluid_code.add_layer(
-                    "transpose",
-                    inputs="nhwc_" + name,
-                    output=node,
-                    param_attr=attr)
-                self.graph.input_nodes[i] = "nhwc_" + name
-        for i, name in enumerate(self.graph.output_nodes):
-            node = self.graph.get_node(name)
-            if node.layer_type != "transpose":
-                if node.fluid_code.layers[-1].op == "transpose":
-                    node.fluid_code.layers[-2].output = name
-                    del node.fluid_code.layers[-1]
-
-    def optimize_sub_graph(self):
-        self.merge_batch_norm()
-        self.merge_prelu()
-        self.merge_scale()
-        self.merge_affine_channel()
-
-    def merge_batch_norm(self):
-        for i, name in enumerate(self.graph.topo_sort):
-            node = self.graph.get_node(name)
-            if node is None:
-                continue
-            is_batch_norm = True
-            if node.layer_type == "Add":
-                in_nodes0 = [
-                    self.graph.get_node(in_name) for in_name in node.inputs
-                ]
-                if in_nodes0[0].layer_type != "Mul" or in_nodes0[
-                        1].layer_type != "Sub":
-                    is_batch_norm = False
-                    continue
-
-                if exist_act(in_nodes0[0]) or exist_act(in_nodes0[1]):
-                    is_batch_norm = False
-                    continue
-
-                in_nodes1 = [
-                    self.graph.get_node(in_name)
-                    for in_name in in_nodes0[0].inputs
-                ]
-                in_nodes2 = [
-                    self.graph.get_node(in_name)
-                    for in_name in in_nodes0[1].inputs
-                ]
-                if len(in_nodes1[0].out_shapes[0]) != 4:
-                    is_batch_norm = False
-                    continue
-                if in_nodes1[1].layer_type != "Mul":
-                    is_batch_norm = False
-                    continue
-                if exist_act(in_nodes1[1]):
-                    is_batch_norm = False
-                    continue
-
-                if in_nodes2[0].layer_type != "Const" or in_nodes2[
-                        1].layer_type != "Mul":
-                    is_batch_norm = False
-                    continue
-                if exist_act(in_nodes2[1]):
-                    is_batch_norm = False
-                    continue
-
-                in_nodes3 = [
-                    self.graph.get_node(in_name)
-                    for in_name in in_nodes1[1].inputs
-                ]
-                if in_nodes3[0].layer_type != "Rsqrt" or in_nodes3[
-                        1].layer_type != "Const":
-                    is_batch_norm = False
-                    continue
-
-                in_nodes4 = [
-                    self.graph.get_node(in_name)
-                    for in_name in in_nodes2[1].inputs
-                ]
-                if in_nodes4[0].layer_type != "Const" or in_nodes4[
-                        1].layer_name != in_nodes1[1].layer_name:
-                    is_batch_norm = False
-                    continue
-
-                in_nodes5 = self.graph.get_node(in_nodes3[0].inputs[0])
-                if in_nodes5.layer_type != "Add":
-                    is_batch_norm = False
-                    continue
-                if exist_act(in_nodes5):
-                    is_batch_norm = False
-                    continue
-
-                in_nodes6 = [
-                    self.graph.get_node(in_name) for in_name in in_nodes5.inputs
-                ]
-                if in_nodes6[0].layer_type != "Const" or in_nodes6[
-                        1].layer_type != "Const":
-                    is_batch_norm = False
-                    continue
-
-                if len(in_nodes0[0].outputs) != 1:
-                    is_batch_norm = False
-                    continue
-                if len(in_nodes0[1].outputs) != 1:
-                    is_batch_norm = False
-                    continue
-                if len(in_nodes1[1].outputs) != 2:
-                    is_batch_norm = False
-                    continue
-                if len(in_nodes2[0].outputs) != 1:
-                    is_batch_norm = False
-                    continue
-                if len(in_nodes2[1].outputs) != 1:
-                    is_batch_norm = False
-                    continue
-                if len(in_nodes3[0].outputs) != 1:
-                    is_batch_norm = False
-                    continue
-                if len(in_nodes3[1].outputs) != 1:
-                    is_batch_norm = False
-                    continue
-                if len(in_nodes4[0].outputs) != 1:
-                    is_batch_norm = False
-                    continue
-                if len(in_nodes5.outputs) != 1:
-                    is_batch_norm = False
-                    continue
-                if len(in_nodes6[0].outputs) != 1:
-                    is_batch_norm = False
-                    continue
-                if len(in_nodes6[1].outputs) != 1:
-                    is_batch_norm = False
-                    continue
-
-                conv_shape = in_nodes1[0].out_shapes[0]
-                if conv_shape[3] < 0:
-                    is_batch_norm = False
-                    continue
-
-                # moving_variance
-                if in_nodes6[0].value.size != conv_shape[3]:
-                    is_batch_norm = False
-                    continue
-
-                # epsilon
-                if in_nodes6[1].value.size != 1:
-                    is_batch_norm = False
-                    continue
-
-                # gamma
-                if in_nodes3[1].value.size != conv_shape[3]:
-                    is_batch_norm = False
-                    continue
-
-                # moving_mean
-                if in_nodes4[0].value.size != conv_shape[3]:
-                    is_batch_norm = False
-                    continue
-
-                # beta
-                if in_nodes2[0].value.size != conv_shape[3]:
-                    is_batch_norm = False
-                    continue
-
-                if is_batch_norm:
-                    index = in_nodes1[0].outputs.index(in_nodes0[0].layer_name)
-                    in_nodes1[0].outputs[index] = node.layer_name
-                    node.layer_type = "FusedBatchNorm"
-                    node.inputs = [in_nodes1[0].layer_name]
-                    act = node.fluid_code.layers[-1].param_attr.get("act", None)
-                    node.fluid_code.clear()
-                    attr = {
-                        "epsilon": in_nodes6[1].value,
-                        "param_attr": string(in_nodes3[1].layer_name),
-                        "bias_attr": string(in_nodes2[0].layer_name),
-                        "moving_mean_name": string(in_nodes4[0].layer_name),
-                        "moving_variance_name": string(in_nodes6[0].layer_name),
-                        "is_test": True,
-                        "act": act
-                    }
-
-                    node.fluid_code.add_layer(
-                        "batch_norm",
-                        inputs=in_nodes1[0].fluid_code.layers[-1].output,
-                        output=node,
-                        param_attr=attr)
-
-                del self.graph.node_map[in_nodes0[0].layer_name]
-                del self.graph.node_map[in_nodes0[1].layer_name]
-                del self.graph.node_map[in_nodes1[1].layer_name]
-                del self.graph.node_map[in_nodes2[1].layer_name]
-                del self.graph.node_map[in_nodes3[0].layer_name]
-                del self.graph.node_map[in_nodes4[0].layer_name]
-                del self.graph.node_map[in_nodes5.layer_name]
-
-    def merge_prelu(self):
-        for i, name in enumerate(self.graph.topo_sort):
-            node = self.graph.get_node(name)
-            if node is None:
-                continue
-            is_prelu = True
-            if node.layer_type == "Add":
-                if exist_act(node):
-                    is_prelu = False
-                    continue
-                in_nodes0 = [
-                    self.graph.get_node(in_name) for in_name in node.inputs
-                ]
-                if in_nodes0[0].layer_type != "Relu" or in_nodes0[
-                        1].layer_type != "Mul":
-                    is_prelu = False
-                    continue
-                if exist_act(in_nodes0[1]):
-                    is_prelu = False
-                    continue
-
-                if len(in_nodes0[0].outputs) != 1 or len(in_nodes0[1]
-                                                         .outputs) != 1:
-                    is_prelu = False
-                    continue
-
-                in_nodes1 = self.graph.get_node(in_nodes0[0].inputs[0])
-                in_nodes2 = [
-                    self.graph.get_node(in_name)
-                    for in_name in in_nodes0[1].inputs
-                ]
-                if in_nodes2[1].layer_type != "Const" or numpy.fabs(in_nodes2[
-                        1].value - 0.5) > 1e-06:
-                    is_prelu = False
-                    continue
-                if in_nodes2[0].layer_type != "Mul":
-                    is_prelu = False
-                    continue
-                if exist_act(in_nodes2[0]):
-                    is_prelu = False
-                    continue
-                if len(in_nodes2[1].outputs) != 1 or len(in_nodes2[0]
-                                                         .outputs) != 1:
-                    is_prelu = False
-                    continue
-
-                in_nodes3 = [
-                    self.graph.get_node(in_name)
-                    for in_name in in_nodes2[0].inputs
-                ]
-                if in_nodes3[0].layer_type != "Const" or in_nodes3[
-                        1].layer_type != "Sub":
-                    is_prelu = False
-                    continue
-                if exist_act(in_nodes3[1]):
-                    is_prelu = False
-                    continue
-                if len(in_nodes3[0].outputs) != 1 or len(in_nodes3[1]
-                                                         .outputs) != 1:
-                    is_prelu = False
-                    continue
-
-                in_nodes4 = [
-                    self.graph.get_node(in_name)
-                    for in_name in in_nodes3[1].inputs
-                ]
-                if in_nodes4[0].layer_name != in_nodes1.layer_name or in_nodes4[
-                        1].layer_type != "Abs":
-                    is_prelu = False
-                    continue
-                if len(in_nodes4[1].outputs) != 1:
-                    is_prelu = False
-                    continue
-
-                in_nodes5 = self.graph.get_node(in_nodes4[1].inputs[0])
-                if in_nodes5.layer_name != in_nodes1.layer_name:
-                    is_prelu = False
-                    continue
-
-                if len(in_nodes0[0].outputs) != 1:
-                    is_prelu = false
-                    continue
-                if len(in_nodes0[1].outputs) != 1:
-                    is_prelu = False
-                    continue
-                if len(in_nodes1.outputs) < 3:
-                    is_prelu = False
-                    continue
-                if len(in_nodes2[0].outputs) != 1:
-                    is_prelu = false
-                    continue
-                if len(in_nodes2[1].outputs) != 1:
-                    is_prelu = False
-                    continue
-                if len(in_nodes3[0].outputs) != 1:
-                    is_prelu = False
-                    continue
-                if len(in_nodes3[1].outputs) != 1:
-                    is_prelu = false
-                    continue
-                if len(in_nodes4[1].outputs) != 1:
-                    is_prelu = False
-                    continue
-
-                mode = None
-                in_shape = in_nodes1.out_shapes[0]
-                if in_shape == list(in_nodes3[0].value.shape):
-                    mode = "element"
-                elif len(in_nodes3[0].value.shape) == 0:
-                    mode = "all"
-                elif len(in_nodes3[0].value.shape) == 1 and in_nodes3[
-                        0].value.shape[0] == 1:
-                    mode = "all"
-                elif len(in_shape) == 4 and len(in_nodes3[
-                        0].value.shape) == 1 and in_nodes3[0].value.shape[
-                            0] == in_shape[-1]:
-                    mode = "channel"
-                    weight = self.op_mapper.weights[in_nodes3[0].layer_name]
-                    weight = numpy.expand_dims(weight, 0)
-                    weight = numpy.expand_dims(weight, 2)
-                    weight = numpy.expand_dims(weight, 3)
-                    self.op_mapper.weights[in_nodes3[0].layer_name] = weight
-                    # fix bug in Paddle1.8.3 and may change in next version.
-                    # self.op_mapper.weights[in_nodes3[0].layer_name +
-                    #                        '_1'] = weight.reshape(1, -1)
-                    in_nodes3[0].fluid_code.layers[0].param_attr["shape"] = [
-                        1, in_shape[-1], 1, 1
-                    ]
-                else:
-                    is_prelu = False
-                    continue
-
-                if is_prelu:
-                    index = in_nodes1.outputs.index(in_nodes0[0].layer_name)
-                    del in_nodes1.outputs[index]
-                    index = in_nodes1.outputs.index(in_nodes3[1].layer_name)
-                    del in_nodes1.outputs[index]
-                    index = in_nodes1.outputs.index(in_nodes4[1].layer_name)
-                    del in_nodes1.outputs[index]
-                    in_nodes1.outputs.append(node.layer_name)
-
-                    node.layer_type = "Prelu"
-                    node.inputs = [in_nodes1.layer_name]
-                    act = node.fluid_code.layers[-1].param_attr.get("act", None)
-                    node.fluid_code.clear()
-                    attr = {
-                        "mode": string(mode),
-                        "param_attr": string(in_nodes3[0].layer_name)
-                    }
-
-                    node.fluid_code.add_layer(
-                        "prelu",
-                        inputs=in_nodes1.fluid_code.layers[-1].output,
-                        output=node,
-                        param_attr=attr)
-                del self.graph.node_map[in_nodes0[0].layer_name]
-                del self.graph.node_map[in_nodes0[1].layer_name]
-                del self.graph.node_map[in_nodes2[0].layer_name]
-                del self.graph.node_map[in_nodes2[1].layer_name]
-                del self.graph.node_map[in_nodes3[1].layer_name]
-                del self.graph.node_map[in_nodes4[1].layer_name]
-
-    def merge_scale(self):
-        for i, name in enumerate(self.graph.topo_sort):
-            node = self.graph.get_node(name)
-            if node is None:
-                continue
-            is_scale = True
-            if node.layer_type == "Sub":
-                in_nodes0 = [
-                    self.graph.get_node(in_name) for in_name in node.inputs
-                ]
-                if in_nodes0[0].layer_type != "Mul" or in_nodes0[
-                        1].layer_type != "Const" or in_nodes0[
-                            1].value.size != 1:
-                    is_scale = False
-                    continue
-                if exist_act(in_nodes0[0]):
-                    is_scale = False
-                    continue
-                if len(in_nodes0[0].outputs) != 1 or len(in_nodes0[1]
-                                                         .outputs) != 1:
-                    is_scale = False
-                    continue
-
-                in_nodes1 = [
-                    self.graph.get_node(in_name)
-                    for in_name in in_nodes0[0].inputs
-                ]
-                if in_nodes1[0].layer_type != "Const" or in_nodes1[
-                        1].layer_type != "RealDiv" or in_nodes1[
-                            0].value.size != 1:
-                    is_scale = False
-                    continue
-                if exist_act(in_nodes1[1]):
-                    is_scale = False
-                    continue
-                if len(in_nodes1[0].outputs) != 1 or len(in_nodes1[1]
-                                                         .outputs) != 1:
-                    is_scale = False
-                    continue
-
-                in_nodes2 = [
-                    self.graph.get_node(in_name)
-                    for in_name in in_nodes1[1].inputs
-                ]
-                if in_nodes2[1].layer_type != "Const" or in_nodes2[
-                        1].value.size != 1:
-                    is_scale = False
-                    continue
-
-                if is_scale:
-                    in_node = self.graph.get_node(in_nodes1[1].inputs[0])
-                    index = in_node.outputs.index(in_nodes1[1].layer_name)
-                    in_node.outputs[index] = node.layer_name
-                    node.layer_type = "Scale"
-                    node.inputs = [in_node.layer_name]
-                    scale = 1.0 / in_nodes2[1].value * in_nodes1[0].value
-                    act = None
-                    if node.fluid_code.layers[0].param_attr is not None:
-                        act = node.fluid_code.layers[0].param_attr.get("act",
-                                                                       None)
-                    node.fluid_code.clear()
-
-                    attr = {
-                        "scale": scale,
-                        "bias": in_nodes0[1].value,
-                        "bias_after_scale": True,
-                        "act": act
-                    }
-                    node.fluid_code.add_layer(
-                        "scale", inputs=in_node, output=node, param_attr=attr)
-
-                    del self.graph.node_map[in_nodes0[0].layer_name]
-                    del self.graph.node_map[in_nodes0[1].layer_name]
-                    del self.graph.node_map[in_nodes1[0].layer_name]
-                    del self.graph.node_map[in_nodes1[1].layer_name]
-                    del self.graph.node_map[in_nodes2[1].layer_name]
-
-    def merge_affine_channel(self):
-        for i, name in enumerate(self.graph.topo_sort):
-            node = self.graph.get_node(name)
-            if node is None:
-                continue
-            is_affine_channel = True
-            if node.layer_type == "RealDiv":
-                in_nodes0 = [
-                    self.graph.get_node(in_name) for in_name in node.inputs
-                ]
-                bias_add = True
-                if (in_nodes0[0].layer_type != "Sub" and in_nodes0[0].layer_type
-                        != "Add") or in_nodes0[1].layer_type != "Const" or len(
-                            in_nodes0[1].value.shape) != 3:
-                    is_affine_channel = False
-                    continue
-                if in_nodes0[0].layer_type == "Sub":
-                    bias_add = False
-                if exist_act(in_nodes0[0]):
-                    is_affine_channel = False
-                    continue
-                if len(in_nodes0[0].outputs) != 1 or len(in_nodes0[1]
-                                                         .outputs) != 1:
-                    is_affine_channel = False
-                    continue
-                in_nodes1 = [
-                    self.graph.get_node(in_name)
-                    for in_name in in_nodes0[0].inputs
-                ]
-                if len(in_nodes1[0].out_shapes[0]) != 4 or in_nodes1[
-                        1].layer_type != "Const" or len(in_nodes1[1]
-                                                        .value.shape) != 3:
-                    is_affine_channel = False
-                    continue
-                if len(in_nodes1[1].outputs) != 1:
-                    is_affine_channel = False
-                    continue
-                channel = in_nodes1[0].out_shapes[0][-1]
-                if channel < 0 or channel != in_nodes0[
-                        1].value.size or channel != in_nodes1[1].value.size:
-                    is_affine_channel = False
-                    continue
-                if in_nodes0[1].out_shapes[0][-1] != in_nodes0[
-                        1].value.size or in_nodes1[1].out_shapes[0][
-                            -1] != in_nodes1[1].value.size:
-                    is_affine_channel = False
-                    continue
-                if is_affine_channel:
-                    in_node = in_nodes1[0]
-                    index = in_node.outputs.index(in_nodes0[0].layer_name)
-                    in_node.outputs[index] = node.layer_name
-                    node.layer_type = "AffineChannel"
-                    node.inputs = [in_node.layer_name]
-                    scale = 1.0 / in_nodes0[1].value.flatten()
-                    bias = in_nodes1[1].value.flatten() / in_nodes0[
-                        1].value.flatten()
-                    if not bias_add:
-                        bias *= -1.0
-                    self.op_mapper.weights[node.layer_name + "_scale"] = scale
-                    self.op_mapper.weights[node.layer_name + "_bias"] = bias
-
-                    act = None
-                    if node.fluid_code.layers[0].param_attr is not None:
-                        act = node.fluid_code.layers[0].param_attr.get("act",
-                                                                       None)
-                    node.fluid_code.clear()
-
-                    attr = {
-                        "dtype": string(scale.dtype),
-                        "shape": [channel],
-                        "name": string(node.layer_name + "_scale")
-                    }
-                    node.fluid_code.add_layer(
-                        "create_parameter",
-                        inputs=None,
-                        output=node.layer_name + "_scale",
-                        param_attr=attr)
-                    attr = {
-                        "dtype": string(scale.dtype),
-                        "shape": [channel],
-                        "name": string(node.layer_name + "_bias")
-                    }
-                    node.fluid_code.add_layer(
-                        "create_parameter",
-                        inputs=None,
-                        output=node.layer_name + "_bias",
-                        param_attr=attr)
-                    inputs = {
-                        "x": in_node,
-                        "scale": node.layer_name + "_scale",
-                        "bias": node.layer_name + "_bias"
-                    }
-                    attr = {"act": act}
-                    node.fluid_code.add_layer(
-                        "affine_channel",
-                        inputs=inputs,
-                        output=node,
-                        param_attr=attr)
-
-                    del self.graph.node_map[in_nodes0[0].layer_name]
-                    del self.graph.node_map[in_nodes0[1].layer_name]
-                    del self.graph.node_map[in_nodes1[1].layer_name]