# Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License" # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # TODO useless node remove from x2paddle.op_mapper.tf_op_mapper import TFOpMapper from x2paddle.core.fluid_code import Layer from x2paddle.core.util import * import copy as cp class TFOptimizer(object): activation_ops = { 'Relu': 'relu', 'Sigmoid': 'sigmoid', 'Relu6': 'relu6', 'swish_f32': 'swish' } layers_with_act = [ 'Conv2D', 'BiasAdd', 'DepthwiseConv2dNative', 'Conv2DBackpropInput', 'FusedBatchNorm', 'conv2d', 'elementwise_add', 'conv2d_transpose', 'batch_norm' ] layers_with_bias = [ 'Conv2D', 'DepthwiseConv2dNative', 'Conv2DBackpropInput', 'conv2d', 'conv2d_transpose' ] def __init__(self, op_mapper): self.op_mapper = op_mapper self.graph = op_mapper.graph def delete_redundance_code(self): for node_name in self.graph.topo_sort: if node_name in self.op_mapper.omit_nodes: node = self.graph.get_node(node_name) if node is None: continue omit_freq = self.op_mapper.omit_nodes.count(node_name) if len(node.outputs) <= omit_freq: node.fluid_code.clear() # remove node from graph input_names = node.inputs output_names = node.outputs for in_name in input_names: in_node = self.graph.get_node(in_name) index = in_node.outputs.index(node_name) del in_node.outputs[index] for out_name in output_names: out_node = self.graph.get_node(out_name) index = out_node.inputs.index(node_name) del out_node.inputs[index] del self.graph.node_map[node_name] def strip_graph(self): visited_nodes = set() def visit(node_name): if node_name in visited_nodes: return visited_nodes.add(node_name) input_names = self.graph.get_node(node_name).inputs for in_name in input_names: visit(in_name) for node_name in self.graph.output_nodes: visit(node_name) for i, node_name in enumerate(self.graph.topo_sort): if node_name not in visited_nodes: node = self.graph.get_node(node_name) if node is None: continue input_names = node.inputs output_names = node.outputs for in_name in input_names: in_node = self.graph.get_node(in_name) index = in_node.outputs.index(node_name) del in_node.outputs[index] for out_name in output_names: out_node = self.graph.get_node(out_name) index = out_node.inputs.index(node_name) del out_node.inputs[index] del self.graph.node_map[node_name] def merge_activation(self): act_nodes = list() for node_name in self.graph.topo_sort: node = self.graph.get_node(node_name) if node is None: continue if node.layer_type in self.activation_ops: act_nodes.append(node_name) for act_node_name in act_nodes: node = self.graph.get_node(act_node_name) input = self.graph.get_node(node.inputs[0]) if input.layer_type not in self.layers_with_act: continue if len(input.fluid_code.layers) == 0: continue if 'act' in input.fluid_code.layers[ -1].param_attr and input.fluid_code.layers[-1].param_attr[ 'act'] is not None: continue if len(input.outputs) != 1: continue index = -1 for i in range(len(input.fluid_code.layers)): if input.fluid_code.layers[i].op in self.layers_with_act: index = i break input.fluid_code.layers[index].param_attr['act'] = string( self.activation_ops[node.layer_type]) input.fluid_code.layers[-1].output = node.fluid_code.layers[ 0].output self.graph.remove_node(act_node_name) def merge_bias(self): for node_name in self.graph.topo_sort: node = self.graph.get_node(node_name) if node is None: continue if node.layer_type == "BiasAdd": input = self.graph.get_node(node.inputs[0]) if input.layer_type not in self.layers_with_bias: continue if len(input.outputs) != 1: continue if len(input.fluid_code.layers) == 0: continue bias_with_act = False if 'act' in node.fluid_code.layers[-1].param_attr: bias_with_act = True layer_with_act = False index = -1 for i in range(len(input.fluid_code.layers)): if input.fluid_code.layers[i].op in self.layers_with_bias: index = i break if 'act' in input.fluid_code.layers[ index].param_attr and input.fluid_code.layers[ index].param_attr['act'] is not None: layer_with_act = True if bias_with_act and layer_with_act: continue if not input.fluid_code.layers[index].param_attr['bias_attr']: bias_name = node.inputs[1] input.fluid_code.layers[index].param_attr[ 'bias_attr'] = string(bias_name) input.fluid_code.layers[-1].output = node.fluid_code.layers[ 0].output if bias_with_act: input.fluid_code.layers[index].param_attr[ 'act'] = node.fluid_code.layers[-1].param_attr[ 'act'] node.fluid_code.clear() self.graph.remove_node(node.layer_name) def remove_transpose(self): graph_copy = cp.deepcopy(self.graph) nhwc_insensitive_ops = [ 'Relu', 'Relu6', 'Abs', 'Sigmoid', 'Exp', 'Rsqrt', 'swish_f32', 'LeakyRelu', 'Cast' ] elementwise_ops = [ 'Sub', 'Add', 'RealDiv', 'Maximum', 'Mul', 'FloorDiv', 'GreaterEqual' ] for node_name in self.graph.topo_sort: node = graph_copy.get_node(node_name) if node is None: continue if node.layer_type in nhwc_insensitive_ops: graph_copy.remove_node(node_name) optimize_ops = [ 'Conv2D', 'MaxPool', 'FusedBatchNorm', 'DepthwiseConv2dNative', 'AvgPool', 'Pad', 'Conv2DBackpropInput', 'ResizeNearestNeighbor', 'ResizeBilinear', "Placeholder" ] for node_name in self.graph.topo_sort: node = graph_copy.get_node(node_name) if node is None: continue if node.layer_type in elementwise_ops: is_nhwc = True for in_name in node.inputs: in_node = graph_copy.get_node(in_name) if hasattr(in_node, "is_nhwc"): if not in_node.is_nhwc: is_nhwc = False else: if len(in_node.fluid_code.layers) < 2: is_nhwc = False continue if in_node.fluid_code.layers[ -1].op != "transpose" or in_node.fluid_code.layers[ -1].param_attr["perm"] != [0, 2, 3, 1]: is_nhwc = False continue node.is_nhwc = is_nhwc for i in range(len(self.graph.topo_sort)): node_name = self.graph.topo_sort[-1 * i - 1] node = graph_copy.get_node(node_name) if node is None: continue if node.layer_type in elementwise_ops: can_be_removed = True if len(node.fluid_code.layers) > 1: can_be_removed = False if not node.is_nhwc: can_be_removed = False for out_name in node.outputs: out_node = graph_copy.get_node(out_name) if hasattr(out_node, "is_nhwc"): if not out_node.is_nhwc: can_be_removed = False else: if len(out_node.fluid_code.layers) < 2: can_be_removed = False break if out_node.fluid_code.layers[ 0].op != "transpose" or out_node.fluid_code.layers[ 0].param_attr["perm"] != [0, 3, 1, 2]: can_be_removed = False break node.can_be_removed = can_be_removed for node_name in self.graph.topo_sort: node = graph_copy.get_node(node_name) if node is None: continue if node.layer_type in optimize_ops: if node.fluid_code.layers[ -1].op != "transpose" or node.fluid_code.layers[ -1].param_attr["perm"] != [0, 2, 3, 1]: continue can_be_removed = True output_names = node.outputs for out_name in output_names: out_node = graph_copy.get_node(out_name) if hasattr(out_node, "can_be_removed"): if not out_node.can_be_removed: can_be_removed = False break elif out_node.fluid_code.layers[ 0].op != "transpose" or out_node.fluid_code.layers[ 0].param_attr["perm"] != [0, 3, 1, 2]: can_be_removed = False break if can_be_removed and len(node.fluid_code.layers) > 1: true_node = self.graph.get_node(node_name) if true_node.layer_type == "Placeholder": index = self.graph.input_nodes.index( true_node.fluid_code.layers[-2].output) if isinstance(true_node.fluid_code.layers[-1].output, str): self.graph.input_nodes[ index] = true_node.fluid_code.layers[-1].output else: self.graph.input_nodes[ index] = true_node.fluid_code.layers[ -1].output.layer_name true_node.fluid_code.layers[ -2].output = true_node.fluid_code.layers[-1].output node.removed = True del true_node.fluid_code.layers[-1] for out_name in output_names: out_node = self.graph.get_node(out_name) if out_node.layer_type in elementwise_ops: continue out_node.fluid_code.layers[ 1].inputs = out_node.fluid_code.layers[0].inputs del out_node.fluid_code.layers[0] for node_name in self.graph.topo_sort: node = graph_copy.get_node(node_name) if node is None: continue if node.layer_type in elementwise_ops: if not node.can_be_removed: true_node = self.graph.get_node(node_name) for i, in_name in enumerate(node.inputs): in_node = graph_copy.get_node(in_name) if hasattr(in_node, "is_nhwc") and in_node.is_nhwc: if i == 0: l = Layer() l.op = "transpose" l.inputs = true_node.fluid_code.layers[ 0].inputs["x"] l.param_attr = {"perm": [0, 2, 3, 1]} l.output = "nhwc_" + l.inputs.layer_name true_node.fluid_code.layers[0].inputs[ "x"] = l.output true_node.fluid_code.layers.insert(0, l) elif i == 1: l = Layer() l.op = "transpose" l.inputs = true_node.fluid_code.layers[ 0].inputs["y"] l.param_attr = {"perm": [0, 2, 3, 1]} l.output = "nhwc_" + l.inputs.layer_name true_node.fluid_code.layers[0].inputs[ "y"] = l.output true_node.fluid_code.layers.insert(0, l) else: raise Exception("Unexpected situation happend") continue else: for out_name in node.outputs: out_node = self.graph.get_node(out_name) if out_node.layer_type not in elementwise_ops: assert out_node.fluid_code.layers[ 0].op == "transpose", "unexpected situation happend" out_node.fluid_code.layers[ 1].inputs = out_node.fluid_code.layers[0].inputs del out_node.fluid_code.layers[0] def make_nchw_input_output(self): for i, name in enumerate(self.graph.input_nodes): node = self.graph.get_node(name) if len(node.out_shapes[0]) == 4 and node.tf_data_format == "NHWC": shape = node.fluid_code.layers[0].param_attr["shape"] shape = [shape[i] for i in [0, 3, 1, 2]] node.fluid_code.layers[0].param_attr["shape"] = shape node.fluid_code.layers[0].output = "nhwc_" + name attr = {"perm": [0, 2, 3, 1]} node.fluid_code.add_layer("transpose", inputs="nhwc_" + name, output=node, param_attr=attr) self.graph.input_nodes[i] = "nhwc_" + name for i, name in enumerate(self.graph.output_nodes): node = self.graph.get_node(name) if node.layer_type != "transpose": if node.fluid_code.layers[-1].op == "transpose": node.fluid_code.layers[-2].output = name del node.fluid_code.layers[-1]