From 79ba1760e4d1d4e13dc620af54b63f9c83047d3c Mon Sep 17 00:00:00 2001 From: liuruilong Date: Mon, 4 Jun 2018 18:56:27 +0800 Subject: [PATCH] add conv add fusion op --- src/common/types.h | 5 +- src/framework/operator.h | 4 +- .../program/program-optimize/node.cpp | 12 ++-- src/framework/program/program-optimize/node.h | 5 +- .../program-optimize/program_optimize.cpp | 35 ++++++++-- src/io.cpp | 2 + src/io.h | 2 +- src/operators/fusion_conv_add.cpp | 29 ++++++++ src/operators/fusion_conv_add.h | 66 +++++++++++++++++++ src/operators/fusion_conv_add_relu_op.h | 4 +- src/operators/fusion_fc_op.h | 6 +- test/framework/test_load.cpp | 4 +- test/framework/test_optimize.cpp | 6 +- test/net/test_mobilenet+ssd.cpp | 4 +- test/test_helper.h | 5 +- 15 files changed, 158 insertions(+), 31 deletions(-) create mode 100644 src/operators/fusion_conv_add.cpp create mode 100644 src/operators/fusion_conv_add.h diff --git a/src/common/types.h b/src/common/types.h index 04b78947a6..cddd96f18f 100644 --- a/src/common/types.h +++ b/src/common/types.h @@ -80,6 +80,7 @@ static const std::string G_OP_TYPE_ELEMENTWISE_ADD = "elementwise_add"; static const std::string G_OP_TYPE_FUSION_CONV_ADD_RELU = "fusion_conv_add_relu"; static const std::string G_OP_TYPE_FC = "fc"; +static const std::string G_OP_TYPE_CONV_ADD = "conv_add"; static const std::string G_OP_TYPE_LRN = "lrn"; static const std::string G_OP_TYPE_MUL = "mul"; static const std::string G_OP_TYPE_MULTICLASS_NMS = "multiclass_nms"; @@ -115,5 +116,7 @@ static std::unordered_map< {{"PriorBox", "PriorBoxVar", "TargetBox"}, {"OutputBox"}}}, {G_OP_TYPE_PRIOR_BOX, {{"Image", "Input"}, {"Boxes", "Variances"}}}, {G_OP_TYPE_MULTICLASS_NMS, {{"BBoxes", "Scores"}, {"Out"}}}, - {G_OP_TYPE_RESHAPE, {{"X"}, {"Out"}}}}; + {G_OP_TYPE_RESHAPE, {{"X"}, {"Out"}}}, + {G_OP_TYPE_DEPTHWISE_CONV, {{"Input"}, {"Output"}}} +}; } // namespace paddle_mobile diff --git a/src/framework/operator.h b/src/framework/operator.h index cb27985244..ffc23eaff6 100644 --- a/src/framework/operator.h +++ b/src/framework/operator.h @@ -145,8 +145,8 @@ class FusionOpMatcher : PaddleMobileObject { virtual std::string Type() = 0; - virtual void FolderNodes(Node *node) { - node->Folder(node_.Depth(), Type(), {}); + virtual void FolderNodes(Node *node, std::vector> *removed_nodes) { + node->Folder(node_.Depth(), Type(), {}, removed_nodes); } virtual Node &BeginNode() { return node_; } diff --git a/src/framework/program/program-optimize/node.cpp b/src/framework/program/program-optimize/node.cpp index c165b6568a..05d30143ec 100644 --- a/src/framework/program/program-optimize/node.cpp +++ b/src/framework/program/program-optimize/node.cpp @@ -236,12 +236,13 @@ uint Node::Depth(uint begin) { Node &Node::Folder( uint size, std::string type, - std::map> change) { + std::map> change, + std::vector> *removed_nodes) { std::shared_ptr op_desc = std::make_shared(); op_desc->inputs_ = this->op_desc_->inputs_; std::vector> outputs; - this->Folder(op_desc, &outputs, size - 1, &change, this); + this->Folder(op_desc, &outputs, size - 1, &change, this, removed_nodes); this->outputs_ = outputs; this->type_ = type; this->op_desc_ = op_desc; @@ -253,7 +254,8 @@ void Node::Folder( std::shared_ptr op_desc, std::vector> *outputs, uint index, std::map> *change, - Node *begin_node) { + Node *begin_node, + std::vector> *removed_nodes) { if (change->find(this->type_) != change->end()) { auto change_pair = (*change)[this->type_]; op_desc->GetInputs()[change_pair.second] = @@ -266,7 +268,8 @@ void Node::Folder( if (index > 0) { --index; for (auto output : outputs_) { - output->Folder(op_desc, outputs, index, change, begin_node); + removed_nodes->push_back(output); + output->Folder(op_desc, outputs, index, change, begin_node, removed_nodes); } } else { for (auto &op_output : this->op_desc_->outputs_) { @@ -279,6 +282,7 @@ void Node::Folder( if (iter != output->inputs_.end()) { output->inputs_.erase(iter); + } output->inputs_.push_back(begin_node); outputs->push_back(output); diff --git a/src/framework/program/program-optimize/node.h b/src/framework/program/program-optimize/node.h index 8ef26f897d..444fe6c3dd 100644 --- a/src/framework/program/program-optimize/node.h +++ b/src/framework/program/program-optimize/node.h @@ -43,7 +43,7 @@ class Node : PaddleMobileObject { uint Depth(uint begin = 0); Node &Folder( uint size, std::string type, - std::map> change_map); + std::map> change_map, std::vector> *removed_nodes); std::vector> OpDescs(uint size); std::vector> OpDescs(); std::shared_ptr OpDescOfNode() { return op_desc_; } @@ -63,7 +63,8 @@ class Node : PaddleMobileObject { std::shared_ptr op_desc, std::vector> *outputs, uint index, std::map> *change, - Node *begin_node); + Node *begin_node, + std::vector> *removed_nodes); std::shared_ptr op_desc_; std::string ToString(std::string blank, const Node *node) const; std::vector> outputs_; diff --git a/src/framework/program/program-optimize/program_optimize.cpp b/src/framework/program/program-optimize/program_optimize.cpp index d9c3c51c3c..88d6b1d47a 100644 --- a/src/framework/program/program-optimize/program_optimize.cpp +++ b/src/framework/program/program-optimize/program_optimize.cpp @@ -31,6 +31,9 @@ std::shared_ptr ProgramOptimize::FushionOptimize( std::unordered_map>> type_map; + + std::vector> nodes; + std::shared_ptr begin_node; auto block = optimize_program->Block(i); // DLOG << " ops size: " << block->Ops().size(); @@ -38,11 +41,12 @@ std::shared_ptr ProgramOptimize::FushionOptimize( auto op = block->Ops()[j]; auto op_type = op->Type(); if (op_input_output_key.find(op->Type()) == op_input_output_key.end()) { - LOG(kLOG_ERROR) << "return null "; + LOG(kLOG_ERROR) << "has not support op return null " << " op type: " << op->Type(); return nullptr; } std::shared_ptr node = std::make_shared(op); + nodes.push_back(node); // type_map[op->Type()].push_back(node); @@ -87,21 +91,29 @@ std::shared_ptr ProgramOptimize::FushionOptimize( // DLOG << " match success " << " fusion node: \n" << // matcher->BeginNode() << "\nsub node: \n" << *sub_node; // DLOG << "match node\n"<< *match_node; - matcher->FolderNodes(match_node.get()); - // DLOG << " after match node\n"<< *match_node; - // match_node->Description(); - // DLOG << "begin node: \n" << *begin_node; + std::vector> removed_nodes; + matcher->FolderNodes(match_node.get(), &removed_nodes); + + for (int j = 0; j < removed_nodes.size(); ++j) { + auto removed_node = removed_nodes[j]; + auto removed_ite = std::find(nodes.begin(), nodes.end(), removed_node); + nodes.erase(removed_ite); + } } } } - // DLOG << "node: \n" << *begin_node; +// DLOG << "node: \n" << *begin_node; std::vector> op_descs; // bool can_splite = begin_node->CanSplit({G_OP_TYPE_CONV, // G_OP_TYPE_BATCHNORM, G_OP_TYPE_DEPTHWISE_CONV}); - GenerateOps(&op_descs, begin_node.get()); + for (int m = 0; m < nodes.size(); ++m) { + auto &node = nodes[m]; + op_descs.push_back(node->op_desc_); + } +// GenerateOps(&op_descs, begin_node.get()); block->ops_ = op_descs; } @@ -116,8 +128,17 @@ std::shared_ptr ProgramOptimize::FushionOptimize( void ProgramOptimize::GenerateOps( std::vector> *op_desc, Node *input_node, Node *current_node) { + if (current_node->inputs_.size() > 1 && input_node != current_node->inputs_.back()) { + DLOG << " current type " << current_node->type_; + + DLOG << " inputs size of current node > 0 "; + + for (int i = 0; i < current_node->inputs_.size(); ++i) { + DLOG << " input i: " << current_node->inputs_[i]->type_; + } + return; } else if (current_node->inputs_.size() > 1 && input_node == current_node->inputs_.back()) { diff --git a/src/io.cpp b/src/io.cpp index 8f6a07f2dd..e208bc71c7 100644 --- a/src/io.cpp +++ b/src/io.cpp @@ -221,6 +221,8 @@ const framework::Program Loader::Load( } } + // originProgramDesc->Description("program: "); + if (optimize) { framework::ProgramOptimize program_optimize; program.optimizeProgram = diff --git a/src/io.h b/src/io.h index ae99197baa..bb36385154 100644 --- a/src/io.h +++ b/src/io.h @@ -32,7 +32,7 @@ template class Loader : PaddleMobileObject { public: const framework::Program Load(const std::string &dirname, - bool optimize = true); + bool optimize = false); private: void LoadVar(framework::Variable *variable, diff --git a/src/operators/fusion_conv_add.cpp b/src/operators/fusion_conv_add.cpp new file mode 100644 index 0000000000..ffddd25dff --- /dev/null +++ b/src/operators/fusion_conv_add.cpp @@ -0,0 +1,29 @@ +/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include "operators/fusion_conv_add.h" +namespace paddle_mobile { +namespace operators { + +template +void FushionConvAddOp::InferShape() const { + +} +template class FushionConvAddOp; +} // namespace operators +} // namespace paddle_mobile + +namespace ops = paddle_mobile::operators; +USE_OP(conv_add); +REGISTER_OPERATOR(conv_add, ops::FushionConvAddOp); diff --git a/src/operators/fusion_conv_add.h b/src/operators/fusion_conv_add.h new file mode 100644 index 0000000000..92ed236d8b --- /dev/null +++ b/src/operators/fusion_conv_add.h @@ -0,0 +1,66 @@ +/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#pragma once + +#include +#include + +#include "framework/operator.h" +#include "framework/program/program-optimize/fusion_op_register.h" + +namespace paddle_mobile { +namespace operators { +using std::string; +using std::vector; +class FusionConvAddMatcher : public framework::FusionOpMatcher { + public: + FusionConvAddMatcher() { + node_ = framework::Node(G_OP_TYPE_CONV); + node_ > std::make_shared(G_OP_TYPE_ELEMENTWISE_ADD); + } + + void FolderNodes(framework::Node *node, std::vector> *removed_nodes) { + vector> origin_descs = + node->OpDescs(node_.Depth()); + node->Folder(node_.Depth(), Type(), {{G_OP_TYPE_ELEMENTWISE_ADD, {"Y", "Y"}}}, removed_nodes); + } + + std::string Type() { return G_OP_TYPE_CONV_ADD; } +}; + +template +class FushionConvAddOp : public framework::OperatorWithKernel { + public: + FushionConvAddOp(const string &type, const VariableNameMap &inputs, + const VariableNameMap &outputs, + const framework::AttributeMap attrs, + std::shared_ptr scope) + : framework::OperatorWithKernel(type, inputs, outputs, attrs, + scope) {} + + void RunImpl() const { + } + + using framework::OperatorWithKernel::OperatorWithKernel; + void InferShape() const override; + + protected: + // FushionFcParam param_; +}; + +//static framework::FusionOpRegistrar fc_registrar(new FusionConvAddMatcher()); + +} // namespace operators +} // namespace paddle_mobile diff --git a/src/operators/fusion_conv_add_relu_op.h b/src/operators/fusion_conv_add_relu_op.h index 2b7f0274f6..9fc9fdee5e 100644 --- a/src/operators/fusion_conv_add_relu_op.h +++ b/src/operators/fusion_conv_add_relu_op.h @@ -28,11 +28,11 @@ class FushionConvAddReluOpMatcher : public framework::FusionOpMatcher { std::make_shared(G_OP_TYPE_RELU); } - void FolderNodes(framework::Node *node) { + void FolderNodes(framework::Node *node, std::vector> *removed_nodes) { std::vector> origin_descs = node->OpDescs(node_.Depth()); node->Folder(node_.Depth(), Type(), - {{G_OP_TYPE_ELEMENTWISE_ADD, {"Y", "Z"}}}); + {{G_OP_TYPE_ELEMENTWISE_ADD, {"Y", "Z"}}}, removed_nodes); } std::string Type() { return G_OP_TYPE_FUSION_CONV_ADD_RELU; } }; diff --git a/src/operators/fusion_fc_op.h b/src/operators/fusion_fc_op.h index 9019ef4d49..a42d100f92 100644 --- a/src/operators/fusion_fc_op.h +++ b/src/operators/fusion_fc_op.h @@ -32,11 +32,11 @@ class FusionFcMatcher : public framework::FusionOpMatcher { node_ > std::make_shared(G_OP_TYPE_ELEMENTWISE_ADD); } - void FolderNodes(framework::Node *node) { + void FolderNodes(framework::Node *node, std::vector> *removed_nodes) { vector> origin_descs = node->OpDescs(node_.Depth()); node->Folder(node_.Depth(), Type(), - {{G_OP_TYPE_ELEMENTWISE_ADD, {"Y", "Z"}}}); + {{G_OP_TYPE_ELEMENTWISE_ADD, {"Y", "Z"}}}, removed_nodes); } std::string Type() { return G_OP_TYPE_FC; } @@ -65,7 +65,7 @@ class FushionFcOp : public framework::OperatorWithKernel { FushionFcParam param_; }; -static framework::FusionOpRegistrar fc_registrar(new FusionFcMatcher()); +//static framework::FusionOpRegistrar fc_registrar(new FusionFcMatcher()); } // namespace operators } // namespace paddle_mobile diff --git a/test/framework/test_load.cpp b/test/framework/test_load.cpp index 95357547e1..45d7324bf5 100644 --- a/test/framework/test_load.cpp +++ b/test/framework/test_load.cpp @@ -19,7 +19,7 @@ int main() { paddle_mobile::Loader loader; // ../../../test/models/googlenet // ../../../test/models/mobilenet - auto program = loader.Load(g_googlenet); - program.optimizeProgram->Description("program desc: "); + auto program = loader.Load(g_mobilenet_ssd, true); + program.originProgram->Description("program desc: "); return 0; } diff --git a/test/framework/test_optimize.cpp b/test/framework/test_optimize.cpp index faa69847c6..8915941562 100644 --- a/test/framework/test_optimize.cpp +++ b/test/framework/test_optimize.cpp @@ -12,20 +12,20 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ +#include "io.h" #include "../test_helper.h" #include "framework/program/program-optimize/node.h" #include "framework/program/program-optimize/program_optimize.h" -#include "io.h" int main() { paddle_mobile::Loader loader; // "../../../test/models/googlenet" - auto program = loader.Load(g_mobilenet_ssd); + auto program = loader.Load(g_mobilenet_ssd, true); paddle_mobile::framework::ProgramOptimize optimize; // program.originProgram->Description("origin"); auto optimize_program = optimize.FushionOptimize(program.originProgram); if (optimize_program != nullptr) { - optimize_program->Description("optimize"); +// optimize_program->Description("optimize"); } else { LOG(paddle_mobile::kLOG_ERROR) << "optimize_program is null"; } diff --git a/test/net/test_mobilenet+ssd.cpp b/test/net/test_mobilenet+ssd.cpp index e9d92e7a51..097d03ad71 100644 --- a/test/net/test_mobilenet+ssd.cpp +++ b/test/net/test_mobilenet+ssd.cpp @@ -19,10 +19,10 @@ limitations under the License. */ int main() { paddle_mobile::Loader loader; auto time1 = time(); - auto program = loader.Load(g_mobilenet_ssd, false); + auto program = loader.Load(g_mobilenet_ssd, true); auto time2 = time(); DLOG << "load cost :" << time_diff(time1, time1) << "ms"; - paddle_mobile::Executor executor(program, 1, false); + paddle_mobile::Executor executor(program, 1, true); std::vector dims{1, 3, 300, 300}; Tensor input_tensor; diff --git a/test/test_helper.h b/test/test_helper.h index 14ade768e7..3b77f0052c 100644 --- a/test/test_helper.h +++ b/test/test_helper.h @@ -22,10 +22,11 @@ limitations under the License. */ #include "framework/ddim.h" #include "framework/tensor.h" -static const std::string g_googlenet = "../models/googlenet"; -static const std::string g_mobilenet = "../models/mobilenet"; static const std::string g_mobilenet_ssd = "../models/mobilenet+ssd"; static const std::string g_squeezenet = "../models/squeezenet"; +static const std::string g_googlenet = "../models/googlenet"; +static const std::string g_mobilenet = "../models/mobilenet"; +static const std::string g_resnet_50 = "../models/resnet_50"; static const std::string g_resnet = "../models/resnet"; static const std::string g_yolo = "../models/yolo"; static const std::string g_test_image_1x3x224x224 = -- GitLab