Merge pull request #352 from codeWorm2015/develop

fix #351  add fusion conv + add , add mali folder

src/common/types.h

@@ -80,6 +80,7 @@ static const std::string G_OP_TYPE_ELEMENTWISE_ADD = "elementwise_add";
 static const std::string G_OP_TYPE_FUSION_CONV_ADD_RELU =
     "fusion_conv_add_relu";
 static const std::string G_OP_TYPE_FC = "fc";
+static const std::string G_OP_TYPE_CONV_ADD = "conv_add";
 static const std::string G_OP_TYPE_LRN = "lrn";
 static const std::string G_OP_TYPE_MUL = "mul";
 static const std::string G_OP_TYPE_MULTICLASS_NMS = "multiclass_nms";
@@ -115,5 +116,6 @@ static std::unordered_map<
          {{"PriorBox", "PriorBoxVar", "TargetBox"}, {"OutputBox"}}},
         {G_OP_TYPE_PRIOR_BOX, {{"Image", "Input"}, {"Boxes", "Variances"}}},
         {G_OP_TYPE_MULTICLASS_NMS, {{"BBoxes", "Scores"}, {"Out"}}},
-        {G_OP_TYPE_RESHAPE, {{"X"}, {"Out"}}}};
+        {G_OP_TYPE_RESHAPE, {{"X"}, {"Out"}}},
+        {G_OP_TYPE_DEPTHWISE_CONV, {{"Input"}, {"Output"}}}};
 }  // namespace paddle_mobile

src/framework/operator.h

@@ -145,8 +145,10 @@ class FusionOpMatcher : PaddleMobileObject {
 
   virtual std::string Type() = 0;
 
-  virtual void FolderNodes(Node *node) {
-    node->Folder(node_.Depth(), Type(), {});
+  virtual void FolderNodes(
+      Node *node,
+      std::vector<std::shared_ptr<framework::Node>> *removed_nodes) {
+    node->Folder(node_.Depth(), Type(), {}, removed_nodes);
   }
 
   virtual Node &BeginNode() { return node_; }

src/framework/program/program-optimize/node.cpp

@@ -236,12 +236,13 @@ uint Node::Depth(uint begin) {
 
 Node &Node::Folder(
     uint size, std::string type,
-    std::map<std::string, std::pair<std::string, std::string>> change) {
+    std::map<std::string, std::pair<std::string, std::string>> change,
+    std::vector<std::shared_ptr<Node>> *removed_nodes) {
   std::shared_ptr<framework::OpDesc> op_desc =
       std::make_shared<framework::OpDesc>();
   op_desc->inputs_ = this->op_desc_->inputs_;
   std::vector<std::shared_ptr<Node>> outputs;
-  this->Folder(op_desc, &outputs, size - 1, &change, this);
+  this->Folder(op_desc, &outputs, size - 1, &change, this, removed_nodes);
   this->outputs_ = outputs;
   this->type_ = type;
   this->op_desc_ = op_desc;
@@ -253,7 +254,7 @@ void Node::Folder(
     std::shared_ptr<framework::OpDesc> op_desc,
     std::vector<std::shared_ptr<Node>> *outputs, uint index,
     std::map<std::string, std::pair<std::string, std::string>> *change,
-    Node *begin_node) {
+    Node *begin_node, std::vector<std::shared_ptr<Node>> *removed_nodes) {
   if (change->find(this->type_) != change->end()) {
     auto change_pair = (*change)[this->type_];
     op_desc->GetInputs()[change_pair.second] =
@@ -266,7 +267,9 @@ void Node::Folder(
   if (index > 0) {
     --index;
     for (auto output : outputs_) {
-      output->Folder(op_desc, outputs, index, change, begin_node);
+      removed_nodes->push_back(output);
+      output->Folder(op_desc, outputs, index, change, begin_node,
+                     removed_nodes);
     }
   } else {
     for (auto &op_output : this->op_desc_->outputs_) {

src/framework/program/program-optimize/node.h

@@ -43,7 +43,8 @@ class Node : PaddleMobileObject {
   uint Depth(uint begin = 0);
   Node &Folder(
       uint size, std::string type,
-      std::map<std::string, std::pair<std::string, std::string>> change_map);
+      std::map<std::string, std::pair<std::string, std::string>> change_map,
+      std::vector<std::shared_ptr<Node>> *removed_nodes);
   std::vector<std::shared_ptr<framework::OpDesc>> OpDescs(uint size);
   std::vector<std::shared_ptr<framework::OpDesc>> OpDescs();
   std::shared_ptr<framework::OpDesc> OpDescOfNode() { return op_desc_; }
@@ -63,7 +64,7 @@ class Node : PaddleMobileObject {
       std::shared_ptr<framework::OpDesc> op_desc,
       std::vector<std::shared_ptr<Node>> *outputs, uint index,
       std::map<std::string, std::pair<std::string, std::string>> *change,
-      Node *begin_node);
+      Node *begin_node, std::vector<std::shared_ptr<Node>> *removed_nodes);
   std::shared_ptr<framework::OpDesc> op_desc_;
   std::string ToString(std::string blank, const Node *node) const;
   std::vector<std::shared_ptr<Node>> outputs_;

src/framework/program/program-optimize/program_optimize.cpp

@@ -31,6 +31,8 @@ std::shared_ptr<ProgramDesc> ProgramOptimize::FushionOptimize(
     std::unordered_map<std::string, std::vector<std::shared_ptr<Node>>>
         type_map;
 
+    std::vector<std::shared_ptr<Node>> nodes;
+
     std::shared_ptr<Node> begin_node;
     auto block = optimize_program->Block(i);
     //        DLOG << " ops size: " << block->Ops().size();
@@ -38,11 +40,13 @@ std::shared_ptr<ProgramDesc> ProgramOptimize::FushionOptimize(
       auto op = block->Ops()[j];
       auto op_type = op->Type();
       if (op_input_output_key.find(op->Type()) == op_input_output_key.end()) {
-        LOG(kLOG_ERROR) << "return null ";
+        LOG(kLOG_ERROR) << "has not support op return null "
+                        << " op type: " << op->Type();
         return nullptr;
       }
 
       std::shared_ptr<Node> node = std::make_shared<Node>(op);
+      nodes.push_back(node);
 
       //
       type_map[op->Type()].push_back(node);
@@ -87,21 +91,30 @@ std::shared_ptr<ProgramDesc> ProgramOptimize::FushionOptimize(
           //          DLOG << " match success " << " fusion node: \n" <<
           //          matcher->BeginNode() << "\nsub node: \n" << *sub_node;
           //          DLOG << "match node\n"<< *match_node;
-          matcher->FolderNodes(match_node.get());
-          //          DLOG << " after match node\n"<< *match_node;
-          //          match_node->Description();
 
-          //          DLOG << "begin node: \n" << *begin_node;
+          std::vector<std::shared_ptr<Node>> removed_nodes;
+          matcher->FolderNodes(match_node.get(), &removed_nodes);
+
+          for (int j = 0; j < removed_nodes.size(); ++j) {
+            auto removed_node = removed_nodes[j];
+            auto removed_ite =
+                std::find(nodes.begin(), nodes.end(), removed_node);
+            nodes.erase(removed_ite);
+          }
         }
       }
     }
 
-    //    DLOG << "node: \n" << *begin_node;
+    //        DLOG << "node: \n" << *begin_node;
 
     std::vector<std::shared_ptr<framework::OpDesc>> op_descs;
     //    bool can_splite = begin_node->CanSplit({G_OP_TYPE_CONV,
     //    G_OP_TYPE_BATCHNORM, G_OP_TYPE_DEPTHWISE_CONV});
-    GenerateOps(&op_descs, begin_node.get());
+    for (int m = 0; m < nodes.size(); ++m) {
+      auto &node = nodes[m];
+      op_descs.push_back(node->op_desc_);
+    }
+    //    GenerateOps(&op_descs, begin_node.get());
     block->ops_ = op_descs;
   }
 
@@ -118,6 +131,14 @@ void ProgramOptimize::GenerateOps(
     Node *current_node) {
   if (current_node->inputs_.size() > 1 &&
       input_node != current_node->inputs_.back()) {
+    DLOG << " current type " << current_node->type_;
+
+    DLOG << " inputs size of current node > 0 ";
+
+    for (int i = 0; i < current_node->inputs_.size(); ++i) {
+      DLOG << " input i: " << current_node->inputs_[i]->type_;
+    }
+
     return;
   } else if (current_node->inputs_.size() > 1 &&
              input_node == current_node->inputs_.back()) {

src/io.cpp

@@ -221,6 +221,8 @@ const framework::Program<Dtype, P> Loader<Dtype, P>::Load(
     }
   }
 
+  //  originProgramDesc->Description("program: ");
+
   if (optimize) {
     framework::ProgramOptimize program_optimize;
     program.optimizeProgram =

src/io.h

@@ -32,7 +32,7 @@ template <typename Dtype, Precision P = Precision::FP32>
 class Loader : PaddleMobileObject {
  public:
   const framework::Program<Dtype, P> Load(const std::string &dirname,
-                                          bool optimize = true);
+                                          bool optimize = false);
 
  private:
   void LoadVar(framework::Variable *variable,

src/operators/fusion_conv_add.cpp

+/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include "operators/fusion_conv_add.h"
+namespace paddle_mobile {
+namespace operators {
+
+template <typename Dtype, typename T>
+void FushionConvAddOp<Dtype, T>::InferShape() const {}
+template class FushionConvAddOp<CPU, float>;
+}  // namespace operators
+}  // namespace paddle_mobile
+
+namespace ops = paddle_mobile::operators;
+USE_OP(conv_add);
+REGISTER_OPERATOR(conv_add, ops::FushionConvAddOp);

src/operators/fusion_conv_add.h

+/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#pragma once
+
+#include <string>
+#include <vector>
+
+#include "framework/operator.h"
+#include "framework/program/program-optimize/fusion_op_register.h"
+
+namespace paddle_mobile {
+namespace operators {
+using std::string;
+using std::vector;
+class FusionConvAddMatcher : public framework::FusionOpMatcher {
+ public:
+  FusionConvAddMatcher() {
+    node_ = framework::Node(G_OP_TYPE_CONV);
+    node_ > std::make_shared<framework::Node>(G_OP_TYPE_ELEMENTWISE_ADD);
+  }
+
+  void FolderNodes(
+      framework::Node *node,
+      std::vector<std::shared_ptr<framework::Node>> *removed_nodes) {
+    vector<std::shared_ptr<framework::OpDesc>> origin_descs =
+        node->OpDescs(node_.Depth());
+    node->Folder(node_.Depth(), Type(),
+                 {{G_OP_TYPE_ELEMENTWISE_ADD, {"Y", "Y"}}}, removed_nodes);
+  }
+
+  std::string Type() { return G_OP_TYPE_CONV_ADD; }
+};
+
+template <typename DeviceType, typename T>
+class FushionConvAddOp : public framework::OperatorWithKernel<DeviceType> {
+ public:
+  FushionConvAddOp(const string &type, const VariableNameMap &inputs,
+                   const VariableNameMap &outputs,
+                   const framework::AttributeMap attrs,
+                   std::shared_ptr<framework::Scope> scope)
+      : framework::OperatorWithKernel<DeviceType>(type, inputs, outputs, attrs,
+                                                  scope) {}
+
+  void RunImpl() const {}
+
+  using framework::OperatorWithKernel<DeviceType>::OperatorWithKernel;
+  void InferShape() const override;
+
+ protected:
+  //  FushionFcParam param_;
+};
+
+// static framework::FusionOpRegistrar fc_registrar(new FusionConvAddMatcher());
+
+}  // namespace operators
+}  // namespace paddle_mobile

src/operators/fusion_conv_add_relu_op.h

@@ -28,11 +28,13 @@ class FushionConvAddReluOpMatcher : public framework::FusionOpMatcher {
         std::make_shared<framework::Node>(G_OP_TYPE_RELU);
   }
 
-  void FolderNodes(framework::Node *node) {
+  void FolderNodes(
+      framework::Node *node,
+      std::vector<std::shared_ptr<framework::Node>> *removed_nodes) {
     std::vector<std::shared_ptr<framework::OpDesc>> origin_descs =
         node->OpDescs(node_.Depth());
     node->Folder(node_.Depth(), Type(),
-                 {{G_OP_TYPE_ELEMENTWISE_ADD, {"Y", "Z"}}});
+                 {{G_OP_TYPE_ELEMENTWISE_ADD, {"Y", "Z"}}}, removed_nodes);
   }
   std::string Type() { return G_OP_TYPE_FUSION_CONV_ADD_RELU; }
 };

src/operators/fusion_fc_op.h

@@ -32,11 +32,13 @@ class FusionFcMatcher : public framework::FusionOpMatcher {
     node_ > std::make_shared<framework::Node>(G_OP_TYPE_ELEMENTWISE_ADD);
   }
 
-  void FolderNodes(framework::Node *node) {
+  void FolderNodes(
+      framework::Node *node,
+      std::vector<std::shared_ptr<framework::Node>> *removed_nodes) {
     vector<std::shared_ptr<framework::OpDesc>> origin_descs =
         node->OpDescs(node_.Depth());
     node->Folder(node_.Depth(), Type(),
-                 {{G_OP_TYPE_ELEMENTWISE_ADD, {"Y", "Z"}}});
+                 {{G_OP_TYPE_ELEMENTWISE_ADD, {"Y", "Z"}}}, removed_nodes);
   }
 
   std::string Type() { return G_OP_TYPE_FC; }
@@ -65,7 +67,7 @@ class FushionFcOp : public framework::OperatorWithKernel<DeviceType> {
   FushionFcParam param_;
 };
 
-static framework::FusionOpRegistrar fc_registrar(new FusionFcMatcher());
+// static framework::FusionOpRegistrar fc_registrar(new FusionFcMatcher());
 
 }  // namespace operators
 }  // namespace paddle_mobile

src/operators/kernel/mali/conv_kernel.cpp

+/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include "operators/kernel/conv_kernel.h"
+
+namespace paddle_mobile {
+namespace operators {
+
+template <>
+void ConvKernel<GPU_MALI, float>::Compute(const ConvParam &param) const {}
+
+template class ConvKernel<GPU_MALI, float>;
+}  // namespace operators
+}  // namespace paddle_mobile

test/framework/test_load.cpp

@@ -19,7 +19,7 @@ int main() {
   paddle_mobile::Loader<paddle_mobile::CPU> loader;
   //  ../../../test/models/googlenet
   //  ../../../test/models/mobilenet
-  auto program = loader.Load(g_googlenet);
-  program.optimizeProgram->Description("program desc: ");
+  auto program = loader.Load(g_mobilenet_ssd, true);
+  program.originProgram->Description("program desc: ");
   return 0;
 }

test/framework/test_optimize.cpp

@@ -20,12 +20,12 @@ limitations under the License. */
 int main() {
   paddle_mobile::Loader<paddle_mobile::CPU> loader;
   //    "../../../test/models/googlenet"
-  auto program = loader.Load(g_mobilenet_ssd);
+  auto program = loader.Load(g_mobilenet_ssd, true);
   paddle_mobile::framework::ProgramOptimize optimize;
   //  program.originProgram->Description("origin");
   auto optimize_program = optimize.FushionOptimize(program.originProgram);
   if (optimize_program != nullptr) {
-    optimize_program->Description("optimize");
+    //    optimize_program->Description("optimize");
   } else {
     LOG(paddle_mobile::kLOG_ERROR) << "optimize_program is null";
   }

test/net/test_mobilenet+ssd.cpp

@@ -19,10 +19,10 @@ limitations under the License. */
 int main() {
   paddle_mobile::Loader<paddle_mobile::CPU> loader;
   auto time1 = time();
-  auto program = loader.Load(g_mobilenet_ssd, false);
+  auto program = loader.Load(g_mobilenet_ssd, true);
   auto time2 = time();
   DLOG << "load cost :" << time_diff(time1, time1) << "ms";
-  paddle_mobile::Executor<paddle_mobile::CPU> executor(program, 1, false);
+  paddle_mobile::Executor<paddle_mobile::CPU> executor(program, 1, true);
 
   std::vector<int64_t> dims{1, 3, 300, 300};
   Tensor input_tensor;

test/test_helper.h

@@ -22,10 +22,11 @@ limitations under the License. */
 #include "framework/ddim.h"
 #include "framework/tensor.h"
 
-static const std::string g_googlenet = "../models/googlenet";
-static const std::string g_mobilenet = "../models/mobilenet";
 static const std::string g_mobilenet_ssd = "../models/mobilenet+ssd";
 static const std::string g_squeezenet = "../models/squeezenet";
+static const std::string g_googlenet = "../models/googlenet";
+static const std::string g_mobilenet = "../models/mobilenet";
+static const std::string g_resnet_50 = "../models/resnet_50";
 static const std::string g_resnet = "../models/resnet";
 static const std::string g_yolo = "../models/yolo";
 static const std::string g_test_image_1x3x224x224 =