support RFCN

src/io/api_paddle_mobile.cc

@@ -110,6 +110,74 @@ bool PaddleMobilePredictor<Device, T>::Run(
   return true;
 }
 
+#ifdef PADDLE_MOBILE_FPGA
+template <typename Device, typename T>
+bool PaddleMobilePredictor<Device, T>::Run(
+    const std::vector<PaddleTensor> &inputs,
+    std::vector<PaddleTensor> *output_data, std::vector<int> *index_data,
+    int batch_size) {
+  if (inputs.empty()) {
+    LOG(kLOG_ERROR) << "At least one output should be set with tensors' names.";
+    return false;
+  }
+  auto input = inputs[0];
+
+  if (input.shape.size() != 4) {
+    LOG(kLOG_ERROR) << "input shape not equal to 4!";
+    return false;
+  }
+  std::vector<int64_t> dims;
+  for (auto d : input.shape) {
+    dims.push_back(static_cast<int64_t>(d));
+  }
+
+  // use tensor
+  framework::DDim ddim =
+      framework::make_ddim({dims[0], dims[1], dims[2], dims[3]});
+
+  framework::Tensor input_tensor;
+  input_tensor.Resize(ddim);
+  int input_length = framework::product(ddim);
+  auto input_ptr = input_tensor.mutable_data<T>();
+
+  memcpy(input_ptr, static_cast<T *>(input.data.data()),
+         input_length * sizeof(T));
+  paddle_mobile_->Predict(input_tensor);
+  auto num_result = index_data->size();
+  if (output_data->size() != num_result) {
+    LOG(kLOG_ERROR) << "index and output number don't match";
+    return false;
+  }
+
+  for (int i = 0; i < num_result; i++) {
+    auto output_tensor = paddle_mobile_->FetchResult((*index_data)[i]);
+
+    if (output_data->empty()) {
+      LOG(kLOG_ERROR)
+          << "At least one output should be set with tensors' names.";
+      return false;
+    }
+
+    auto &output = (*output_data)[i];
+    int output_length = output_tensor->numel();
+    std::vector<int64_t> tensor_shape =
+        framework::vectorize(output_tensor->dims());
+
+    for (auto d : tensor_shape) {
+      output.shape.push_back(static_cast<int>(d));
+    }
+
+    if (output.data.length() < output_length * sizeof(T)) {
+      output.data.Resize(output_length * sizeof(T));
+    }
+
+    memcpy(output.data.data(), output_tensor->template data<T>(),
+           output_length * sizeof(T));
+  }
+
+  return true;
+}
+#endif
 template <typename Device, typename T>
 PaddleMobilePredictor<Device, T>::~PaddleMobilePredictor() {
   paddle_mobile_->Clear();

src/io/api_paddle_mobile.h

@@ -31,7 +31,11 @@ class PaddleMobilePredictor : public PaddlePredictor {
   bool Run(const std::vector<PaddleTensor>& inputs,
            std::vector<PaddleTensor>* output_data,
            int batch_size = -1) override;
-
+#ifdef PADDLE_MOBILE_FPGA
+  bool Run(const std::vector<PaddleTensor>& inputs,
+           std::vector<PaddleTensor>* output_data, std::vector<int>* index_data,
+           int batch_size = -1) override;
+#endif
   ~PaddleMobilePredictor() override;
 
  private:

src/io/paddle_inference_api.h

@@ -26,8 +26,16 @@ limitations under the License. */
 #include <string>
 #include <vector>
 
+// #define PADDLE_MOBILE_FPGA
+
 namespace paddle_mobile {
 
+#ifdef PADDLE_MOBILE_FPGA
+namespace fpga {
+int open_device();
+}
+#endif
+
 enum PaddleDType {
   FLOAT32,
   INT64,
@@ -107,6 +115,11 @@ class PaddlePredictor {
     std::string prog_file;
     std::string param_file;
   };
+#ifdef PADDLE_MOBILE_FPGA
+  virtual bool Run(const std::vector<PaddleTensor>& inputs,
+                   std::vector<PaddleTensor>* output_data,
+                   std::vector<int>* index_data, int batch_size = -1) = 0;
+#endif
 
  protected:
   PaddlePredictor() = default;

src/operators/detection_ops.cpp

@@ -22,6 +22,7 @@ namespace operators {
 template <typename DeviceType, typename T>
 void AnchorGeneratorOp<DeviceType, T>::InferShape() const {
   const auto &input_dims = this->param_.input_->dims();
+  // DLOG << "AnchorGenerator input dim =" << input_dims.size();
   PADDLE_MOBILE_ENFORCE(input_dims.size() == 4, "The layout of input is NCHW.");
   const auto &anchor_sizes = this->param_.anchor_sizes_;
   const auto &aspect_ratios = this->param_.aspect_ratios_;
@@ -98,3 +99,15 @@ REGISTER_OPERATOR_CPU(psroi_pool, ops::PSRoiPoolOp);
 REGISTER_OPERATOR_CPU(roi_perspective_transform, ops::RoiPerspectiveOp);
 #endif
 #endif
+
+#ifdef PADDLE_MOBILE_FPGA
+#ifdef ANCHOR_GENERATOR_OP
+REGISTER_OPERATOR_FPGA(anchor_generator, ops::AnchorGeneratorOp);
+#endif
+#ifdef PROPOSAL_OP
+REGISTER_OPERATOR_FPGA(generate_proposals, ops::ProposalOp);
+#endif
+#ifdef PSROI_POOL_OP
+REGISTER_OPERATOR_FPGA(psroi_pool, ops::PSRoiPoolOp);
+#endif
+#endif

src/operators/kernel/detection_kernel.h

@@ -28,11 +28,11 @@ class AnchorGeneratorParam : public OpParam {
   AnchorGeneratorParam(const VariableNameMap &inputs,
                        const VariableNameMap &outputs,
                        const AttributeMap &attrs, const Scope &scope) {
-    input_ = OpParam::GetVarValue<framework::Tensor>("Input", inputs, scope);
+    input_ = OpParam::GetVarValue<framework::LoDTensor>("Input", inputs, scope);
     output_anchors_ =
-        OpParam::GetVarValue<framework::Tensor>("Anchors", outputs, scope);
+        OpParam::GetVarValue<framework::LoDTensor>("Anchors", outputs, scope);
     output_variances_ =
-        OpParam::GetVarValue<framework::Tensor>("Variances", outputs, scope);
+        OpParam::GetVarValue<framework::LoDTensor>("Variances", outputs, scope);
 
     anchor_sizes_ = OpParam::GetAttr<std::vector<float>>("anchor_sizes", attrs);
     aspect_ratios_ =
@@ -65,14 +65,16 @@ class ProposalParam : public OpParam {
  public:
   ProposalParam(const VariableNameMap &inputs, const VariableNameMap &outputs,
                 const AttributeMap &attrs, const Scope &scope) {
-    scores_ = OpParam::GetVarValue<framework::Tensor>("Scores", inputs, scope);
+    scores_ =
+        OpParam::GetVarValue<framework::LoDTensor>("Scores", inputs, scope);
     bbox_deltas_ =
-        OpParam::GetVarValue<framework::Tensor>("BboxDeltas", inputs, scope);
-    im_info_ = OpParam::GetVarValue<framework::Tensor>("ImInfo", inputs, scope);
+        OpParam::GetVarValue<framework::LoDTensor>("BboxDeltas", inputs, scope);
+    im_info_ =
+        OpParam::GetVarValue<framework::LoDTensor>("ImInfo", inputs, scope);
     anchors_ =
-        OpParam::GetVarValue<framework::Tensor>("Anchors", inputs, scope);
+        OpParam::GetVarValue<framework::LoDTensor>("Anchors", inputs, scope);
     variances_ =
-        OpParam::GetVarValue<framework::Tensor>("Variances", inputs, scope);
+        OpParam::GetVarValue<framework::LoDTensor>("Variances", inputs, scope);
 
     rpn_rois_ =
         OpParam::GetVarValue<framework::LoDTensor>("RpnRois", outputs, scope);
@@ -112,10 +114,10 @@ class PSRoiPoolParam : public OpParam {
  public:
   PSRoiPoolParam(const VariableNameMap &inputs, const VariableNameMap &outputs,
                  const AttributeMap &attrs, const Scope &scope) {
-    input_x_ = OpParam::GetVarValue<framework::Tensor>("X", inputs, scope);
+    input_x_ = OpParam::GetVarValue<framework::LoDTensor>("X", inputs, scope);
     input_rois_ =
         OpParam::GetVarValue<framework::LoDTensor>("ROIs", inputs, scope);
-    output_ = OpParam::GetVarValue<framework::Tensor>("Out", outputs, scope);
+    output_ = OpParam::GetVarValue<framework::LoDTensor>("Out", outputs, scope);
 
     output_channels_ = OpParam::GetAttr<int>("output_channels", attrs);
     pooled_height_ = OpParam::GetAttr<int>("pooled_height", attrs);
@@ -143,10 +145,10 @@ class RoiPerspectiveParam : public OpParam {
   RoiPerspectiveParam(const VariableNameMap &inputs,
                       const VariableNameMap &outputs, const AttributeMap &attrs,
                       const Scope &scope) {
-    input_x_ = OpParam::GetVarValue<framework::Tensor>("X", inputs, scope);
+    input_x_ = OpParam::GetVarValue<framework::LoDTensor>("X", inputs, scope);
     input_rois_ =
         OpParam::GetVarValue<framework::LoDTensor>("ROIs", inputs, scope);
-    output_ = OpParam::GetVarValue<framework::Tensor>("Out", outputs, scope);
+    output_ = OpParam::GetVarValue<framework::LoDTensor>("Out", outputs, scope);
 
     spatial_scale_ = OpParam::GetAttr<float>("spatial_scale", attrs);
     transformed_height_ = OpParam::GetAttr<int>("transformed_height", attrs);

src/operators/kernel/fpga/V1/anchor_generator_kernel.cpp

+/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#ifdef ANCHOR_GENERATOR_OP
+
+#include <vector>
+#include "operators/kernel/detection_kernel.h"
+
+namespace paddle_mobile {
+namespace operators {
+
+template <>
+bool AnchorGeneratorKernel<FPGA, float>::Init(
+    AnchorGeneratorParam<FPGA> *param) {
+  return true;
+  
+}
+
+template <>
+void AnchorGeneratorKernel<FPGA, float>::Compute(
+    const AnchorGeneratorParam<FPGA> &param) {
+  // TODO(hjchen2)
+}
+
+}  // namespace operators
+}  // namespace paddle_mobile
+
+#endif  // ANCHOR_GENERATOR_OP

src/operators/kernel/fpga/V1/proposal_kernel.cpp

+/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#ifdef PROPOSAL_OP
+
+#include <vector>
+#include "operators/kernel/detection_kernel.h"
+
+namespace paddle_mobile {
+namespace operators {
+
+template <>
+bool ProposalKernel<FPGA, float>::Init(ProposalParam<FPGA> *param) {
+  return true;
+}
+
+template <>
+void ProposalKernel<FPGA, float>::Compute(const ProposalParam<FPGA> &param) {
+  // TODO(hjchen2)
+}
+
+}  // namespace operators
+}  // namespace paddle_mobile
+
+#endif  // PROPOSAL_OP

src/operators/kernel/fpga/V1/psroi_pool_kernel.cpp

+/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#ifdef PSROI_POOL_OP
+
+#include <vector>
+#include "operators/kernel/detection_kernel.h"
+
+namespace paddle_mobile {
+namespace operators {
+
+template <>
+bool PSRoiPoolKernel<FPGA, float>::Init(PSRoiPoolParam<FPGA> *param) {
+  return true;
+}
+
+template <>
+void PSRoiPoolKernel<FPGA, float>::Compute(const PSRoiPoolParam<FPGA> &param) {
+  // TODO(hjchen2)
+}
+
+}  // namespace operators
+}  // namespace paddle_mobile
+
+#endif  // PSROI_POOL_OP

src/operators/kernel/fpga/V1/reshape2_kernel.cpp

+/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#ifdef RESHAPE2_OP
+
+#include "operators/kernel/reshape2_kernel.h"
+
+namespace paddle_mobile {
+namespace operators {
+
+template <>
+bool Reshape2Kernel<FPGA, float>::Init(Reshape2Param<FPGA> *param) {
+  return true;
+}
+
+template <>
+void Reshape2Kernel<FPGA, float>::Compute(const Reshape2Param<FPGA> &param) {
+  return;
+}
+
+}  // namespace operators
+}  // namespace paddle_mobile
+
+#endif

src/operators/kernel/fpga/V1/slice_kernel.cpp

+/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#ifdef SLICE_OP
+
+#include "operators/kernel/slice_kernel.h"
+
+namespace paddle_mobile {
+namespace operators {
+
+template <>
+bool SliceKernel<FPGA, float>::Init(SliceParam<FPGA>* param) {
+  return true;
+}
+template <>
+void SliceKernel<FPGA, float>::Compute(const SliceParam<FPGA>& param) {}
+}  // namespace operators
+}  // namespace paddle_mobile
+#endif

src/operators/reshape2_op.cpp

@@ -43,5 +43,8 @@ REGISTER_OPERATOR_CPU(reshape2, ops::Reshape2Op);
 #ifdef PADDLE_MOBILE_MALI_GPU
 REGISTER_OPERATOR_MALI_GPU(reshape2, ops::Reshape2Op);
 #endif
+#ifdef PADDLE_MOBILE_FPGA
+REGISTER_OPERATOR_FPGA(reshape2, ops::Reshape2Op);
+#endif
 
 #endif

test/CMakeLists.txt

@@ -74,6 +74,9 @@ if (CON GREATER -1)
     ADD_EXECUTABLE(test-densebox fpga/test_densebox_combine.cpp test_helper.h test_include.h executor_for_test.h)
     target_link_libraries(test-densebox paddle-mobile)
 
+    ADD_EXECUTABLE(test-rfcn fpga/test_rfcn.cpp test_helper.h test_include.h executor_for_test.h)
+    target_link_libraries(test-rfcn paddle-mobile)
+
     set(FOUND_MATCH ON)
 endif ()
 

test/fpga/test_rfcn.cpp

+/* Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include <iostream>
+#include "../test_helper.h"
+#include "../test_include.h"
+
+#ifdef PADDLE_MOBILE_FPGA_V1
+#include "fpga/V1/api.h"
+#endif
+#ifdef PADDLE_MOBILE_FPGA_V2
+#include "fpga/V2/api.h"
+#endif
+
+// static const char *g_densebox_combine = "../models/densebox";
+static const char *g_densebox_combine = "../models/rfcn";
+int main() {
+  paddle_mobile::fpga::open_device();
+  paddle_mobile::PaddleMobile<paddle_mobile::FPGA> paddle_mobile;
+  // paddle_mobile.SetThreadNum(4);
+  if (paddle_mobile.Load(std::string(g_densebox_combine) + "/model",
+                         std::string(g_densebox_combine) + "/params", true,
+                         false, 1, true)) {
+    // std::vector<float> input;
+    // std::vector<int64_t> dims{1, 3, 512, 1024};
+    // GetInput<float>(g_test_image_1x3x224x224_banana, &input, dims);
+
+    // auto vec_result = paddle_mobile.Predict(input, dims);
+    return 0;
+
+    Tensor input_tensor;
+    SetupTensor<float>(&input_tensor, {1, 3, 512, 1024}, static_cast<float>(0),
+                       static_cast<float>(1));
+    // readStream(g_image_src_float,
+    //           input_tensor.mutable_data<float>({1, 3, 224, 224}));
+    paddle_mobile.FeedData(input_tensor);
+    paddle_mobile.Predict_To(-1);
+  }
+
+  return 0;
+}

tools/op.cmake

@@ -126,6 +126,11 @@ if (CON GREATER -1)
   set(RESHAPE_OP ON)
   set(FUSION_CONVADDBNRELU_OP ON)
   set(FUSION_CONVADDBN_OP ON)
+  set(RESHAPE2_OP ON)
+  set(PSROI_POOL_OP ON)
+  set(PROPOSAL_OP ON)
+  set(ANCHOR_GENERATOR_OP ON)
+  set(SLICE_OP ON)
 
   set(FOUND_MATCH ON)
 endif()