[Inference]Fix the ort Backend multiple input bug (#43621)

* fix or backend many inputs bug

* fix or backend many inputs bug

* fix or backend many inputs bug

* fix or backend many inputs bug

* code format

* code format

cmake/external/onnxruntime.cmake

@@ -52,8 +52,9 @@ else()
   )
 endif()
 
-include_directories(${ONNXRUNTIME_INC_DIR}
-)# For ONNXRUNTIME code to include internal headers.
+# For ONNXRUNTIME code to include internal headers.
+include_directories(${ONNXRUNTIME_INC_DIR})
+
 if(WIN32)
   set(ONNXRUNTIME_SOURCE_LIB
       "${ONNXRUNTIME_SOURCE_DIR}/lib/onnxruntime.dll"

cmake/external/paddle2onnx.cmake

@@ -34,15 +34,11 @@ set(PADDLE2ONNX_INC_DIR
 set(PADDLE2ONNX_LIB_DIR
     "${PADDLE2ONNX_INSTALL_DIR}/lib"
     CACHE PATH "onnxruntime lib directory." FORCE)
-set(CMAKE_BUILD_RPATH "${CMAKE_BUILD_RPATH}"
-                      "${PADDLE2ONNX_INSTALL_DIR}/${LIBDIR}")
+set(CMAKE_BUILD_RPATH "${CMAKE_BUILD_RPATH}" "${PADDLE2ONNX_LIB_DIR}")
 
-include_directories(${PADDLE2ONNX_INC_DIR}
-)# For PADDLE2ONNX code to include internal headers.
+# For PADDLE2ONNX code to include internal headers.
+include_directories(${PADDLE2ONNX_INC_DIR})
 if(WIN32)
-  set(PADDLE2ONNX_SOURCE_LIB
-      "${PADDLE2ONNX_SOURCE_DIR}/lib/libpaddle2onnx.dylib"
-      CACHE FILEPATH "Paddle2ONNX source library." FORCE)
   set(PADDLE2ONNX_LIB
       "${PADDLE2ONNX_INSTALL_DIR}/lib/paddle2onnx.dll"
       CACHE FILEPATH "paddle2onnx library." FORCE)
@@ -50,9 +46,6 @@ if(WIN32)
       "${PADDLE2ONNX_INSTALL_DIR}/lib/paddle2onnx.lib"
       CACHE FILEPATH "paddle2onnx compile library." FORCE)
 elseif(APPLE)
-  set(PADDLE2ONNX_SOURCE_LIB
-      "${PADDLE2ONNX_SOURCE_DIR}/lib/libpaddle2onnx.dylib"
-      CACHE FILEPATH "Paddle2ONNX source library." FORCE)
   set(PADDLE2ONNX_LIB
       "${PADDLE2ONNX_INSTALL_DIR}/lib/libpaddle2onnx.dylib"
       CACHE FILEPATH "PADDLE2ONNX library." FORCE)
@@ -60,9 +53,6 @@ elseif(APPLE)
       "${PADDLE2ONNX_INSTALL_DIR}/lib/libpaddle2onnx.dylib"
       CACHE FILEPATH "paddle2onnx compile library." FORCE)
 else()
-  set(PADDLE2ONNX_SOURCE_LIB
-      "${PADDLE2ONNX_SOURCE_DIR}/lib/libpaddle2onnx.so"
-      CACHE FILEPATH "Paddle2ONNX source library." FORCE)
   set(PADDLE2ONNX_LIB
       "${PADDLE2ONNX_INSTALL_DIR}/lib/libpaddle2onnx.so"
       CACHE FILEPATH "PADDLE2ONNX library." FORCE)

paddle/fluid/inference/api/demo_ci/onnxruntime_mobilenet_demo.cc

@@ -13,17 +13,19 @@ See the License for the specific language governing permissions and
 limitations under the License. */
 
 /*
- * This file contains demo of mobilenet for tensorrt.
+ * This file contains demo of mobilenet for onnxruntime backend.
  */
-
 #include <glog/logging.h>  // use glog instead of CHECK to avoid importing other paddle header files.
 
+#include <algorithm>
+#include <numeric>
 #include <vector>
 
 #include "gflags/gflags.h"
 #include "utils.h"  // NOLINT
 
 DEFINE_string(modeldir, "", "Directory of the inference model.");
+DEFINE_string(data, "", "path of data");
 
 namespace paddle {
 namespace demo {
@@ -39,8 +41,21 @@ void Main() {
   auto predictor = paddle_infer::CreatePredictor(config);
 
   // Inference.
+  LOG(INFO) << "--- prepare input data ----";
   std::vector<int> input_shape = {1, 3, 224, 224};
-  std::vector<float> input_data(1 * 3 * 224 * 224, 1.0);
+  std::vector<float> input_data;
+  std::string line;
+  std::ifstream file(FLAGS_data);
+  std::getline(file, line);
+  file.close();
+  std::vector<std::string> data_strs;
+  split(line, ' ', &data_strs);
+  int input_num = 0;
+  for (auto& d : data_strs) {
+    input_num += 1;
+    input_data.push_back(std::stof(d));
+  }
+
   std::vector<float> out_data;
   out_data.resize(1000);
   auto input_names = predictor->GetInputNames();
@@ -53,7 +68,19 @@ void Main() {
   predictor->Run();
   output_tensor->CopyToCpu(out_data.data());
 
-  VLOG(3) << "output.size " << out_data.size();
+  std::vector<int> out_index(out_data.size());
+  std::iota(out_index.begin(), out_index.end(), 0);
+  std::sort(
+      out_index.begin(), out_index.end(), [&out_data](int index1, int index2) {
+        return out_data[index1] > out_data[index2];
+      });
+  LOG(INFO) << "output.size " << out_data.size()
+            << "  max_index:" << out_index[0];
+  CHECK_EQ(out_data.size(), 1000);
+  int max_index = out_index[0];
+  CHECK_EQ(max_index, 13);
+  float max_score = out_data[max_index];
+  CHECK_LE(fabs(max_score - 0.99981), 1e-4);
 }
 
 }  // namespace demo

paddle/fluid/inference/api/demo_ci/run.sh

@@ -52,15 +52,17 @@ if [ $7 == ON ]; then
   mkdir -p MobileNetV2
   cd MobileNetV2
   if [[ -e "MobileNetV2.inference.model.tar.gz" ]]; then
-    echo "MobileNetV2.inference.model.tar.gz has been downloaded."
-  else
+    rm -rf MobileNetV2.inference.model.tar.gz
+  fi
+    # echo "MobileNetV2.inference.model.tar.gz has been downloaded."
+  # else
     if [ $WIN_DETECT != "" ]; then
       wget -q -Y off http://paddle-inference-dist.bj.bcebos.com/MobileNetV2.inference.model.tar.gz
     else
       wget -q --no-proxy http://paddle-inference-dist.bj.bcebos.com/MobileNetV2.inference.model.tar.gz
     fi
     tar xzf *.tar.gz
-  fi
+  # fi
   cd ..
 fi
 
@@ -265,7 +267,8 @@ for WITH_STATIC_LIB in ON OFF; do
         -DWITH_ONNXRUNTIME=$WITH_ONNXRUNTIME
       make -j$(nproc)
       ./onnxruntime_mobilenet_demo \
-        --modeldir=$DATA_DIR/MobileNetV2/MobileNetV2
+        --modeldir=$DATA_DIR/MobileNetV2/MobileNetV2 \
+        --data=$DATA_DIR/MobileNetV2/MobileNetV2/data.txt
       if [ $? -ne 0 ]; then
         echo "onnxruntime_mobilenet_demo runs failed " >> ${current_dir}/test_summary.txt
         EXIT_CODE=1

paddle/fluid/inference/api/onnxruntime_predictor.cc

@@ -71,13 +71,16 @@ bool CheckConvertToONNX(const AnalysisConfig &config) {
   } else if (config.prog_file().empty() || config.params_file().empty()) {
     LOG(ERROR) << string::Sprintf(
         "not valid model path '%s' or program path '%s' or params path '%s'.",
-        config.model_dir(), config.prog_file(), config.params_file());
+        config.model_dir(),
+        config.prog_file(),
+        config.params_file());
     return false;
   }
   if (config.model_from_memory()) {
-    return paddle2onnx::IsExportable(
-        config.prog_file().data(), config.prog_file().size(),
-        config.params_file().data(), config.params_file().size());
+    return paddle2onnx::IsExportable(config.prog_file().data(),
+                                     config.prog_file().size(),
+                                     config.params_file().data(),
+                                     config.params_file().size());
   } else {
     return paddle2onnx::IsExportable(config.prog_file().c_str(),
                                      config.params_file().c_str());
@@ -98,12 +101,17 @@ bool ONNXRuntimePredictor::Init() {
   char *onnx_proto = nullptr;
   int out_size;
   if (config_.model_from_memory()) {
-    paddle2onnx::Export(config_.prog_file().data(), config_.prog_file().size(),
+    paddle2onnx::Export(config_.prog_file().data(),
+                        config_.prog_file().size(),
                         config_.params_file().data(),
-                        config_.params_file().size(), &onnx_proto, &out_size);
+                        config_.params_file().size(),
+                        &onnx_proto,
+                        &out_size);
   } else {
     paddle2onnx::Export(config_.prog_file().c_str(),
-                        config_.params_file().c_str(), &onnx_proto, &out_size);
+                        config_.params_file().c_str(),
+                        &onnx_proto,
+                        &out_size);
   }
 
   Ort::SessionOptions session_options;
@@ -134,8 +142,8 @@ bool ONNXRuntimePredictor::Init() {
   session_ = {env_, onnx_proto, static_cast<size_t>(out_size), session_options};
   binding_ = std::make_shared<Ort::IoBinding>(session_);
 
-  Ort::MemoryInfo memory_info(device_name, OrtDeviceAllocator,
-                              place_.GetDeviceId(), OrtMemTypeDefault);
+  Ort::MemoryInfo memory_info(
+      device_name, OrtDeviceAllocator, place_.GetDeviceId(), OrtMemTypeDefault);
   Ort::Allocator allocator(session_, memory_info);
 
   size_t n_inputs = session_.GetInputCount();
@@ -160,8 +168,10 @@ bool ONNXRuntimePredictor::Init() {
         type_info.GetTensorTypeAndShapeInfo().GetElementType();
     output_desc_.emplace_back(ONNXDesc{output_name, shape, data_type});
 
-    Ort::MemoryInfo out_memory_info(device_name, OrtDeviceAllocator,
-                                    place_.GetDeviceId(), OrtMemTypeDefault);
+    Ort::MemoryInfo out_memory_info(device_name,
+                                    OrtDeviceAllocator,
+                                    place_.GetDeviceId(),
+                                    OrtMemTypeDefault);
     binding_->BindOutput(output_name, out_memory_info);
 
     allocator.Free(output_name);
@@ -181,7 +191,8 @@ CreatePaddlePredictor<AnalysisConfig, PaddleEngineKind::kONNXRuntime>(
   }
 
   PADDLE_ENFORCE_EQ(
-      config.is_valid(), true,
+      config.is_valid(),
+      true,
       platform::errors::InvalidArgument(
           "Note: Each config can only be used for one predictor."));
 
@@ -238,7 +249,8 @@ bool ONNXRuntimePredictor::FindONNXDesc(const std::string &name,
 
 std::unique_ptr<ZeroCopyTensor> ONNXRuntimePredictor::GetInputTensor(
     const std::string &name) {
-  PADDLE_ENFORCE_EQ(FindONNXDesc(name, true), true,
+  PADDLE_ENFORCE_EQ(FindONNXDesc(name, true),
+                    true,
                     platform::errors::PreconditionNotMet(
                         "The in variable named %s is not found in the "
                         "ONNXPredictor.",
@@ -254,12 +266,21 @@ std::unique_ptr<ZeroCopyTensor> ONNXRuntimePredictor::GetInputTensor(
   }
   res->SetOrtMark(true);
   res->SetOrtBinding(binding_);
+  auto iter = input_buffers_.find(name);
+  if (iter == input_buffers_.end()) {
+    std::vector<int8_t> i_vector;
+    input_buffers_[name] = std::make_shared<std::vector<int8_t>>(i_vector);
+    res->SetOrtBuffer(input_buffers_[name]);
+  } else {
+    res->SetOrtBuffer(iter->second);
+  }
   return res;
 }
 
 std::unique_ptr<ZeroCopyTensor> ONNXRuntimePredictor::GetOutputTensor(
     const std::string &name) {
-  PADDLE_ENFORCE_EQ(FindONNXDesc(name, false), true,
+  PADDLE_ENFORCE_EQ(FindONNXDesc(name, false),
+                    true,
                     platform::errors::PreconditionNotMet(
                         "The out variable named %s is not found in the "
                         "ONNXPredictor.",
@@ -296,8 +317,10 @@ bool ONNXRuntimePredictor::ZeroCopyRun() {
   try {
     const char *device_name = place_ == PlaceType::kCPU ? "Cpu" : "Cuda";
     for (auto output : output_desc_) {
-      Ort::MemoryInfo out_memory_info(device_name, OrtDeviceAllocator,
-                                      place_.GetDeviceId(), OrtMemTypeDefault);
+      Ort::MemoryInfo out_memory_info(device_name,
+                                      OrtDeviceAllocator,
+                                      place_.GetDeviceId(),
+                                      OrtMemTypeDefault);
       binding_->BindOutput(output.name.c_str(), out_memory_info);
     }
     session_.Run({}, *(binding_.get()));
@@ -330,8 +353,9 @@ const void *ONNXRuntimePredictor::GetDeviceContexts() const {
   paddle::platform::DeviceContextPool &pool =
       paddle::platform::DeviceContextPool::Instance();
   const auto &dev_ctxs = pool.device_contexts();
-  return &const_cast<std::map<
-      phi::Place, std::shared_future<std::unique_ptr<phi::DeviceContext>>> &>(
+  return &const_cast<
+      std::map<phi::Place,
+               std::shared_future<std::unique_ptr<phi::DeviceContext>>> &>(
       dev_ctxs);
 }
 

paddle/fluid/inference/api/onnxruntime_predictor.h

@@ -202,6 +202,7 @@ class ONNXRuntimePredictor : public PaddlePredictor {
   platform::Place place_;
   std::vector<ONNXDesc> input_desc_;
   std::vector<ONNXDesc> output_desc_;
+  std::map<std::string, std::shared_ptr<std::vector<int8_t>>> input_buffers_;
   int predictor_id_;
 
 // Some more detailed tests, they are made the friends of the predictor, so that

paddle/fluid/inference/api/paddle_tensor.h

@@ -110,7 +110,8 @@ class PD_INFER_DECL Tensor {
   /// \param place The place of data.
   /// \param layout The layout of data. Only NCHW is supported now.
   template <typename T>
-  void ShareExternalData(const T* data, const std::vector<int>& shape,
+  void ShareExternalData(const T* data,
+                         const std::vector<int>& shape,
                          PlaceType place,
                          DataLayout layout = DataLayout::kNCHW);
 
@@ -171,7 +172,9 @@ class PD_INFER_DECL Tensor {
   void SetName(const std::string& name);
 
   template <typename T>
-  void CopyToCpuImpl(T* data, void* stream = nullptr, CallbackFunc cb = nullptr,
+  void CopyToCpuImpl(T* data,
+                     void* stream = nullptr,
+                     CallbackFunc cb = nullptr,
                      void* cb_params = nullptr) const;
 
   std::string name_;
@@ -188,7 +191,7 @@ class PD_INFER_DECL Tensor {
 #ifdef PADDLE_WITH_ONNXRUNTIME
   bool is_ort_tensor_{false};
   std::vector<int64_t> shape_;
-  std::vector<int8_t> buffer_;
+  std::weak_ptr<std::vector<int8_t>> buffer_;
   std::weak_ptr<Ort::IoBinding> binding_;
   int idx_{-1};
 
@@ -196,6 +199,8 @@ class PD_INFER_DECL Tensor {
 
   void SetOrtBinding(const std::shared_ptr<Ort::IoBinding> binding);
 
+  void SetOrtBuffer(const std::shared_ptr<std::vector<int8_t>> buffer);
+
   template <typename T>
   void ORTCopyFromCpu(const T* data);
 

paddle/fluid/pybind/CMakeLists.txt

@@ -384,12 +384,12 @@ if(WITH_PYTHON)
         set(PADDLE2ONNX_PYBIND_OUT
             ${CMAKE_CURRENT_BINARY_DIR}/libpaddle2onnx.dylib)
         set(ONNXRUNTIME_PYBIND_OUT
-            ${CMAKE_CURRENT_BINARY_DIR}/libonnxruntime.dylib)
+            ${CMAKE_CURRENT_BINARY_DIR}/libonnxruntime.1.10.0.dylib)
       else()
         set(PADDLE2ONNX_PYBIND_OUT
             ${CMAKE_CURRENT_BINARY_DIR}/libpaddle2onnx.so)
         set(ONNXRUNTIME_PYBIND_OUT
-            ${CMAKE_CURRENT_BINARY_DIR}/libonnxruntime.so)
+            ${CMAKE_CURRENT_BINARY_DIR}/libonnxruntime.so.1.10.0)
       endif()
 
       add_custom_command(