Merge pull request #11162 from tensor-tang/infer/api

enable infer api with multi-threads

paddle/contrib/inference/demo/simple_on_word2vec.cc

@@ -65,7 +65,10 @@ void Main(bool use_gpu) {
 }
 
 TEST(demo, word2vec_cpu) { Main(false /*use_gpu*/); }
+
+#ifdef PADDLE_WITH_CUDA
 TEST(demo, word2vec_gpu) { Main(true /*use_gpu*/); }
+#endif
 
 }  // namespace demo
 }  // namespace paddle

paddle/contrib/inference/paddle_inference_api.h

@@ -63,6 +63,7 @@ class PaddlePredictor {
   struct Config;
   PaddlePredictor() = default;
   PaddlePredictor(const PaddlePredictor&) = delete;
+  PaddlePredictor& operator=(const PaddlePredictor&) = delete;
 
   // Predict an record.
   // The caller should be responsible for allocating and releasing the memory of
@@ -76,7 +77,7 @@ class PaddlePredictor {
   virtual std::unique_ptr<PaddlePredictor> Clone() = 0;
 
   // Destroy the Predictor.
-  virtual ~PaddlePredictor() {}
+  virtual ~PaddlePredictor() = default;
 
   // The common configs for all the predictors.
   struct Config {

paddle/contrib/inference/paddle_inference_api_impl.cc

@@ -54,7 +54,8 @@ std::string num2str(T a) {
 }
 }  // namespace
 
-bool NativePaddlePredictor::Init() {
+bool NativePaddlePredictor::Init(
+    std::shared_ptr<framework::Scope> parent_scope) {
   VLOG(3) << "Predictor::init()";
 
   if (config_.use_gpu) {
@@ -62,9 +63,15 @@ bool NativePaddlePredictor::Init() {
   } else {
     place_ = paddle::platform::CPUPlace();
   }
+  if (parent_scope) {
+    scope_ = parent_scope;
+    sub_scope_ = &(parent_scope->NewScope());
+  } else {
     paddle::framework::InitDevices(false);
-  executor_.reset(new paddle::framework::Executor(place_));
     scope_.reset(new paddle::framework::Scope());
+  }
+
+  executor_.reset(new paddle::framework::Executor(place_));
 
   // Initialize the inference program
   if (!config_.model_dir.empty()) {
@@ -83,13 +90,8 @@ bool NativePaddlePredictor::Init() {
     return false;
   }
   ctx_ = executor_->Prepare(*inference_program_, 0);
-
-  // Create temporary variables first, so that the first batch do not need to
-  // create variables in the runtime. This is the logics of the old inference
-  // API.
-  // TODO(Superjomn) this should be modified when `Clone` is valid for
-  // multi-thread application.
-  executor_->CreateVariables(*inference_program_, scope_.get(), 0);
+  executor_->CreateVariables(
+      *inference_program_, sub_scope_ ? sub_scope_ : scope_.get(), 0);
 
   // Get the feed_target_names and fetch_target_names
   feed_target_names_ = inference_program_->GetFeedTargetNames();
@@ -97,6 +99,13 @@ bool NativePaddlePredictor::Init() {
   return true;
 }
 
+NativePaddlePredictor::~NativePaddlePredictor() {
+  if (sub_scope_) {
+    PADDLE_ENFORCE_NOT_NULL(scope_, "Should have parent scope!");
+    scope_->DeleteScope(sub_scope_);
+  }
+};
+
 bool NativePaddlePredictor::Run(const std::vector<PaddleTensor> &inputs,
                                 std::vector<PaddleTensor> *output_data) {
   VLOG(3) << "Predictor::predict";
@@ -121,8 +130,9 @@ bool NativePaddlePredictor::Run(const std::vector<PaddleTensor> &inputs,
   }
   // Run the inference program
   // if share variables, we need not create variables
-  executor_->RunPreparedContext(ctx_.get(),
-                                scope_.get(),
+  executor_->RunPreparedContext(
+      ctx_.get(),
+      sub_scope_ != nullptr ? sub_scope_ : scope_.get(),
       &feed_targets,
       &fetch_targets,
       false /* don't create variable eatch time */);
@@ -138,7 +148,7 @@ std::unique_ptr<PaddlePredictor> NativePaddlePredictor::Clone() {
   VLOG(3) << "Predictor::clone";
   std::unique_ptr<PaddlePredictor> cls(new NativePaddlePredictor(config_));
 
-  if (!dynamic_cast<NativePaddlePredictor *>(cls.get())->Init()) {
+  if (!dynamic_cast<NativePaddlePredictor *>(cls.get())->Init(scope_)) {
     LOG(ERROR) << "fail to call Init";
     return nullptr;
   }
@@ -266,7 +276,7 @@ CreatePaddlePredictor<NativeConfig, PaddleEngineKind::kNative>(
   }
 
   std::unique_ptr<PaddlePredictor> predictor(new NativePaddlePredictor(config));
-  if (!dynamic_cast<NativePaddlePredictor *>(predictor.get())->Init()) {
+  if (!dynamic_cast<NativePaddlePredictor *>(predictor.get())->Init(nullptr)) {
     return nullptr;
   }
   return std::move(predictor);

paddle/contrib/inference/paddle_inference_api_impl.h

@@ -34,14 +34,15 @@ class NativePaddlePredictor : public PaddlePredictor {
   explicit NativePaddlePredictor(const NativeConfig &config)
       : config_(config) {}
 
-  bool Init();
+  // will only create sub scope if have global scope
+  bool Init(std::shared_ptr<framework::Scope> parent_scope);
 
   bool Run(const std::vector<PaddleTensor> &inputs,
            std::vector<PaddleTensor> *output_data) override;
 
   std::unique_ptr<PaddlePredictor> Clone() override;
 
-  ~NativePaddlePredictor() override{};
+  ~NativePaddlePredictor() override;
 
  private:
   bool SetFeed(const std::vector<PaddleTensor> &input_datas,
@@ -52,11 +53,13 @@ class NativePaddlePredictor : public PaddlePredictor {
   NativeConfig config_;
   platform::Place place_;
   std::unique_ptr<framework::Executor> executor_;
-  std::unique_ptr<framework::Scope> scope_;
+  std::shared_ptr<framework::Scope> scope_;
   std::unique_ptr<framework::ExecutorPrepareContext> ctx_;
   std::unique_ptr<framework::ProgramDesc> inference_program_;
   std::vector<std::string> feed_target_names_;
   std::vector<std::string> fetch_target_names_;
+  // Do not use unique_ptr, use parent scope to delete
+  framework::Scope *sub_scope_{nullptr};
 };
 
 }  // namespace paddle