Merge branch 'clone_predictor' of https://github.com/DannyIsFunny/Paddle-Lite into test_result

lite/api/cxx_api.cc

@@ -30,14 +30,14 @@ void Predictor::SaveModel(const std::string &dir,
   if (!program_) {
     GenRuntimeProgram();
   }
-  program_->SaveOpInfosToProgram(&program_desc_);
-  program_->UpdateVarsOfProgram(&program_desc_);
+  program_->SaveOpInfosToProgram(program_desc_.get());
+  program_->UpdateVarsOfProgram(program_desc_.get());
   switch (model_type) {
     case lite_api::LiteModelType::kProtobuf:
-      SaveModelPb(dir, *program_->exec_scope(), program_desc_, true);
+      SaveModelPb(dir, *program_->exec_scope(), *program_desc_.get(), true);
       break;
     case lite_api::LiteModelType::kNaiveBuffer:
-      SaveModelNaive(dir, *program_->exec_scope(), program_desc_);
+      SaveModelNaive(dir, *program_->exec_scope(), *program_desc_.get());
       break;
     default:
       LOG(FATAL) << "Unknown model type";
@@ -231,9 +231,8 @@ std::vector<const lite::Tensor *> Predictor::GetOutputs() const {
 #endif
 
 const cpp::ProgramDesc &Predictor::program_desc() const {
-  return program_desc_;
+  return *program_desc_.get();
 }
-
 const RuntimeProgram &Predictor::runtime_program() const { return *program_; }
 
 void Predictor::Build(const lite_api::CxxConfig &config,
@@ -275,14 +274,14 @@ void Predictor::Build(const std::string &model_path,
                   model_file,
                   param_file,
                   scope_.get(),
-                  &program_desc_,
+                  program_desc_.get(),
                   combined_param,
                   model_from_memory);
     } break;
     case lite_api::LiteModelType::kNaiveBuffer:
       CHECK(!model_path.empty())
           << "NaiveBuffer backend only supported combined param";
-      LoadModelNaiveFromFile(model_path, scope_.get(), &program_desc_);
+      LoadModelNaiveFromFile(model_path, scope_.get(), program_desc_.get());
       break;
     default:
       LOG(FATAL) << "Unknown model type";
@@ -290,7 +289,7 @@ void Predictor::Build(const std::string &model_path,
   Build(program_desc_, valid_places, passes);
 }
 
-void Predictor::Build(const cpp::ProgramDesc &desc,
+void Predictor::Build(const std::shared_ptr<cpp::ProgramDesc> &desc,
                       const std::vector<Place> &valid_places,
                       const std::vector<std::string> &passes) {
   program_desc_ = desc;
@@ -299,7 +298,6 @@ void Predictor::Build(const cpp::ProgramDesc &desc,
   inner_places.emplace_back(TARGET(kHost), PRECISION(kAny), DATALAYOUT(kAny));
   inner_places.emplace_back(
       TARGET(kHost), PRECISION(kFloat), DATALAYOUT(kNCHW));
-
   const std::vector<std::string> quant_dequant_op = {
       "fake_quantize_abs_max",
       "fake_quantize_range_abs_max",
@@ -308,9 +306,9 @@ void Predictor::Build(const cpp::ProgramDesc &desc,
       "fake_dequantize_max_abs",
       "fake_channel_wise_dequantize_max_abs"};
   bool is_quantized_model = false;
-  for (size_t i = 0; i < program_desc_.BlocksSize() && !is_quantized_model;
+  for (size_t i = 0; i < program_desc_->BlocksSize() && !is_quantized_model;
        ++i) {
-    auto *block_desc = program_desc_.GetBlock<cpp::BlockDesc>(i);
+    auto *block_desc = program_desc_->GetBlock<cpp::BlockDesc>(i);
     for (size_t j = 0; j < block_desc->OpsSize() && !is_quantized_model; ++j) {
       auto *op_desc = block_desc->GetOp<cpp::OpDesc>(j);
       std::string op_type = op_desc->Type();
@@ -325,7 +323,8 @@ void Predictor::Build(const cpp::ProgramDesc &desc,
     inner_places.emplace_back(Place{TARGET(kARM), PRECISION(kInt8)});
   }
 
-  Program program(desc, scope_, inner_places);
+  Program program(*desc.get(), scope_, inner_places);
+  valid_places_ = inner_places;
 
   core::KernelPickFactor factor;
   factor.ConsiderTarget();

lite/api/cxx_api.h

@@ -42,10 +42,24 @@ static const char TAILORD_KERNELS_LIST_NAME[] = ".tailored_kernels_list";
 class LITE_API Predictor {
  public:
   // Create an empty predictor.
-  Predictor() { scope_ = std::make_shared<Scope>(); }
+  Predictor() {
+    scope_ = std::make_shared<Scope>();
+    program_desc_ = std::make_shared<cpp::ProgramDesc>();
+  }
   // Create a predictor with the weight variable scope set.
   explicit Predictor(const std::shared_ptr<lite::Scope>& root_scope)
       : scope_(root_scope) {}
+  Predictor(const std::shared_ptr<cpp::ProgramDesc>& desc,
+            const std::shared_ptr<Scope>& root,
+            const std::vector<Place>& valid_places)
+      : program_desc_(desc), scope_(root) {
+    Program program(*desc.get(), scope_, valid_places);
+    optimizer_ = Optimizer(std::move(program), valid_places);
+    exec_scope_ = optimizer_.exec_scope();
+    GenRuntimeProgram();
+    valid_places_ = valid_places;
+    PrepareFeedFetch();
+  }
 
   // Build from a model, with places set for hardware config.
   void Build(
@@ -63,10 +77,20 @@ class LITE_API Predictor {
       lite_api::LiteModelType model_type = lite_api::LiteModelType::kProtobuf,
       bool memory_from_memory = false);
 
-  void Build(const cpp::ProgramDesc& desc,
+  void Build(const std::shared_ptr<cpp::ProgramDesc>& desc,
              const std::vector<Place>& valid_places,
              const std::vector<std::string>& passes = {});
 
+  std::shared_ptr<Predictor> Clone() const {
+    //    CHECK(program_desc_) << "Both program and scope of current predicotr
+    //    should  be not be nullptr in Clone mode." ;
+    //    CHECK(scope_) << "Both program and scope of current predicotr should
+    //    be not be nullptr in Clone mode.";
+    auto predictor =
+        std::make_shared<Predictor>(program_desc_, scope_, valid_places_);
+    return predictor;
+  }
+
   void GenRuntimeProgram();
 
   // Run the predictor for a single batch of data.
@@ -118,18 +142,21 @@ class LITE_API Predictor {
 
  private:
   Optimizer optimizer_;
-  cpp::ProgramDesc program_desc_;
+  std::shared_ptr<cpp::ProgramDesc> program_desc_;
   std::shared_ptr<Scope> scope_;
   const Scope* exec_scope_;
   std::unique_ptr<RuntimeProgram> program_;
   bool program_generated_{false};
   std::vector<std::string> input_names_;
   std::vector<std::string> output_names_;
+  std::vector<Place> valid_places_;
 };
 
 class CxxPaddleApiImpl : public lite_api::PaddlePredictor {
  public:
   CxxPaddleApiImpl() {}
+  explicit CxxPaddleApiImpl(const std::shared_ptr<Predictor>& raw_predictor)
+      : raw_predictor_(raw_predictor) {}
 
   /// Create a new predictor from a config.
   void Init(const lite_api::CxxConfig& config);
@@ -167,9 +194,10 @@ class CxxPaddleApiImpl : public lite_api::PaddlePredictor {
       bool record_info = false) override;
 
  private:
-  Predictor raw_predictor_;
+  std::shared_ptr<Predictor> raw_predictor_;
   lite_api::CxxConfig config_;
   std::mutex mutex_;
+  bool status_is_cloned_{false};
 };
 
 /*

lite/api/cxx_api_impl.cc

@@ -34,17 +34,21 @@ void CxxPaddleApiImpl::Init(const lite_api::CxxConfig &config) {
 #ifdef LITE_WITH_CUDA
   Env<TARGET(kCUDA)>::Init();
 #endif
-  auto places = config.valid_places();
-  std::vector<std::string> passes{};
-  auto use_layout_preprocess_pass =
-      config.model_dir().find("OPENCL_PRE_PRECESS");
-  VLOG(1) << "use_layout_preprocess_pass:" << use_layout_preprocess_pass;
-  if (places[0].target == TARGET(kOpenCL) &&
-      use_layout_preprocess_pass != std::string::npos) {
-    passes = {"type_layout_cast_preprocess_pass"};
-    VLOG(1) << "add pass:" << passes[0];
+  if (!status_is_cloned_) {
+    auto places = config.valid_places();
+    std::vector<std::string> passes{};
+    auto use_layout_preprocess_pass =
+        config.model_dir().find("OPENCL_PRE_PRECESS");
+    VLOG(1) << "use_layout_preprocess_pass:" << use_layout_preprocess_pass;
+    if (places[0].target == TARGET(kOpenCL) &&
+        use_layout_preprocess_pass != std::string::npos) {
+      passes = {"type_layout_cast_preprocess_pass"};
+      VLOG(1) << "add pass:" << passes[0];
+    }
+    raw_predictor_->Build(config, places, passes);
+  } else {
+    CHECK(raw_predictor_) << "The Predictor can not be nullptr in Clone mode.";
   }
-  raw_predictor_.Build(config, places, passes);
   mode_ = config.power_mode();
   threads_ = config.threads();
 
@@ -61,18 +65,18 @@ void CxxPaddleApiImpl::Init(const lite_api::CxxConfig &config) {
 }
 
 std::unique_ptr<lite_api::Tensor> CxxPaddleApiImpl::GetInput(int i) {
-  auto *x = raw_predictor_.GetInput(i);
+  auto *x = raw_predictor_->GetInput(i);
   return std::unique_ptr<lite_api::Tensor>(new lite_api::Tensor(x));
 }
 
 std::unique_ptr<const lite_api::Tensor> CxxPaddleApiImpl::GetOutput(
     int i) const {
-  const auto *x = raw_predictor_.GetOutput(i);
+  const auto *x = raw_predictor_->GetOutput(i);
   return std::unique_ptr<lite_api::Tensor>(new lite_api::Tensor(x));
 }
 
 std::vector<std::string> CxxPaddleApiImpl::GetInputNames() {
-  return raw_predictor_.GetInputNames();
+  return raw_predictor_->GetInputNames();
 }
 
 std::vector<std::string> CxxPaddleApiImpl::GetParamNames() {
@@ -80,19 +84,21 @@ std::vector<std::string> CxxPaddleApiImpl::GetParamNames() {
 }
 
 std::vector<std::string> CxxPaddleApiImpl::GetOutputNames() {
-  return raw_predictor_.GetOutputNames();
+  return raw_predictor_->GetOutputNames();
 }
 
 void CxxPaddleApiImpl::Run() {
 #ifdef LITE_WITH_ARM
   lite::DeviceInfo::Global().SetRunMode(mode_, threads_);
 #endif
-  raw_predictor_.Run();
+  raw_predictor_->Run();
 }
 
 std::shared_ptr<lite_api::PaddlePredictor> CxxPaddleApiImpl::Clone() {
   std::lock_guard<std::mutex> lock(mutex_);
-  auto predictor = std::make_shared<lite::CxxPaddleApiImpl>();
+  auto predictor =
+      std::make_shared<lite::CxxPaddleApiImpl>(raw_predictor_->Clone());
+  status_is_cloned_ = true;
   predictor->Init(config_);
   return predictor;
 }
@@ -101,7 +107,7 @@ std::string CxxPaddleApiImpl::GetVersion() const { return version(); }
 
 std::unique_ptr<const lite_api::Tensor> CxxPaddleApiImpl::GetTensor(
     const std::string &name) const {
-  auto *x = raw_predictor_.GetTensor(name);
+  auto *x = raw_predictor_->GetTensor(name);
   return std::unique_ptr<const lite_api::Tensor>(new lite_api::Tensor(x));
 }
 
@@ -114,13 +120,13 @@ std::unique_ptr<lite_api::Tensor> CxxPaddleApiImpl::GetMutableTensor(
 std::unique_ptr<lite_api::Tensor> CxxPaddleApiImpl::GetInputByName(
     const std::string &name) {
   return std::unique_ptr<lite_api::Tensor>(
-      new lite_api::Tensor(raw_predictor_.GetInputByName(name)));
+      new lite_api::Tensor(raw_predictor_->GetInputByName(name)));
 }
 
 void CxxPaddleApiImpl::SaveOptimizedModel(const std::string &model_dir,
                                           lite_api::LiteModelType model_type,
                                           bool record_info) {
-  raw_predictor_.SaveModel(model_dir, model_type, record_info);
+  raw_predictor_->SaveModel(model_dir, model_type, record_info);
 }
 
 }  // namespace lite

lite/api/cxx_api_test.cc

@@ -53,6 +53,44 @@ TEST(CXXApi, save_model) {
                       lite_api::LiteModelType::kNaiveBuffer);
 }
 
+TEST(CXXApi, clone_predictor) {
+  lite::Predictor predictor;
+  std::vector<Place> valid_places({Place{TARGET(kX86), PRECISION(kFloat)}});
+  predictor.Build(FLAGS_model_dir, "", "", valid_places);
+  auto cloned_predictor = predictor.Clone();
+  // primary predicotr
+  auto* input_tensor = predictor.GetInput(0);
+  input_tensor->Resize(std::vector<int64_t>({1, 100}));
+  auto* data = input_tensor->mutable_data<float>();
+  for (int i = 0; i < 100; i++) {
+    data[i] = 1;
+  }
+
+  predictor.Run();
+  auto* output_tensor = predictor.GetOutput(0);
+  auto output_shape = output_tensor->dims().Vectorize();
+  ASSERT_EQ(output_shape.size(), 2);
+  ASSERT_EQ(output_shape[0], 1);
+  ASSERT_EQ(output_shape[1], 500);
+
+  // cloned predictor
+  auto* cloned_input_tensor = cloned_predictor->GetInput(0);
+  cloned_input_tensor->Resize(std::vector<int64_t>({1, 100}));
+  auto* cloned_data = cloned_input_tensor->mutable_data<float>();
+  for (int i = 0; i < 100; i++) {
+    cloned_data[i] = 1;
+  }
+  cloned_predictor->Run();
+  auto* cloned_output_tensor = cloned_predictor->GetOutput(0);
+
+  int step = 50;
+  for (int i = 0; i < output_tensor->data_size(); i += step) {
+    EXPECT_NEAR(output_tensor->data<float>()[i],
+                cloned_output_tensor->data<float>()[i],
+                1e-6);
+  }
+}
+
 /*TEST(CXXTrainer, train) {
   Place place({TARGET(kHost), PRECISION(kFloat), DATALAYOUT(kNCHW)});
   std::vector<Place> valid_places({place});

lite/core/optimizer.h

@@ -17,6 +17,7 @@
 #include <memory>
 #include <set>
 #include <string>
+#include <utility>
 #include <vector>
 #include "lite/core/mir/generate_program_pass.h"
 #include "lite/core/mir/pass_manager.h"
@@ -37,6 +38,22 @@ namespace lite {
  */
 class Optimizer {
  public:
+  Optimizer() {}
+
+  Optimizer(Program&& program, const std::vector<Place>& valid_places) {
+    program_ = &program;
+    valid_places_ = valid_places;
+    CHECK(!valid_places.empty()) << "At least one valid_place should be set";
+    CHECK(!graph_) << "duplicate optimize found";
+
+    core::KernelPickFactor factor;
+    factor.ConsiderTarget();
+    factor.ConsiderPrecision();
+    factor.ConsiderDataLayout();
+
+    Run(std::move(program), valid_places, factor, {});
+  }
+
   void Run(Program&& program,
            const std::vector<Place>& valid_places,
            core::KernelPickFactor kernel_pick_factor,

lite/core/scope.cc

@@ -13,11 +13,16 @@
 // limitations under the License.
 
 #include "lite/core/scope.h"
+#define SCOPE_KIDS_READER_LOCK lite::fluid::AutoRDLock auto_lock(kids_lock_);
+#define SCOPE_KIDS_WRITER_LOCK lite::fluid::AutoWRLock auto_lock(kids_lock_);
+#define SCOPE_VARS_READER_LOCK lite::fluid::AutoRDLock auto_lock(vars_lock_);
+#define SCOPE_VARS_WRITER_LOCK lite::fluid::AutoWRLock auto_lock(vars_lock_);
 
 namespace paddle {
 namespace lite {
 
 Scope::~Scope() {
+  SCOPE_KIDS_WRITER_LOCK
   for (auto *x : kids_) {
     if (x) {
       delete x;
@@ -26,15 +31,16 @@ Scope::~Scope() {
 }
 
 Scope &Scope::NewScope() const {
+  SCOPE_KIDS_WRITER_LOCK
   kids_.push_back(new Scope);
   kids_.back()->parent_ = this;
   return *kids_.back();
 }
 
 Variable *Scope::Var(const std::string &name) {
+  SCOPE_VARS_WRITER_LOCK
   auto *var = FindVar(name);
   if (var) return var;
-
   // create a new variable.
   vars_.emplace(name, std::unique_ptr<Variable>(new Variable));
   return vars_[name].get();
@@ -44,19 +50,23 @@ Variable *Scope::FindVar(const std::string &name) const {
   Variable *var{nullptr};
   var = FindLocalVar(name);
   const Scope *cur_scope = this;
+  rwlock_->RDLock();
   while (!var && cur_scope->parent()) {
     cur_scope = cur_scope->parent();
     var = cur_scope->FindLocalVar(name);
   }
-
+  rwlock_->UNLock();
   return var;
 }
 
 Variable *Scope::FindLocalVar(const std::string &name) const {
+  rwlock_->RDLock();
   auto it = vars_.find(name);
   if (it != vars_.end()) {
+    rwlock_->UNLock();
     return it->second.get();
   }
+  rwlock_->UNLock();
   return nullptr;
 }
 
@@ -85,8 +95,12 @@ std::vector<std::string> Scope::AttributeVarNames() const {
 
 std::vector<std::string> Scope::LocalVarNames() const {
   std::vector<std::string> keys;
-  for (const auto &item : vars_) {
-    keys.push_back(item.first);
+  {
+    rwlock_->RDLock();
+    for (const auto &item : vars_) {
+      keys.push_back(item.first);
+    }
+    rwlock_->UNLock();
   }
   return keys;
 }

lite/core/scope.h

@@ -20,13 +20,18 @@
 #include <utility>
 #include <vector>
 #include "lite/core/variable.h"
+#include "lite/fluid/rw_lock.h"
 
 namespace paddle {
 namespace lite {
 
 class Scope final {
  public:
-  Scope() {}
+  Scope() {
+    kids_lock_ = new lite::fluid::RWLock;
+    vars_lock_ = new lite::fluid::RWLock;
+    rwlock_.reset(new lite::fluid::RWLock);
+  }
   // delete below two functions to allow pybind to recognise it cannot make a
   // copy
   // link:
@@ -75,6 +80,9 @@ class Scope final {
   mutable std::list<Scope*> kids_;
   const Scope* parent_{nullptr};
   std::unordered_map<std::string, std::unique_ptr<Variable>> vars_;
+  lite::fluid::RWLock* kids_lock_{nullptr};
+  lite::fluid::RWLock* vars_lock_{nullptr};
+  std::unique_ptr<lite::fluid::RWLock> rwlock_{nullptr};
 };
 
 }  // namespace lite