From 9d2bd0ac387080c5a047565ba9f28d380d1c42f1 Mon Sep 17 00:00:00 2001
From: 123malin <malin10@baidu.com>
Date: Wed, 3 Jun 2020 14:05:17 +0800
Subject: [PATCH] =?UTF-8?q?downpour=5Fworker=E5=A2=9E=E5=8A=A0try=5Fcatch?=
 =?UTF-8?q?=E6=9C=BA=E5=88=B6=EF=BC=8C=E6=89=93=E5=8D=B0program=E6=89=80?=
 =?UTF-8?q?=E6=9C=89=E5=8F=82=E6=95=B0=20(#24700)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* test=develop, add try_catch for debug
---
 paddle/fluid/framework/device_worker.cc   |  7 ++--
 paddle/fluid/framework/device_worker.h    |  3 +-
 paddle/fluid/framework/downpour_worker.cc | 43 +++++++++++++++++++++++
 paddle/fluid/framework/hogwild_worker.cc  |  1 +
 4 files changed, 49 insertions(+), 5 deletions(-)

diff --git a/paddle/fluid/framework/device_worker.cc b/paddle/fluid/framework/device_worker.cc
index e39ebf8a7d..5cfe664203 100644
--- a/paddle/fluid/framework/device_worker.cc
+++ b/paddle/fluid/framework/device_worker.cc
@@ -25,7 +25,7 @@ void DeviceWorker::SetDataFeed(DataFeed* data_feed) {
 }
 
 template <typename T>
-std::string PrintLodTensorType(LoDTensor* tensor, int64_t start, int64_t end) {
+std::string PrintLodTensorType(Tensor* tensor, int64_t start, int64_t end) {
   auto count = tensor->numel();
   if (start < 0 || end > count) {
     VLOG(3) << "access violation";
@@ -38,8 +38,7 @@ std::string PrintLodTensorType(LoDTensor* tensor, int64_t start, int64_t end) {
   return os.str();
 }
 
-std::string PrintLodTensorIntType(LoDTensor* tensor, int64_t start,
-                                  int64_t end) {
+std::string PrintLodTensorIntType(Tensor* tensor, int64_t start, int64_t end) {
   auto count = tensor->numel();
   if (start < 0 || end > count) {
     VLOG(3) << "access violation";
@@ -52,7 +51,7 @@ std::string PrintLodTensorIntType(LoDTensor* tensor, int64_t start,
   return os.str();
 }
 
-std::string PrintLodTensor(LoDTensor* tensor, int64_t start, int64_t end) {
+std::string PrintLodTensor(Tensor* tensor, int64_t start, int64_t end) {
   std::string out_val;
   if (tensor->type() == proto::VarType::FP32) {
     out_val = PrintLodTensorType<float>(tensor, start, end);
diff --git a/paddle/fluid/framework/device_worker.h b/paddle/fluid/framework/device_worker.h
index 8d50f476ea..2b4751691b 100644
--- a/paddle/fluid/framework/device_worker.h
+++ b/paddle/fluid/framework/device_worker.h
@@ -45,7 +45,7 @@ limitations under the License. */
 namespace paddle {
 namespace framework {
 
-std::string PrintLodTensor(LoDTensor* tensor, int64_t start, int64_t end);
+std::string PrintLodTensor(Tensor* tensor, int64_t start, int64_t end);
 std::pair<int64_t, int64_t> GetTensorBound(LoDTensor* tensor, int index);
 bool CheckValidOutput(LoDTensor* tensor, size_t batch_size);
 
@@ -171,6 +171,7 @@ class DeviceWorker {
   bool need_dump_field_;
   const std::vector<std::string>* dump_param_;
   const std::vector<std::string>* dump_fields_;
+  std::vector<std::string> all_param_;
 
   int dump_mode_ = 0;
   int dump_interval_ = 10000;
diff --git a/paddle/fluid/framework/downpour_worker.cc b/paddle/fluid/framework/downpour_worker.cc
index 243e7b97c2..cbdfa00652 100644
--- a/paddle/fluid/framework/downpour_worker.cc
+++ b/paddle/fluid/framework/downpour_worker.cc
@@ -771,7 +771,50 @@ void DownpourWorker::TrainFiles() {
         }
       }
       if (!need_skip) {
+#ifdef PADDLE_WITH_PSLIB
+        try {
+          op->Run(*thread_scope_, place_);
+        } catch (std::exception& e) {
+          fprintf(stderr, "error message: %s\n", e.what());
+          auto& ins_id_vec = device_reader_->GetInsIdVec();
+          size_t batch_size = device_reader_->GetCurBatchSize();
+          std::string s = "";
+          for (auto& ins_id : ins_id_vec) {
+            if (s != "") s += ",";
+            s += ins_id;
+          }
+          fprintf(stderr, "batch_size: %zu, ins_ids_vec: %s\n", batch_size,
+                  s.c_str());
+          s = "";
+          for (auto& param : all_param_) {
+            Variable* var = thread_scope_->FindVar(param);
+            if (var == nullptr) {
+              continue;
+            }
+            Tensor* tensor = nullptr;
+            int64_t len = 0;
+            if (var->IsType<framework::LoDTensor>()) {
+              tensor = var->GetMutable<LoDTensor>();
+              len = tensor->numel();
+            } else if (var->IsType<SelectedRows>()) {
+              auto selected_rows = var->GetMutable<SelectedRows>();
+              tensor = selected_rows->mutable_value();
+              len = tensor->numel();
+            }
+            if (!tensor->IsInitialized()) {
+              continue;
+            }
+            s += param + ":" + std::to_string(len) + ":";
+            s += PrintLodTensor(tensor, 0, len);
+            fprintf(stderr, "%s\n", s.c_str());
+            fflush(stderr);
+            s = "";
+          }
+          throw e;
+        }
+#else
         op->Run(*thread_scope_, place_);
+#endif
       }
     }
 
diff --git a/paddle/fluid/framework/hogwild_worker.cc b/paddle/fluid/framework/hogwild_worker.cc
index 4d930337e8..c51f091c54 100644
--- a/paddle/fluid/framework/hogwild_worker.cc
+++ b/paddle/fluid/framework/hogwild_worker.cc
@@ -58,6 +58,7 @@ void HogwildWorker::CreateThreadScope(const ProgramDesc &program) {
   thread_scope_ = &root_scope_->NewScope();
 
   for (auto &var : block.AllVars()) {
+    all_param_.push_back(var->Name());
     if (var->Persistable()) {
       auto *ptr = root_scope_->Var(var->Name());
       InitializeVariable(ptr, var->GetType());
-- 
GitLab