add logs to left var memory size, test=develop (#19722)

paddle/fluid/framework/details/scope_buffered_ssa_graph_executor.cc

@@ -25,6 +25,45 @@
 namespace paddle {
 namespace framework {
 namespace details {
+
+static void CollectUniqueAllocations(
+    const Variable &var,
+    std::unordered_set<memory::Allocation *> *allocation_set) {
+  if (var.IsType<LoDTensor>()) {
+    allocation_set->insert(var.Get<LoDTensor>().Holder().get());
+  } else if (var.IsType<SelectedRows>()) {
+    allocation_set->insert(var.Get<SelectedRows>().value().Holder().get());
+  } else if (var.IsType<LoDTensorArray>()) {
+    for (auto &t : var.Get<LoDTensorArray>()) {
+      allocation_set->insert(t.Holder().get());
+    }
+  }
+}
+
+static void CollectUniqueAllocations(
+    const Scope &scope,
+    std::unordered_set<memory::Allocation *> *allocation_set) {
+  for (auto &var_name : scope.LocalVarNames()) {
+    CollectUniqueAllocations(*scope.FindVar(var_name), allocation_set);
+  }
+
+  for (auto *kid : scope.kids()) {
+    CollectUniqueAllocations(*kid, allocation_set);
+  }
+}
+
+static size_t GetScopeVarMemorySize(const Scope &scope) {
+  std::unordered_set<memory::Allocation *> allocation_set;
+  CollectUniqueAllocations(scope, &allocation_set);
+  size_t memory_size = 0;
+  for (auto *allocation : allocation_set) {
+    if (allocation) {
+      memory_size += allocation->size();
+    }
+  }
+  return memory_size;
+}
+
 ScopeBufferedSSAGraphExecutor::ScopeBufferedSSAGraphExecutor(
     ExecutionStrategy strategy, std::vector<Scope *> local_scopes,
     std::vector<Scope *> local_exec_scopes, std::vector<VariableInfo> var_infos,
@@ -55,10 +94,27 @@ FeedFetchList ScopeBufferedSSAGraphExecutor::Run(
     eptr = std::current_exception();
   }
 
+  if (VLOG_IS_ON(5)) {
+    for (auto *scope : local_exec_scopes_) {
+      VLOG(5) << "Left "
+              << string::HumanReadableSize(GetScopeVarMemorySize(*scope))
+              << " on scope " << scope << " before deleting";
+    }
+  }
+
   ++drop_scope_counter_;
   if (drop_scope_counter_ == strategy_.num_iteration_per_drop_scope_) {
     DropLocalExeScopes();
   }
+
+  if (VLOG_IS_ON(5)) {
+    for (auto *scope : local_exec_scopes_) {
+      VLOG(5) << "Left "
+              << string::HumanReadableSize(GetScopeVarMemorySize(*scope))
+              << " on scope " << scope << " after deleting";
+    }
+  }
+
   if (eptr) {
     std::rethrow_exception(eptr);
   } else {

python/paddle/fluid/tests/unittests/CMakeLists.txt

@@ -317,6 +317,10 @@ if (WITH_MKLDNN)
     add_subdirectory(mkldnn)
 endif()
 
+if (WITH_TESTING)
+    set_property(TEST test_parallel_executor_mnist PROPERTY ENVIRONMENT GLOG_vmodule=scope_buffered_ssa_graph_executor=5)
+endif()
+
 set_tests_properties(test_parallel_executor_test_while_train test_parallel_executor_mnist
         test_parallel_executor_seresnext_base_gpu test_parallel_executor_seresnext_with_reduce_gpu
         test_parallel_executor_seresnext_with_fuse_all_reduce_gpu