未验证 提交 bb4f8dee 编写于 作者: Z Zeng Jinle 提交者: GitHub

add logs to left var memory size, test=develop (#19722)

上级 428b2b9e
...@@ -25,6 +25,45 @@ ...@@ -25,6 +25,45 @@
namespace paddle { namespace paddle {
namespace framework { namespace framework {
namespace details { namespace details {
static void CollectUniqueAllocations(
const Variable &var,
std::unordered_set<memory::Allocation *> *allocation_set) {
if (var.IsType<LoDTensor>()) {
allocation_set->insert(var.Get<LoDTensor>().Holder().get());
} else if (var.IsType<SelectedRows>()) {
allocation_set->insert(var.Get<SelectedRows>().value().Holder().get());
} else if (var.IsType<LoDTensorArray>()) {
for (auto &t : var.Get<LoDTensorArray>()) {
allocation_set->insert(t.Holder().get());
}
}
}
static void CollectUniqueAllocations(
const Scope &scope,
std::unordered_set<memory::Allocation *> *allocation_set) {
for (auto &var_name : scope.LocalVarNames()) {
CollectUniqueAllocations(*scope.FindVar(var_name), allocation_set);
}
for (auto *kid : scope.kids()) {
CollectUniqueAllocations(*kid, allocation_set);
}
}
static size_t GetScopeVarMemorySize(const Scope &scope) {
std::unordered_set<memory::Allocation *> allocation_set;
CollectUniqueAllocations(scope, &allocation_set);
size_t memory_size = 0;
for (auto *allocation : allocation_set) {
if (allocation) {
memory_size += allocation->size();
}
}
return memory_size;
}
ScopeBufferedSSAGraphExecutor::ScopeBufferedSSAGraphExecutor( ScopeBufferedSSAGraphExecutor::ScopeBufferedSSAGraphExecutor(
ExecutionStrategy strategy, std::vector<Scope *> local_scopes, ExecutionStrategy strategy, std::vector<Scope *> local_scopes,
std::vector<Scope *> local_exec_scopes, std::vector<VariableInfo> var_infos, std::vector<Scope *> local_exec_scopes, std::vector<VariableInfo> var_infos,
...@@ -55,10 +94,27 @@ FeedFetchList ScopeBufferedSSAGraphExecutor::Run( ...@@ -55,10 +94,27 @@ FeedFetchList ScopeBufferedSSAGraphExecutor::Run(
eptr = std::current_exception(); eptr = std::current_exception();
} }
if (VLOG_IS_ON(5)) {
for (auto *scope : local_exec_scopes_) {
VLOG(5) << "Left "
<< string::HumanReadableSize(GetScopeVarMemorySize(*scope))
<< " on scope " << scope << " before deleting";
}
}
++drop_scope_counter_; ++drop_scope_counter_;
if (drop_scope_counter_ == strategy_.num_iteration_per_drop_scope_) { if (drop_scope_counter_ == strategy_.num_iteration_per_drop_scope_) {
DropLocalExeScopes(); DropLocalExeScopes();
} }
if (VLOG_IS_ON(5)) {
for (auto *scope : local_exec_scopes_) {
VLOG(5) << "Left "
<< string::HumanReadableSize(GetScopeVarMemorySize(*scope))
<< " on scope " << scope << " after deleting";
}
}
if (eptr) { if (eptr) {
std::rethrow_exception(eptr); std::rethrow_exception(eptr);
} else { } else {
......
...@@ -317,6 +317,10 @@ if (WITH_MKLDNN) ...@@ -317,6 +317,10 @@ if (WITH_MKLDNN)
add_subdirectory(mkldnn) add_subdirectory(mkldnn)
endif() endif()
if (WITH_TESTING)
set_property(TEST test_parallel_executor_mnist PROPERTY ENVIRONMENT GLOG_vmodule=scope_buffered_ssa_graph_executor=5)
endif()
set_tests_properties(test_parallel_executor_test_while_train test_parallel_executor_mnist set_tests_properties(test_parallel_executor_test_while_train test_parallel_executor_mnist
test_parallel_executor_seresnext_base_gpu test_parallel_executor_seresnext_with_reduce_gpu test_parallel_executor_seresnext_base_gpu test_parallel_executor_seresnext_with_reduce_gpu
test_parallel_executor_seresnext_with_fuse_all_reduce_gpu test_parallel_executor_seresnext_with_fuse_all_reduce_gpu
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册