Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
afeixing77
Paddle
提交
8175983e
P
Paddle
项目概览
afeixing77
/
Paddle
与 Fork 源项目一致
Fork自
PaddlePaddle / Paddle
通知
1
Star
0
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
P
Paddle
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
未验证
提交
8175983e
编写于
12月 12, 2018
作者:
Y
Yu Yang
提交者:
GitHub
12月 12, 2018
浏览文件
操作
浏览文件
下载
差异文件
Merge pull request #14814 from reyoung/feature/gprof
Add gperftools supports for PE
上级
eeca721a
9f8d8685
变更
6
隐藏空白更改
内联
并排
Showing
6 changed file
with
160 addition
and
41 deletion
+160
-41
CMakeLists.txt
CMakeLists.txt
+7
-1
cmake/FindGperftools.cmake
cmake/FindGperftools.cmake
+63
-0
cmake/generic.cmake
cmake/generic.cmake
+16
-0
paddle/fluid/framework/parallel_executor.cc
paddle/fluid/framework/parallel_executor.cc
+30
-1
paddle/fluid/platform/enforce.h
paddle/fluid/platform/enforce.h
+42
-38
python/paddle/fluid/__init__.py
python/paddle/fluid/__init__.py
+2
-1
未找到文件。
CMakeLists.txt
浏览文件 @
8175983e
...
...
@@ -54,7 +54,7 @@ option(WITH_PYTHON "Compile PaddlePaddle with python interpreter" ON)
option
(
WITH_DOUBLE
"Compile PaddlePaddle with double precision"
OFF
)
option
(
WITH_RDMA
"Compile PaddlePaddle with RDMA support"
OFF
)
option
(
WITH_TIMER
"Compile PaddlePaddle with stats timer"
OFF
)
option
(
WITH_PROFILER
"Compile PaddlePaddle with GPU profiler"
OFF
)
option
(
WITH_PROFILER
"Compile PaddlePaddle with GPU profiler
and gperftools
"
OFF
)
option
(
WITH_DOC
"Compile PaddlePaddle with documentation"
OFF
)
option
(
WITH_COVERAGE
"Compile PaddlePaddle with code coverage"
OFF
)
option
(
COVERALLS_UPLOAD
"Package code coverage data to coveralls"
OFF
)
...
...
@@ -254,6 +254,12 @@ elseif()
set
(
WITH_ANAKIN OFF CACHE STRING
"Anakin is used in MKL only now."
FORCE
)
endif
()
if
(
WITH_PROFILER
)
find_package
(
Gperftools REQUIRED
)
include_directories
(
${
GPERFTOOLS_INCLUDE_DIR
}
)
add_definitions
(
-DWITH_GPERFTOOLS
)
endif
()
include
(
generic
)
# simplify cmake module
include
(
package
)
# set paddle packages
include
(
ccache
)
# set ccache for compilation
...
...
cmake/FindGperftools.cmake
0 → 100644
浏览文件 @
8175983e
# Tries to find Gperftools.
#
# Usage of this module as follows:
#
# find_package(Gperftools)
#
# Variables used by this module, they can change the default behaviour and need
# to be set before calling find_package:
#
# Gperftools_ROOT_DIR Set this variable to the root installation of
# Gperftools if the module has problems finding
# the proper installation path.
#
# Variables defined by this module:
#
# GPERFTOOLS_FOUND System has Gperftools libs/headers
# GPERFTOOLS_LIBRARIES The Gperftools libraries (tcmalloc & profiler)
# GPERFTOOLS_INCLUDE_DIR The location of Gperftools headers
find_library
(
GPERFTOOLS_TCMALLOC
NAMES tcmalloc
HINTS
${
Gperftools_ROOT_DIR
}
/lib
)
find_library
(
GPERFTOOLS_PROFILER
NAMES profiler
HINTS
${
Gperftools_ROOT_DIR
}
/lib
)
find_library
(
GPERFTOOLS_TCMALLOC_AND_PROFILER
NAMES tcmalloc_and_profiler
HINTS
${
Gperftools_ROOT_DIR
}
/lib
)
find_path
(
GPERFTOOLS_INCLUDE_DIR
NAMES gperftools/heap-profiler.h
HINTS
${
Gperftools_ROOT_DIR
}
/include
)
set
(
GPERFTOOLS_LIBRARIES
${
GPERFTOOLS_TCMALLOC_AND_PROFILER
}
)
include
(
FindPackageHandleStandardArgs
)
find_package_handle_standard_args
(
Gperftools
DEFAULT_MSG
GPERFTOOLS_LIBRARIES
GPERFTOOLS_INCLUDE_DIR
)
mark_as_advanced
(
Gperftools_ROOT_DIR
GPERFTOOLS_TCMALLOC
GPERFTOOLS_PROFILER
GPERFTOOLS_TCMALLOC_AND_PROFILER
GPERFTOOLS_LIBRARIES
GPERFTOOLS_INCLUDE_DIR
)
# create IMPORTED targets
if
(
Gperftools_FOUND AND NOT TARGET gperftools::tcmalloc
)
add_library
(
gperftools::tcmalloc UNKNOWN IMPORTED
)
set_target_properties
(
gperftools::tcmalloc PROPERTIES
IMPORTED_LOCATION
${
GPERFTOOLS_TCMALLOC
}
INTERFACE_INCLUDE_DIRECTORIES
"
${
GPERFTOOLS_INCLUDE_DIR
}
"
)
add_library
(
gperftools::profiler UNKNOWN IMPORTED
)
set_target_properties
(
gperftools::profiler PROPERTIES
IMPORTED_LOCATION
${
GPERFTOOLS_PROFILER
}
INTERFACE_INCLUDE_DIRECTORIES
"
${
GPERFTOOLS_INCLUDE_DIR
}
"
)
endif
()
cmake/generic.cmake
浏览文件 @
8175983e
...
...
@@ -110,6 +110,14 @@ function(find_fluid_modules TARGET_NAME)
endif
()
endfunction
(
find_fluid_modules
)
function
(
common_link TARGET_NAME
)
if
(
WITH_PROFILER
)
target_link_libraries
(
${
TARGET_NAME
}
gperftools::profiler
)
endif
()
endfunction
()
# find all third_party modules is used for paddle static library
# for reduce the dependency when building the inference libs.
set_property
(
GLOBAL PROPERTY FLUID_THIRD_PARTY
)
...
...
@@ -274,6 +282,7 @@ function(cc_library TARGET_NAME)
endif
()
target_link_libraries
(
${
TARGET_NAME
}
${
cc_library_DEPS
}
)
add_dependencies
(
${
TARGET_NAME
}
${
cc_library_DEPS
}
)
common_link
(
${
TARGET_NAME
}
)
endif
()
# cpplint code style
...
...
@@ -340,6 +349,7 @@ function(cc_binary TARGET_NAME)
if
(
cc_binary_DEPS
)
target_link_libraries
(
${
TARGET_NAME
}
${
cc_binary_DEPS
}
)
add_dependencies
(
${
TARGET_NAME
}
${
cc_binary_DEPS
}
)
common_link
(
${
TARGET_NAME
}
)
endif
()
endfunction
(
cc_binary
)
...
...
@@ -362,6 +372,7 @@ function(cc_test TARGET_NAME)
target_link_libraries
(
${
TARGET_NAME
}
${
win32_deps
}
)
endif
(
WIN32
)
add_dependencies
(
${
TARGET_NAME
}
${
cc_test_DEPS
}
paddle_gtest_main lod_tensor memory gtest gflags glog
)
common_link
(
${
TARGET_NAME
}
)
add_test
(
NAME
${
TARGET_NAME
}
COMMAND
${
TARGET_NAME
}
${
cc_test_ARGS
}
WORKING_DIRECTORY
${
CMAKE_CURRENT_BINARY_DIR
}
)
...
...
@@ -420,6 +431,7 @@ function(nv_binary TARGET_NAME)
if
(
nv_binary_DEPS
)
target_link_libraries
(
${
TARGET_NAME
}
${
nv_binary_DEPS
}
)
add_dependencies
(
${
TARGET_NAME
}
${
nv_binary_DEPS
}
)
common_link
(
${
TARGET_NAME
}
)
endif
()
endif
()
endfunction
(
nv_binary
)
...
...
@@ -433,6 +445,7 @@ function(nv_test TARGET_NAME)
cuda_add_executable
(
${
TARGET_NAME
}
${
nv_test_SRCS
}
)
target_link_libraries
(
${
TARGET_NAME
}
${
nv_test_DEPS
}
paddle_gtest_main lod_tensor memory gtest gflags glog
)
add_dependencies
(
${
TARGET_NAME
}
${
nv_test_DEPS
}
paddle_gtest_main lod_tensor memory gtest gflags glog
)
common_link
(
${
TARGET_NAME
}
)
add_test
(
${
TARGET_NAME
}
${
TARGET_NAME
}
)
if
(
nv_test_SERIAL
)
set_property
(
TEST
${
TARGET_NAME
}
PROPERTY RUN_SERIAL 1
)
...
...
@@ -499,6 +512,7 @@ function(hip_binary TARGET_NAME)
if
(
hip_binary_DEPS
)
target_link_libraries
(
${
TARGET_NAME
}
${
hip_binary_DEPS
}
)
add_dependencies
(
${
TARGET_NAME
}
${
hip_binary_DEPS
}
)
common_link
(
${
TARGET_NAME
}
)
endif
()
endif
()
endfunction
(
hip_binary
)
...
...
@@ -518,6 +532,7 @@ function(hip_test TARGET_NAME)
set_target_properties
(
${
TARGET_NAME
}
PROPERTIES LINKER_LANGUAGE HIP
)
target_link_libraries
(
${
TARGET_NAME
}
${
hip_test_DEPS
}
paddle_gtest_main memory gtest gflags
)
add_dependencies
(
${
TARGET_NAME
}
${
hip_test_DEPS
}
paddle_gtest_main memory gtest gflags
)
common_link
(
${
TARGET_NAME
}
)
add_test
(
${
TARGET_NAME
}
${
TARGET_NAME
}
)
endif
()
endfunction
(
hip_test
)
...
...
@@ -560,6 +575,7 @@ function(go_library TARGET_NAME)
endif
()
if
(
go_library_DEPS
)
add_dependencies
(
${
TARGET_NAME
}
${
go_library_DEPS
}
)
common_link
(
${
TARGET_NAME
}
)
endif
(
go_library_DEPS
)
# The "source file" of the library is `${dummyfile}` which never
...
...
paddle/fluid/framework/parallel_executor.cc
浏览文件 @
8175983e
...
...
@@ -30,13 +30,36 @@ limitations under the License. */
#include "paddle/fluid/framework/details/threaded_ssa_graph_executor.h"
#include "paddle/fluid/platform/profiler.h"
#ifdef WITH_GPERFTOOLS
#include "gperftools/profiler.h"
#endif
DEFINE_string
(
pe_profile_fname
,
""
,
"Profiler filename for PE, which generated by gperftools."
"Only valid when compiled `WITH_PRIFILER=ON`. Empty if disable."
);
namespace
paddle
{
namespace
framework
{
static
std
::
once_flag
gProfileOnce
;
#ifdef WITH_GPERFTOOLS
static
bool
gProfileStarted
=
false
;
#endif
class
ParallelExecutorPrivate
{
public:
explicit
ParallelExecutorPrivate
(
const
std
::
vector
<
platform
::
Place
>
&
places
)
:
places_
(
places
)
{}
:
places_
(
places
)
{
if
(
!
FLAGS_pe_profile_fname
.
empty
())
{
std
::
call_once
(
gProfileOnce
,
[]
{
#ifdef WITH_GPERFTOOLS
ProfilerStart
(
FLAGS_pe_profile_fname
.
c_str
());
gProfileStarted
=
true
;
#else
LOG
(
WARNING
)
<<
"Paddle is not compiled with gperftools. "
"FLAGS_pe_profile_fname will be ignored"
;
#endif
});
}
}
~
ParallelExecutorPrivate
()
{
if
(
own_local_scope_
)
{
...
...
@@ -270,6 +293,12 @@ void ParallelExecutor::BCastParamsToDevices(
void
ParallelExecutor
::
Run
(
const
std
::
vector
<
std
::
string
>
&
fetch_tensors
,
const
std
::
string
&
fetched_var_name
)
{
#ifdef WITH_GPERFTOOLS
if
(
gProfileStarted
)
{
ProfilerFlush
();
}
#endif
platform
::
RecordBlock
b
(
0
);
#ifdef PADDLE_WITH_CUDA
if
(
!
gcs_
.
empty
())
{
...
...
paddle/fluid/platform/enforce.h
浏览文件 @
8175983e
...
...
@@ -62,45 +62,54 @@ inline std::string demangle(std::string name) { return name; }
#endif
struct
EnforceNotMet
:
public
std
::
exception
{
std
::
exception_ptr
exp_
;
std
::
string
err_str_
;
EnforceNotMet
(
std
::
exception_ptr
e
,
const
char
*
f
,
int
l
)
:
exp_
(
e
)
{
static
constexpr
int
TRACE_STACK_LIMIT
=
100
;
EnforceNotMet
(
std
::
exception_ptr
e
,
const
char
*
f
,
int
l
)
{
try
{
std
::
rethrow_exception
(
exp_
);
}
catch
(
const
std
::
exception
&
exp
)
{
std
::
ostringstream
sout
;
std
::
rethrow_exception
(
e
);
}
catch
(
std
::
exception
&
e
)
{
Init
(
e
.
what
(),
f
,
l
);
}
}
sout
<<
string
::
Sprintf
(
"%s at [%s:%d]"
,
exp
.
what
(),
f
,
l
)
<<
std
::
endl
;
sout
<<
"PaddlePaddle Call Stacks: "
<<
std
::
endl
;
template
<
typename
...
ARGS
>
EnforceNotMet
(
const
char
*
f
,
int
l
,
ARGS
...
args
)
{
Init
(
string
::
Sprintf
(
args
...),
f
,
l
);
}
const
char
*
what
()
const
noexcept
override
{
return
err_str_
.
c_str
();
}
private:
template
<
typename
StrType
>
inline
void
Init
(
StrType
what
,
const
char
*
f
,
int
l
)
{
static
constexpr
int
TRACE_STACK_LIMIT
=
100
;
std
::
ostringstream
sout
;
sout
<<
string
::
Sprintf
(
"%s at [%s:%d]"
,
what
,
f
,
l
)
<<
std
::
endl
;
sout
<<
"PaddlePaddle Call Stacks: "
<<
std
::
endl
;
#if !defined(_WIN32)
void
*
call_stack
[
TRACE_STACK_LIMIT
];
auto
size
=
backtrace
(
call_stack
,
TRACE_STACK_LIMIT
);
auto
symbols
=
backtrace_symbols
(
call_stack
,
size
);
Dl_info
info
;
for
(
int
i
=
0
;
i
<
size
;
++
i
)
{
if
(
dladdr
(
call_stack
[
i
],
&
info
)
&&
info
.
dli_sname
)
{
auto
demangled
=
demangle
(
info
.
dli_sname
);
auto
addr_offset
=
static_cast
<
char
*>
(
call_stack
[
i
])
-
static_cast
<
char
*>
(
info
.
dli_saddr
);
sout
<<
string
::
Sprintf
(
"%-3d %*0p %s + %zd
\n
"
,
i
,
2
+
sizeof
(
void
*
)
*
2
,
call_stack
[
i
],
demangled
,
addr_offset
);
}
else
{
sout
<<
string
::
Sprintf
(
"%-3d %*0p
\n
"
,
i
,
2
+
sizeof
(
void
*
)
*
2
,
call_stack
[
i
]);
}
void
*
call_stack
[
TRACE_STACK_LIMIT
];
auto
size
=
backtrace
(
call_stack
,
TRACE_STACK_LIMIT
);
auto
symbols
=
backtrace_symbols
(
call_stack
,
size
);
Dl_info
info
;
for
(
int
i
=
0
;
i
<
size
;
++
i
)
{
if
(
dladdr
(
call_stack
[
i
],
&
info
)
&&
info
.
dli_sname
)
{
auto
demangled
=
demangle
(
info
.
dli_sname
);
auto
addr_offset
=
static_cast
<
char
*>
(
call_stack
[
i
])
-
static_cast
<
char
*>
(
info
.
dli_saddr
);
sout
<<
string
::
Sprintf
(
"%-3d %*0p %s + %zd
\n
"
,
i
,
2
+
sizeof
(
void
*
)
*
2
,
call_stack
[
i
],
demangled
,
addr_offset
);
}
else
{
sout
<<
string
::
Sprintf
(
"%-3d %*0p
\n
"
,
i
,
2
+
sizeof
(
void
*
)
*
2
,
call_stack
[
i
]);
}
free
(
symbols
);
}
free
(
symbols
);
#else
sout
<<
"Windows not support stack backtrace yet."
;
sout
<<
"Windows not support stack backtrace yet."
;
#endif
err_str_
=
sout
.
str
();
}
err_str_
=
sout
.
str
();
}
const
char
*
what
()
const
noexcept
{
return
err_str_
.
c_str
();
}
};
struct
EOFException
:
public
std
::
exception
{
...
...
@@ -242,13 +251,8 @@ inline void throw_on_error(T e) {
throw_on_error
(
e
,
""
);
}
#define PADDLE_THROW(...) \
do { \
throw ::paddle::platform::EnforceNotMet( \
std::make_exception_ptr( \
std::runtime_error(paddle::string::Sprintf(__VA_ARGS__))), \
__FILE__, __LINE__); \
} while (false)
#define PADDLE_THROW(...) \
throw ::paddle::platform::EnforceNotMet(__FILE__, __LINE__, __VA_ARGS__)
#ifndef REPLACE_ENFORCE_GLOG
#define PADDLE_ENFORCE(...) \
...
...
python/paddle/fluid/__init__.py
浏览文件 @
8175983e
...
...
@@ -127,7 +127,8 @@ def __bootstrap__():
'use_ngraph'
,
'initial_cpu_memory_in_mb'
,
'init_allocated_mem'
,
'free_idle_memory'
,
'paddle_num_threads'
,
"dist_threadpool_size"
,
'eager_delete_tensor_gb'
,
'allocator_strategy'
,
'reader_queue_speed_test_mode'
,
'print_sub_graph_dir'
'reader_queue_speed_test_mode'
,
'print_sub_graph_dir'
,
'pe_profile_fname'
]
if
'Darwin'
not
in
sysstr
:
read_env_flags
.
append
(
'use_pinned_memory'
)
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录