Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
urhero
Paddle
提交
2912d531
P
Paddle
项目概览
urhero
/
Paddle
与 Fork 源项目一致
Fork自
PaddlePaddle / Paddle
通知
1
Star
0
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
P
Paddle
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
2912d531
编写于
12月 13, 2018
作者:
H
heqiaozhi
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
fix code style bug & change pslib.cmake & change Cmakelist adapt pslib
上级
c59cdf3a
变更
7
显示空白变更内容
内联
并排
Showing
7 changed file
with
28 addition
and
20 deletion
+28
-20
CMakeLists.txt
CMakeLists.txt
+13
-6
cmake/external/pslib.cmake
cmake/external/pslib.cmake
+6
-5
paddle/fluid/framework/async_executor.cc
paddle/fluid/framework/async_executor.cc
+3
-4
paddle/fluid/framework/async_executor.h
paddle/fluid/framework/async_executor.h
+2
-2
paddle/fluid/framework/data_feed.cc
paddle/fluid/framework/data_feed.cc
+1
-0
paddle/fluid/framework/executor_thread_worker.cc
paddle/fluid/framework/executor_thread_worker.cc
+2
-2
paddle/fluid/framework/executor_thread_worker.h
paddle/fluid/framework/executor_thread_worker.h
+1
-1
未找到文件。
CMakeLists.txt
浏览文件 @
2912d531
...
...
@@ -65,6 +65,7 @@ option(WITH_GOLANG "Compile PaddlePaddle with GOLANG" OFF)
option
(
GLIDE_INSTALL
"Download and install go dependencies "
ON
)
option
(
USE_NNPACK
"Compile PaddlePaddle with NNPACK library"
OFF
)
option
(
WITH_DISTRIBUTE
"Compile with distributed support"
OFF
)
option
(
WITH_PSLIB
"Compile with pslib support"
OFF
)
option
(
USE_EIGEN_FOR_BLAS
"Use matrix multiplication in Eigen"
OFF
)
option
(
EIGEN_USE_THREADS
"Compile with multi-threaded Eigen"
OFF
)
option
(
WITH_ARM_FP16
"Use half precision support on armv8.2-a cpu"
OFF
)
...
...
@@ -216,9 +217,12 @@ include(external/warpctc) # download, build, install warpctc
include
(
cupti
)
include
(
external/gzstream
)
endif
(
NOT WIN32
)
include
(
external/libmct
)
include
(
external/pslib_brpc
)
include
(
external/pslib
)
if
(
WITH_PSLIB
)
include
(
external/libmct
)
include
(
external/pslib_brpc
)
include
(
external/pslib
)
endif
()
if
(
WITH_DISTRIBUTE
)
if
(
WITH_GRPC
)
...
...
@@ -279,11 +283,14 @@ set(EXTERNAL_LIBS
protobuf
zlib
${
PYTHON_LIBRARIES
}
pslib
pslib_brpc
libmct
)
if
(
WITH_PSLIB
)
list
(
APPEND EXTERNAL_LIBS pslib
)
list
(
APPEND EXTERNAL_LIBS pslib_brpc
)
list
(
APPEND EXTERNAL_LIBS libmct
)
endif
(
WITH_PSLIB
)
if
(
WITH_AMD_GPU
)
find_package
(
HIP
)
include
(
hip
)
...
...
cmake/external/pslib.cmake
浏览文件 @
2912d531
...
...
@@ -30,9 +30,10 @@ SET(PSLIB_PROJECT "extern_pslib")
IF
((
NOT DEFINED PSLIB_VER
)
OR
(
NOT DEFINED PSLIB_URL
))
MESSAGE
(
STATUS
"use pre defined download url"
)
SET
(
PSLIB_VER
"0.1.0"
CACHE STRING
""
FORCE
)
SET
(
PSLIB_URL
"https://raw.githubusercontent.com/PaddlePaddle/Fleet/release/
${
PSLIB_VER
}
/pslib.tar.gz"
CACHE STRING
""
FORCE
)
SET
(
PSLIB_NAME
"pslib"
CACHE STRING
""
FORCE
)
SET
(
PSLIB_URL
"https://raw.githubusercontent.com/PaddlePaddle/Fleet/release/
${
PSLIB_VER
}
/
${
PSLIB_NAME
}
.tar.gz"
CACHE STRING
""
FORCE
)
ENDIF
()
MESSAGE
(
STATUS
"PSLIB_
VER:
${
PSLIB_VER
}
, PSLIB_URL:
${
PSLIB_URL
}
"
)
MESSAGE
(
STATUS
"PSLIB_
NAME:
${
PSLIB_NAME
}
, PSLIB_URL:
${
PSLIB_URL
}
"
)
SET
(
PSLIB_SOURCE_DIR
"
${
THIRD_PARTY_PATH
}
/pslib"
)
SET
(
PSLIB_DOWNLOAD_DIR
"
${
PSLIB_SOURCE_DIR
}
/src/
${
PSLIB_PROJECT
}
"
)
SET
(
PSLIB_DST_DIR
"pslib"
)
...
...
@@ -50,7 +51,7 @@ INCLUDE_DIRECTORIES(${PSLIB_INC_DIR})
FILE
(
WRITE
${
PSLIB_DOWNLOAD_DIR
}
/CMakeLists.txt
"PROJECT(PSLIB)
\n
"
"cmake_minimum_required(VERSION 3.0)
\n
"
"install(DIRECTORY
${
PSLIB_
VER
}
/include
${
PSLIB_VER
}
/lib
\n
"
"install(DIRECTORY
${
PSLIB_
NAME
}
/include
${
PSLIB_NAME
}
/lib
\n
"
" DESTINATION
${
PSLIB_DST_DIR
}
)
\n
"
)
ExternalProject_Add
(
...
...
@@ -58,8 +59,8 @@ ExternalProject_Add(
${
EXTERNAL_PROJECT_LOG_ARGS
}
PREFIX
${
PSLIB_SOURCE_DIR
}
DOWNLOAD_DIR
${
PSLIB_DOWNLOAD_DIR
}
DOWNLOAD_COMMAND wget --no-check-certificate
${
PSLIB_URL
}
-c -q -O
${
PSLIB_
VER
}
.tar.gz
&& tar zxvf
${
PSLIB_
VER
}
.tar.gz
DOWNLOAD_COMMAND wget --no-check-certificate
${
PSLIB_URL
}
-c -q -O
${
PSLIB_
NAME
}
.tar.gz
&& tar zxvf
${
PSLIB_
NAME
}
.tar.gz
DOWNLOAD_NO_PROGRESS 1
UPDATE_COMMAND
""
CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=
${
PSLIB_INSTALL_ROOT
}
...
...
paddle/fluid/framework/async_executor.cc
浏览文件 @
2912d531
...
...
@@ -50,7 +50,6 @@ void AsyncExecutor::CreateThreads(
worker
->
BindingDataFeedMemory
();
worker
->
SetPSlibPtr
(
_pslib_ptr
);
worker
->
SetPullDenseThread
(
_pull_dense_thread
);
worker
->
BindingSlotVariableMemory
();
worker
->
SetParamConfig
(
&
_param_config
);
}
...
...
@@ -79,7 +78,7 @@ void AsyncExecutor::InitWorker(const std::string& dist_desc,
_pslib_ptr
=
std
::
shared_ptr
<
paddle
::
distributed
::
PSlib
>
(
new
paddle
::
distributed
::
PSlib
());
_pslib_ptr
->
init_worker
(
dist_desc
,
host_sign_list
.
data
(
),
node_num
,
index
);
dist_desc
,
(
uint64_t
*
)(
host_sign_list
.
data
()
),
node_num
,
index
);
InitParamConfig
();
}
...
...
@@ -93,8 +92,8 @@ void AsyncExecutor::StopServer() {
}
void
AsyncExecutor
::
GatherServers
(
std
::
vector
<
uint64_t
>&
host_sign_list
,
int
node_num
)
{
_pslib_ptr
->
gather_servers
(
host_sign_list
.
data
(
),
node_num
);
const
std
::
vector
<
uint64_t
>&
host_sign_list
,
int
node_num
)
{
_pslib_ptr
->
gather_servers
(
(
uint64_t
*
)(
host_sign_list
.
data
()
),
node_num
);
}
void
AsyncExecutor
::
InitParamConfig
()
{
...
...
paddle/fluid/framework/async_executor.h
浏览文件 @
2912d531
...
...
@@ -43,9 +43,9 @@ inline std::default_random_engine& local_random_engine() {
struct
engine_wrapper_t
{
std
::
default_random_engine
engine
;
engine_wrapper_t
()
{
static
std
::
atomic
<
uint64
>
x
(
0
);
static
std
::
atomic
<
uint64
_t
>
x
(
0
);
std
::
seed_seq
sseq
=
{
x
++
,
x
++
,
x
++
,
static_cast
<
uint64
>
(
current_realtime
()
*
1000
)};
static_cast
<
uint64
_t
>
(
current_realtime
()
*
1000
)};
engine
.
seed
(
sseq
);
}
};
...
...
paddle/fluid/framework/data_feed.cc
浏览文件 @
2912d531
...
...
@@ -68,6 +68,7 @@ bool DataFeed::PickOneFile(std::string* filename) {
return
false
;
}
*
filename
=
filelist_
[
file_idx_
++
];
LOG
(
ERROR
)
<<
"pick file:"
<<
*
filename
;
return
true
;
}
...
...
paddle/fluid/framework/executor_thread_worker.cc
浏览文件 @
2912d531
...
...
@@ -637,7 +637,7 @@ void AsyncExecutorThreadWorker::collect_feasign_info(
}
void
AsyncExecutorThreadWorker
::
check_pull_push_memory
(
std
::
vector
<
uint64_t
>&
features
,
const
std
::
vector
<
uint64_t
>&
features
,
std
::
vector
<
std
::
vector
<
float
>>&
push_g
,
int
dim
)
{
push_g
.
resize
(
features
.
size
()
+
1
);
...
...
@@ -647,7 +647,7 @@ void AsyncExecutorThreadWorker::check_pull_push_memory(
}
void
AsyncExecutorThreadWorker
::
check_pull_push_memory
(
std
::
vector
<
uint64_t
>&
features
,
const
std
::
vector
<
uint64_t
>&
features
,
std
::
vector
<
float
*>&
push_g
,
int
dim
)
{
if
(
features
.
size
()
>
push_g
.
size
())
{
...
...
paddle/fluid/framework/executor_thread_worker.h
浏览文件 @
2912d531
...
...
@@ -155,7 +155,7 @@ class ExecutorThreadWorker {
// set fetch variable names from python interface assigned by users
void
SetFetchVarNames
(
const
std
::
vector
<
std
::
string
>&
fetch_var_names
);
virtual
void
SetPSlibPtr
(
std
::
shared_ptr
<
paddle
::
distributed
::
PSlib
>
pslib_ptr
);
std
::
shared_ptr
<
paddle
::
distributed
::
PSlib
>
pslib_ptr
)
{}
;
virtual
void
SetPullDenseThread
(
std
::
shared_ptr
<
DensePullThread
>
dpt
)
{}
virtual
void
SetParamConfig
(
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录