Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
PaddlePaddle
Paddle
提交
81848fff
P
Paddle
项目概览
PaddlePaddle
/
Paddle
11 个月 前同步成功
通知
2292
Star
20931
Fork
5422
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
1423
列表
看板
标记
里程碑
合并请求
543
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
P
Paddle
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
1,423
Issue
1,423
列表
看板
标记
里程碑
合并请求
543
合并请求
543
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
未验证
提交
81848fff
编写于
3月 17, 2022
作者:
Y
Yuang Liu
提交者:
GitHub
3月 17, 2022
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
[fleet executor] fleet executor for npu (#40607)
上级
06fee998
变更
7
显示空白变更内容
内联
并排
Showing
7 changed file
with
19 addition
and
37 deletion
+19
-37
paddle/fluid/distributed/fleet_executor/CMakeLists.txt
paddle/fluid/distributed/fleet_executor/CMakeLists.txt
+1
-1
paddle/fluid/distributed/fleet_executor/message_bus.cc
paddle/fluid/distributed/fleet_executor/message_bus.cc
+4
-8
paddle/fluid/distributed/fleet_executor/message_bus.h
paddle/fluid/distributed/fleet_executor/message_bus.h
+3
-6
paddle/fluid/distributed/fleet_executor/message_service.cc
paddle/fluid/distributed/fleet_executor/message_service.cc
+1
-2
paddle/fluid/distributed/fleet_executor/message_service.h
paddle/fluid/distributed/fleet_executor/message_service.h
+1
-2
paddle/fluid/inference/api/analysis_predictor.cc
paddle/fluid/inference/api/analysis_predictor.cc
+6
-12
paddle/fluid/inference/api/analysis_predictor.h
paddle/fluid/inference/api/analysis_predictor.h
+3
-6
未找到文件。
paddle/fluid/distributed/fleet_executor/CMakeLists.txt
浏览文件 @
81848fff
...
@@ -4,7 +4,7 @@ if(WITH_PYTHON)
...
@@ -4,7 +4,7 @@ if(WITH_PYTHON)
endif
()
endif
()
proto_library
(
interceptor_message_proto SRCS interceptor_message.proto
)
proto_library
(
interceptor_message_proto SRCS interceptor_message.proto
)
if
(
WITH_DISTRIBUTE AND WITH_PSCORE
AND
NOT
(
WITH_ASCEND OR WITH_ASCEND_CL
)
)
if
(
WITH_DISTRIBUTE AND WITH_PSCORE
)
set
(
BRPC_DEPS brpc ssl crypto protobuf zlib leveldb snappy gflags glog
)
set
(
BRPC_DEPS brpc ssl crypto protobuf zlib leveldb snappy gflags glog
)
else
()
else
()
set
(
BRPC_DEPS
""
)
set
(
BRPC_DEPS
""
)
...
...
paddle/fluid/distributed/fleet_executor/message_bus.cc
浏览文件 @
81848fff
...
@@ -67,8 +67,7 @@ bool MessageBus::IsInit() const { return is_init_; }
...
@@ -67,8 +67,7 @@ bool MessageBus::IsInit() const { return is_init_; }
MessageBus
::~
MessageBus
()
{
MessageBus
::~
MessageBus
()
{
VLOG
(
3
)
<<
"Message bus releases resource."
;
VLOG
(
3
)
<<
"Message bus releases resource."
;
#if defined(PADDLE_WITH_DISTRIBUTE) && defined(PADDLE_WITH_PSCORE) && \
#if defined(PADDLE_WITH_DISTRIBUTE) && defined(PADDLE_WITH_PSCORE)
!defined(PADDLE_WITH_ASCEND_CL)
server_
.
Stop
(
1000
);
server_
.
Stop
(
1000
);
server_
.
Join
();
server_
.
Join
();
#endif
#endif
...
@@ -87,8 +86,7 @@ bool MessageBus::Send(int64_t dst_rank,
...
@@ -87,8 +86,7 @@ bool MessageBus::Send(int64_t dst_rank,
IsInit
(),
true
,
IsInit
(),
true
,
platform
::
errors
::
PreconditionNotMet
(
platform
::
errors
::
PreconditionNotMet
(
"Using message bus since it has not been initialized."
));
"Using message bus since it has not been initialized."
));
#if defined(PADDLE_WITH_DISTRIBUTE) && defined(PADDLE_WITH_PSCORE) && \
#if defined(PADDLE_WITH_DISTRIBUTE) && defined(PADDLE_WITH_PSCORE)
!defined(PADDLE_WITH_ASCEND_CL)
int
retry_time
=
0
;
// message bus will retry sending for 10 times
int
retry_time
=
0
;
// message bus will retry sending for 10 times
while
(
retry_time
<
10
)
{
while
(
retry_time
<
10
)
{
++
retry_time
;
++
retry_time
;
...
@@ -173,8 +171,7 @@ void MessageBus::ListenPort() {
...
@@ -173,8 +171,7 @@ void MessageBus::ListenPort() {
LOG
(
INFO
)
<<
"No need listen to port since training on single card."
;
LOG
(
INFO
)
<<
"No need listen to port since training on single card."
;
return
;
return
;
}
}
#if defined(PADDLE_WITH_DISTRIBUTE) && defined(PADDLE_WITH_PSCORE) && \
#if defined(PADDLE_WITH_DISTRIBUTE) && defined(PADDLE_WITH_PSCORE)
!defined(PADDLE_WITH_ASCEND_CL)
// function keep listen the port and handle the message
// function keep listen the port and handle the message
PADDLE_ENFORCE_EQ
(
PADDLE_ENFORCE_EQ
(
server_
.
AddService
(
&
message_service_
,
brpc
::
SERVER_DOESNT_OWN_SERVICE
),
0
,
server_
.
AddService
(
&
message_service_
,
brpc
::
SERVER_DOESNT_OWN_SERVICE
),
0
,
...
@@ -203,8 +200,7 @@ void MessageBus::ListenPort() {
...
@@ -203,8 +200,7 @@ void MessageBus::ListenPort() {
#endif
#endif
}
}
#if defined(PADDLE_WITH_DISTRIBUTE) && defined(PADDLE_WITH_PSCORE) && \
#if defined(PADDLE_WITH_DISTRIBUTE) && defined(PADDLE_WITH_PSCORE)
!defined(PADDLE_WITH_ASCEND_CL)
bool
MessageBus
::
SendInterRank
(
int64_t
dst_rank
,
bool
MessageBus
::
SendInterRank
(
int64_t
dst_rank
,
const
InterceptorMessage
&
interceptor_message
)
{
const
InterceptorMessage
&
interceptor_message
)
{
const
auto
&
dst_addr
=
GetAddr
(
dst_rank
);
const
auto
&
dst_addr
=
GetAddr
(
dst_rank
);
...
...
paddle/fluid/distributed/fleet_executor/message_bus.h
浏览文件 @
81848fff
...
@@ -20,8 +20,7 @@
...
@@ -20,8 +20,7 @@
#include <thread>
#include <thread>
#include <unordered_map>
#include <unordered_map>
#if defined(PADDLE_WITH_DISTRIBUTE) && defined(PADDLE_WITH_PSCORE) && \
#if defined(PADDLE_WITH_DISTRIBUTE) && defined(PADDLE_WITH_PSCORE)
!defined(PADDLE_WITH_ASCEND_CL)
#include "brpc/channel.h"
#include "brpc/channel.h"
#include "brpc/server.h"
#include "brpc/server.h"
#include "paddle/fluid/distributed/fleet_executor/message_service.h"
#include "paddle/fluid/distributed/fleet_executor/message_service.h"
...
@@ -64,8 +63,7 @@ class MessageBus final {
...
@@ -64,8 +63,7 @@ class MessageBus final {
const
std
::
string
&
GetAddr
(
int64_t
rank
)
const
;
const
std
::
string
&
GetAddr
(
int64_t
rank
)
const
;
#if defined(PADDLE_WITH_DISTRIBUTE) && defined(PADDLE_WITH_PSCORE) && \
#if defined(PADDLE_WITH_DISTRIBUTE) && defined(PADDLE_WITH_PSCORE)
!defined(PADDLE_WITH_ASCEND_CL)
// send the message inter rank (dst is different rank with src)
// send the message inter rank (dst is different rank with src)
bool
SendInterRank
(
int64_t
dst_rank
,
bool
SendInterRank
(
int64_t
dst_rank
,
const
InterceptorMessage
&
interceptor_message
);
const
InterceptorMessage
&
interceptor_message
);
...
@@ -81,8 +79,7 @@ class MessageBus final {
...
@@ -81,8 +79,7 @@ class MessageBus final {
// the ip needs to be listened
// the ip needs to be listened
std
::
string
addr_
;
std
::
string
addr_
;
#if defined(PADDLE_WITH_DISTRIBUTE) && defined(PADDLE_WITH_PSCORE) && \
#if defined(PADDLE_WITH_DISTRIBUTE) && defined(PADDLE_WITH_PSCORE)
!defined(PADDLE_WITH_ASCEND_CL)
MessageServiceImpl
message_service_
;
MessageServiceImpl
message_service_
;
// brpc server
// brpc server
brpc
::
Server
server_
;
brpc
::
Server
server_
;
...
...
paddle/fluid/distributed/fleet_executor/message_service.cc
浏览文件 @
81848fff
...
@@ -11,8 +11,7 @@
...
@@ -11,8 +11,7 @@
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// See the License for the specific language governing permissions and
// limitations under the License.
// limitations under the License.
#if defined(PADDLE_WITH_DISTRIBUTE) && defined(PADDLE_WITH_PSCORE) && \
#if defined(PADDLE_WITH_DISTRIBUTE) && defined(PADDLE_WITH_PSCORE)
!defined(PADDLE_WITH_ASCEND_CL)
#include "paddle/fluid/distributed/fleet_executor/message_service.h"
#include "paddle/fluid/distributed/fleet_executor/message_service.h"
#include "brpc/server.h"
#include "brpc/server.h"
#include "paddle/fluid/distributed/fleet_executor/global.h"
#include "paddle/fluid/distributed/fleet_executor/global.h"
...
...
paddle/fluid/distributed/fleet_executor/message_service.h
浏览文件 @
81848fff
...
@@ -11,8 +11,7 @@
...
@@ -11,8 +11,7 @@
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// See the License for the specific language governing permissions and
// limitations under the License.
// limitations under the License.
#if defined(PADDLE_WITH_DISTRIBUTE) && defined(PADDLE_WITH_PSCORE) && \
#if defined(PADDLE_WITH_DISTRIBUTE) && defined(PADDLE_WITH_PSCORE)
!defined(PADDLE_WITH_ASCEND_CL)
#pragma once
#pragma once
#include "brpc/server.h"
#include "brpc/server.h"
...
...
paddle/fluid/inference/api/analysis_predictor.cc
浏览文件 @
81848fff
...
@@ -50,8 +50,7 @@
...
@@ -50,8 +50,7 @@
#include "paddle/phi/api/ext/op_meta_info.h"
#include "paddle/phi/api/ext/op_meta_info.h"
#include "paddle/utils/string/split.h"
#include "paddle/utils/string/split.h"
#if defined(PADDLE_WITH_DISTRIBUTE) && defined(PADDLE_WITH_PSCORE) && \
#if defined(PADDLE_WITH_DISTRIBUTE) && defined(PADDLE_WITH_PSCORE)
!defined(PADDLE_WITH_ASCEND_CL)
#include "paddle/fluid/distributed/fleet_executor/fleet_executor.h"
#include "paddle/fluid/distributed/fleet_executor/fleet_executor.h"
#include "paddle/fluid/distributed/fleet_executor/fleet_executor_desc.pb.h"
#include "paddle/fluid/distributed/fleet_executor/fleet_executor_desc.pb.h"
#include "paddle/fluid/distributed/fleet_executor/task_node.h"
#include "paddle/fluid/distributed/fleet_executor/task_node.h"
...
@@ -374,8 +373,7 @@ static void DisablePrepareDataOpt(
...
@@ -374,8 +373,7 @@ static void DisablePrepareDataOpt(
}
}
bool
AnalysisPredictor
::
PrepareExecutor
()
{
bool
AnalysisPredictor
::
PrepareExecutor
()
{
#if defined(PADDLE_WITH_DISTRIBUTE) && defined(PADDLE_WITH_PSCORE) && \
#if defined(PADDLE_WITH_DISTRIBUTE) && defined(PADDLE_WITH_PSCORE)
!defined(PADDLE_WITH_ASCEND_CL)
if
(
config_
.
dist_config
().
use_dist_model
())
{
if
(
config_
.
dist_config
().
use_dist_model
())
{
VLOG
(
3
)
<<
"use_dist_model is enabled, will init FleetExecutor."
;
VLOG
(
3
)
<<
"use_dist_model is enabled, will init FleetExecutor."
;
return
PrepareFleetExecutor
();
return
PrepareFleetExecutor
();
...
@@ -393,8 +391,7 @@ bool AnalysisPredictor::PrepareExecutor() {
...
@@ -393,8 +391,7 @@ bool AnalysisPredictor::PrepareExecutor() {
return
true
;
return
true
;
}
}
#if defined(PADDLE_WITH_DISTRIBUTE) && defined(PADDLE_WITH_PSCORE) && \
#if defined(PADDLE_WITH_DISTRIBUTE) && defined(PADDLE_WITH_PSCORE)
!defined(PADDLE_WITH_ASCEND_CL)
bool
AnalysisPredictor
::
PrepareFleetExecutor
()
{
bool
AnalysisPredictor
::
PrepareFleetExecutor
()
{
VLOG
(
3
)
<<
"AnalysisPredictor::PrepareFleetExecutor()"
;
VLOG
(
3
)
<<
"AnalysisPredictor::PrepareFleetExecutor()"
;
if
(
config_
.
dist_config
().
nranks
()
>
1
&&
!
CommInit
())
{
if
(
config_
.
dist_config
().
nranks
()
>
1
&&
!
CommInit
())
{
...
@@ -1194,8 +1191,7 @@ std::vector<std::string> AnalysisPredictor::GetOutputNames() {
...
@@ -1194,8 +1191,7 @@ std::vector<std::string> AnalysisPredictor::GetOutputNames() {
std
::
unique_ptr
<
ZeroCopyTensor
>
AnalysisPredictor
::
GetInputTensor
(
std
::
unique_ptr
<
ZeroCopyTensor
>
AnalysisPredictor
::
GetInputTensor
(
const
std
::
string
&
name
)
{
const
std
::
string
&
name
)
{
framework
::
Scope
*
scope
;
framework
::
Scope
*
scope
;
#if defined(PADDLE_WITH_DISTRIBUTE) && defined(PADDLE_WITH_PSCORE) && \
#if defined(PADDLE_WITH_DISTRIBUTE) && defined(PADDLE_WITH_PSCORE)
!defined(PADDLE_WITH_ASCEND_CL)
if
(
config_
.
dist_config
().
use_dist_model
())
{
if
(
config_
.
dist_config
().
use_dist_model
())
{
scope
=
scope_
.
get
();
scope
=
scope_
.
get
();
}
else
{
}
else
{
...
@@ -1244,8 +1240,7 @@ std::unique_ptr<ZeroCopyTensor> AnalysisPredictor::GetInputTensor(
...
@@ -1244,8 +1240,7 @@ std::unique_ptr<ZeroCopyTensor> AnalysisPredictor::GetInputTensor(
std
::
unique_ptr
<
ZeroCopyTensor
>
AnalysisPredictor
::
GetOutputTensor
(
std
::
unique_ptr
<
ZeroCopyTensor
>
AnalysisPredictor
::
GetOutputTensor
(
const
std
::
string
&
name
)
{
const
std
::
string
&
name
)
{
framework
::
Scope
*
scope
;
framework
::
Scope
*
scope
;
#if defined(PADDLE_WITH_DISTRIBUTE) && defined(PADDLE_WITH_PSCORE) && \
#if defined(PADDLE_WITH_DISTRIBUTE) && defined(PADDLE_WITH_PSCORE)
!defined(PADDLE_WITH_ASCEND_CL)
if
(
config_
.
dist_config
().
use_dist_model
())
{
if
(
config_
.
dist_config
().
use_dist_model
())
{
scope
=
scope_
.
get
();
scope
=
scope_
.
get
();
}
else
{
}
else
{
...
@@ -1292,8 +1287,7 @@ std::unique_ptr<ZeroCopyTensor> AnalysisPredictor::GetOutputTensor(
...
@@ -1292,8 +1287,7 @@ std::unique_ptr<ZeroCopyTensor> AnalysisPredictor::GetOutputTensor(
}
}
bool
AnalysisPredictor
::
ZeroCopyRun
()
{
bool
AnalysisPredictor
::
ZeroCopyRun
()
{
#if defined(PADDLE_WITH_DISTRIBUTE) && defined(PADDLE_WITH_PSCORE) && \
#if defined(PADDLE_WITH_DISTRIBUTE) && defined(PADDLE_WITH_PSCORE)
!defined(PADDLE_WITH_ASCEND_CL)
if
(
config_
.
dist_config
().
use_dist_model
())
{
if
(
config_
.
dist_config
().
use_dist_model
())
{
VLOG
(
3
)
<<
"ZeroCopyRun will use the fleet executor."
;
VLOG
(
3
)
<<
"ZeroCopyRun will use the fleet executor."
;
inference
::
Timer
timer
;
inference
::
Timer
timer
;
...
...
paddle/fluid/inference/api/analysis_predictor.h
浏览文件 @
81848fff
...
@@ -18,8 +18,7 @@
...
@@ -18,8 +18,7 @@
#include <memory>
#include <memory>
#include <string>
#include <string>
#include <vector>
#include <vector>
#if defined(PADDLE_WITH_DISTRIBUTE) && defined(PADDLE_WITH_PSCORE) && \
#if defined(PADDLE_WITH_DISTRIBUTE) && defined(PADDLE_WITH_PSCORE)
!defined(PADDLE_WITH_ASCEND_CL)
#include "paddle/fluid/distributed/fleet_executor/fleet_executor.h"
#include "paddle/fluid/distributed/fleet_executor/fleet_executor.h"
#endif
#endif
#include "paddle/fluid/framework/naive_executor.h"
#include "paddle/fluid/framework/naive_executor.h"
...
@@ -395,8 +394,7 @@ class AnalysisPredictor : public PaddlePredictor {
...
@@ -395,8 +394,7 @@ class AnalysisPredictor : public PaddlePredictor {
void
StatisticShapeRangeInfo
();
void
StatisticShapeRangeInfo
();
void
CollectShapeRangeInfo
();
void
CollectShapeRangeInfo
();
#if defined(PADDLE_WITH_DISTRIBUTE) && defined(PADDLE_WITH_PSCORE) && \
#if defined(PADDLE_WITH_DISTRIBUTE) && defined(PADDLE_WITH_PSCORE)
!defined(PADDLE_WITH_ASCEND_CL)
// fleet exe related
// fleet exe related
///
///
...
@@ -488,8 +486,7 @@ class AnalysisPredictor : public PaddlePredictor {
...
@@ -488,8 +486,7 @@ class AnalysisPredictor : public PaddlePredictor {
std
::
map
<
std
::
string
,
std
::
vector
<
std
::
vector
<
int32_t
>>>
shape_info_
;
std
::
map
<
std
::
string
,
std
::
vector
<
std
::
vector
<
int32_t
>>>
shape_info_
;
static
int
clone_num_
;
static
int
clone_num_
;
#if defined(PADDLE_WITH_DISTRIBUTE) && defined(PADDLE_WITH_PSCORE) && \
#if defined(PADDLE_WITH_DISTRIBUTE) && defined(PADDLE_WITH_PSCORE)
!defined(PADDLE_WITH_ASCEND_CL)
// fleet executor related
// fleet executor related
distributed
::
FleetExecutorDesc
executor_desc_
;
distributed
::
FleetExecutorDesc
executor_desc_
;
std
::
shared_ptr
<
distributed
::
FleetExecutor
>
fleet_exe_
;
std
::
shared_ptr
<
distributed
::
FleetExecutor
>
fleet_exe_
;
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录