Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
PaddlePaddle
PaddleDetection
提交
b6dc3a59
P
PaddleDetection
项目概览
PaddlePaddle
/
PaddleDetection
大约 1 年 前同步成功
通知
695
Star
11112
Fork
2696
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
184
列表
看板
标记
里程碑
合并请求
40
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
P
PaddleDetection
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
184
Issue
184
列表
看板
标记
里程碑
合并请求
40
合并请求
40
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
提交
b6dc3a59
编写于
6月 28, 2018
作者:
F
fengjiayi
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
Add DataBalanceOpHandle to MultiDeviceSSAGragh
上级
a1f1a5ed
变更
7
隐藏空白更改
内联
并排
Showing
7 changed file
with
229 addition
and
5 deletion
+229
-5
paddle/fluid/framework/details/CMakeLists.txt
paddle/fluid/framework/details/CMakeLists.txt
+2
-1
paddle/fluid/framework/details/data_balance_op_handle.cc
paddle/fluid/framework/details/data_balance_op_handle.cc
+138
-0
paddle/fluid/framework/details/data_balance_op_handle.h
paddle/fluid/framework/details/data_balance_op_handle.h
+50
-0
paddle/fluid/framework/details/multi_devices_graph_builder.cc
...le/fluid/framework/details/multi_devices_graph_builder.cc
+26
-1
paddle/fluid/framework/details/multi_devices_graph_builder.h
paddle/fluid/framework/details/multi_devices_graph_builder.h
+3
-0
paddle/fluid/framework/lod_tensor.cc
paddle/fluid/framework/lod_tensor.cc
+2
-1
paddle/fluid/operators/read_op.cc
paddle/fluid/operators/read_op.cc
+8
-2
未找到文件。
paddle/fluid/framework/details/CMakeLists.txt
浏览文件 @
b6dc3a59
...
...
@@ -25,11 +25,12 @@ else()
cc_library
(
broadcast_op_handle SRCS broadcast_op_handle.cc DEPS op_handle_base scope ddim memory variable_visitor
)
endif
()
cc_library
(
data_balance_op_handle SRCS data_balance_op_handle.cc DEPS op_handle_base scope lod_tensor
)
cc_library
(
gather_op_handle SRCS gather_op_handle.cc DEPS op_handle_base scope ddim memory variable_visitor
)
cc_library
(
fuse_vars_op_handle SRCS fuse_vars_op_handle.cc DEPS op_handle_base scope
)
cc_library
(
multi_devices_graph_builder SRCS multi_devices_graph_builder.cc DEPS ssa_graph_builder computation_op_handle
scale_loss_grad_op_handle rpc_op_handle all_reduce_op_handle reduce_op_handle broadcast_op_handle
)
scale_loss_grad_op_handle rpc_op_handle all_reduce_op_handle reduce_op_handle broadcast_op_handle
data_balance_op_handle
)
cc_library
(
ssa_graph_builder_factory SRCS ssa_graph_builder_factory.cc DEPS multi_devices_graph_builder ssa_graph_printer ssa_graph_checker
)
...
...
paddle/fluid/framework/details/data_balance_op_handle.cc
0 → 100644
浏览文件 @
b6dc3a59
// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "paddle/fluid/framework/details/data_balance_op_handle.h"
#include <algorithm>
#include "paddle/fluid/framework/details/container_cast.h"
namespace
paddle
{
namespace
framework
{
namespace
details
{
DataBalanceOpHandle
::
DataBalanceOpHandle
(
const
std
::
vector
<
Scope
*>
&
local_scopes
,
const
std
::
vector
<
platform
::
Place
>
&
places
)
:
local_scopes_
(
local_scopes
),
places_
(
places
)
{}
std
::
string
DataBalanceOpHandle
::
Name
()
const
{
return
"data balance"
;
}
std
::
vector
<
std
::
array
<
int
,
3
>>
DataBalanceOpHandle
::
GetBalancePlan
(
const
std
::
vector
<
int
>
&
device_sizes
)
{
int
device_num
=
device_sizes
.
size
();
int
total_size
=
0
;
int
empty_num
=
0
;
std
::
vector
<
std
::
array
<
int
,
2
>>
size_device_vec
;
size_device_vec
.
reserve
(
device_num
);
for
(
int
i
=
0
;
i
<
device_num
;
++
i
)
{
if
(
device_sizes
[
i
]
==
0
)
{
++
empty_num
;
}
total_size
+=
device_sizes
[
i
];
size_device_vec
.
push_back
({{
device_sizes
[
i
],
i
}});
}
std
::
vector
<
std
::
array
<
int
,
3
>>
res
;
if
(
empty_num
==
0
)
{
// No need to do data balance.
return
res
;
}
if
(
total_size
<
device_num
)
{
// No enough data.
PADDLE_THROW
(
"There is no next data."
);
}
std
::
sort
(
size_device_vec
.
begin
(),
size_device_vec
.
end
(),
[](
const
std
::
array
<
int
,
2
>
&
a
,
const
std
::
array
<
int
,
2
>
&
b
)
{
return
a
[
0
]
>
b
[
0
];
});
int
expected_device_size
=
total_size
/
device_num
;
int
src_idx
=
0
;
for
(
int
dst_idx
=
device_num
-
empty_num
;
dst_idx
<
device_num
;
++
dst_idx
)
{
if
(
size_device_vec
[
src_idx
][
0
]
<=
expected_device_size
)
{
++
src_idx
;
PADDLE_ENFORCE_LT
(
src_idx
,
device_num
-
empty_num
);
}
size_device_vec
[
src_idx
][
0
]
-=
expected_device_size
;
size_device_vec
[
dst_idx
][
0
]
+=
expected_device_size
;
res
.
push_back
({{
size_device_vec
[
src_idx
][
1
],
size_device_vec
[
dst_idx
][
1
],
expected_device_size
}});
}
return
res
;
}
void
DataBalanceOpHandle
::
RunImpl
()
{
if
(
places_
.
size
()
==
1
)
{
return
;
}
auto
in_var_handles
=
DynamicCast
<
VarHandle
>
(
inputs_
);
auto
out_var_handles
=
DynamicCast
<
VarHandle
>
(
outputs_
);
PADDLE_ENFORCE
(
in_var_handles
.
size
()
%
places_
.
size
()
==
0
);
PADDLE_ENFORCE_EQ
(
in_var_handles
.
size
(),
out_var_handles
.
size
(),
"The NoDummyInputSize and NoDummyOutputSize should be equal."
);
int
data_num
=
in_var_handles
.
size
()
/
places_
.
size
();
WaitInputVarGenerated
();
std
::
vector
<
std
::
vector
<
LoDTensor
*>>
lod_tensors
;
std
::
vector
<
int
>
device_sizes
;
for
(
int
i
=
0
;
i
<
static_cast
<
int
>
(
in_var_handles
.
size
());
++
i
)
{
PADDLE_ENFORCE_EQ
(
in_var_handles
[
i
]
->
name_
,
out_var_handles
[
i
]
->
name_
,
"The name of input and output should be equal."
);
int
place_idx
=
i
/
data_num
;
int
data_idx
=
i
%
data_num
;
auto
*
local_scope
=
local_scopes_
[
place_idx
]
->
FindVar
(
kLocalExecScopeName
)
->
Get
<
Scope
*>
();
auto
*
tensor_var
=
local_scope
->
FindVar
(
in_var_handles
[
i
]
->
name_
);
PADDLE_ENFORCE
(
tensor_var
->
IsType
<
LoDTensor
>
());
auto
*
tensor
=
tensor_var
->
GetMutable
<
LoDTensor
>
();
PADDLE_ENFORCE
(
places_
[
place_idx
]
==
tensor
->
place
());
lod_tensors
[
data_idx
].
push_back
(
tensor
);
int
ins_size
=
tensor
->
lod
().
empty
()
?
tensor
->
dims
()[
0
]
:
tensor
->
NumElements
();
if
(
data_idx
==
0
)
{
device_sizes
.
emplace_back
(
ins_size
);
}
else
{
PADDLE_ENFORCE_EQ
(
ins_size
,
device_sizes
.
at
(
place_idx
));
}
}
const
auto
&
balance_plan
=
GetBalancePlan
(
device_sizes
);
for
(
const
auto
&
trans
:
balance_plan
)
{
for
(
int
data_idx
=
0
;
data_idx
<
data_num
;
++
data_idx
)
{
LoDTensor
*
src_tensor
=
lod_tensors
[
data_idx
][
trans
[
0
]];
LoDTensor
*
dst_tensor
=
lod_tensors
[
data_idx
][
trans
[
1
]];
int
trans_ins_size
=
trans
[
2
];
LoD
src_lod
=
src_tensor
->
lod
();
int
src_ins_size
=
src_lod
.
empty
()
?
src_tensor
->
dims
()[
0
]
:
src_tensor
->
NumElements
();
int
cut_point
=
src_ins_size
-
trans_ins_size
;
if
(
!
src_lod
.
empty
())
{
for
(
auto
&
level
:
src_lod
)
{
cut_point
=
level
[
cut_point
];
}
}
TensorCopySync
(
src_tensor
->
Slice
(
cut_point
,
src_tensor
->
dims
()[
0
]),
dst_tensor
->
place
(),
dst_tensor
);
src_tensor
->
ShareDataWith
(
src_tensor
->
Slice
(
0
,
cut_point
));
if
(
!
src_lod
.
empty
())
{
dst_tensor
->
set_lod
(
SliceInLevel
(
src_lod
,
0
,
src_ins_size
-
trans_ins_size
,
src_ins_size
));
src_tensor
->
set_lod
(
SliceInLevel
(
src_lod
,
0
,
0
,
src_ins_size
-
trans_ins_size
));
}
}
}
}
}
// namespace details
}
// namespace framework
}
// namespace paddle
paddle/fluid/framework/details/data_balance_op_handle.h
0 → 100644
浏览文件 @
b6dc3a59
// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
#include <string>
#include <vector>
#include "paddle/fluid/framework/details/op_handle_base.h"
#include "paddle/fluid/framework/lod_tensor.h"
#include "paddle/fluid/framework/scope.h"
namespace
paddle
{
namespace
framework
{
namespace
details
{
struct
DataBalanceOpHandle
:
public
OpHandleBase
{
public:
DataBalanceOpHandle
(
const
std
::
vector
<
Scope
*>
&
local_scopes
,
const
std
::
vector
<
platform
::
Place
>
&
places
);
std
::
string
Name
()
const
override
;
bool
IsMultiDeviceTransfer
()
override
{
return
false
;
};
protected:
void
RunImpl
()
override
;
private:
// std::vector<(src_dev_id, dst_dev_id, trans_size)>
std
::
vector
<
std
::
array
<
int
,
3
>>
GetBalancePlan
(
const
std
::
vector
<
int
>
&
batch_size_per_device
);
const
std
::
vector
<
Scope
*>
&
local_scopes_
;
const
std
::
vector
<
platform
::
Place
>
&
places_
;
};
}
// namespace details
}
// namespace framework
}
// namespace paddle
paddle/fluid/framework/details/multi_devices_graph_builder.cc
浏览文件 @
b6dc3a59
...
...
@@ -20,6 +20,7 @@
#include "paddle/fluid/framework/details/all_reduce_op_handle.h"
#include "paddle/fluid/framework/details/broadcast_op_handle.h"
#include "paddle/fluid/framework/details/computation_op_handle.h"
#include "paddle/fluid/framework/details/data_balance_op_handle.h"
#include "paddle/fluid/framework/details/multi_devices_graph_builder.h"
#include "paddle/fluid/framework/details/reduce_op_handle.h"
#include "paddle/fluid/framework/details/rpc_op_handle.h"
...
...
@@ -217,6 +218,11 @@ std::unique_ptr<SSAGraph> MultiDevSSAGraphBuilder::Build(
// gradients.
CreateComputationalOps
(
&
result
,
*
op
,
places_
.
size
());
if
(
op
->
Type
()
==
"read"
)
{
const
auto
&
data_var_names
=
op
->
Output
(
"Out"
);
InsertDataBalanceOp
(
&
result
,
data_var_names
);
}
if
(
!
is_forwarding
&&
places_
.
size
()
>
1
)
{
// Currently, we assume that once gradient is generated, it can be
// broadcast, and each gradient is only broadcast once.
...
...
@@ -360,6 +366,24 @@ void MultiDevSSAGraphBuilder::InsertAllReduceOp(SSAGraph *result,
}
}
void
MultiDevSSAGraphBuilder
::
InsertDataBalanceOp
(
SSAGraph
*
result
,
const
std
::
vector
<
std
::
string
>
&
datas
)
const
{
result
->
ops_
.
emplace_back
(
new
DataBalanceOpHandle
(
local_scopes_
,
places_
));
auto
*
op_handle
=
result
->
ops_
.
back
().
get
();
for
(
size_t
i
=
0
;
i
<
places_
.
size
();
++
i
)
{
auto
&
p
=
places_
[
i
];
SetCommunicationContext
(
op_handle
,
p
);
for
(
const
std
::
string
&
d_name
:
datas
)
{
auto
&
vars
=
result
->
vars_
[
i
][
d_name
];
PADDLE_ENFORCE
(
!
vars
.
empty
());
op_handle
->
AddInput
(
vars
.
back
().
get
());
auto
var
=
new
VarHandle
(
vars
.
size
(),
i
,
d_name
,
p
);
vars
.
emplace_back
(
var
);
op_handle
->
AddOutput
(
var
);
}
}
}
bool
MultiDevSSAGraphBuilder
::
IsParameterGradientOnce
(
const
std
::
string
&
og
,
std
::
unordered_set
<
std
::
string
>
*
og_has_been_broadcast
)
const
{
...
...
@@ -509,7 +533,8 @@ void MultiDevSSAGraphBuilder::CreateRPCOp(SSAGraph *result,
op_dev_id
=
GetVarDeviceID
(
op
.
InputArgumentNames
()[
0
]);
// the variable name which contains .block means it was splited by
// split_byref op
// so that we can balance the variable blocks to all the pserver instances.
// so that we can balance the variable blocks to all the pserver
// instances.
if
(
strategy_
.
reduce_
==
BuildStrategy
::
ReduceStrategy
::
kAllReduce
&&
op
.
InputArgumentNames
()[
0
].
find
(
".block"
)
==
std
::
string
::
npos
)
{
op_dev_id
=
GetAppropriateDeviceID
(
op
.
InputArgumentNames
());
...
...
paddle/fluid/framework/details/multi_devices_graph_builder.h
浏览文件 @
b6dc3a59
...
...
@@ -101,6 +101,9 @@ class MultiDevSSAGraphBuilder : public SSAGraphBuilder {
void
InsertAllReduceOp
(
SSAGraph
*
result
,
const
std
::
string
&
og
)
const
;
void
InsertDataBalanceOp
(
SSAGraph
*
result
,
const
std
::
vector
<
std
::
string
>
&
datas
)
const
;
void
CreateBroadcastOp
(
SSAGraph
*
result
,
const
std
::
string
&
p_name
,
size_t
src_dev_id
)
const
;
...
...
paddle/fluid/framework/lod_tensor.cc
浏览文件 @
b6dc3a59
...
...
@@ -68,7 +68,7 @@ std::ostream &operator<<(std::ostream &os, const LoDTensor &t) {
// only print first ten elements
int64_t
size
=
t
.
numel
()
<
10
?
t
.
numel
()
:
10
;
for
(
int64_t
i
=
0
;
i
<
size
;
++
i
)
{
if
(
t
.
type
().
hash_code
()
==
typeid
(
float
).
hash_code
())
{
if
(
t
.
type
().
hash_code
()
==
typeid
(
float
).
hash_code
())
{
// NOLINT
os
<<
t
.
data
<
float
>
()[
i
]
<<
" "
;
}
else
if
(
t
.
type
().
hash_code
()
==
typeid
(
int64_t
).
hash_code
())
{
os
<<
t
.
data
<
int64_t
>
()[
i
]
<<
" "
;
...
...
@@ -89,6 +89,7 @@ std::string LoDToString(const LoD &lod) {
LoD
SliceInLevel
(
const
LoD
&
in
,
size_t
level
,
size_t
elem_begin
,
size_t
elem_end
)
{
PADDLE_ENFORCE_LT
(
level
,
in
.
size
());
PADDLE_ENFORCE_LT
(
elem_begin
,
elem_end
);
PADDLE_ENFORCE_LT
(
elem_end
,
in
[
level
].
size
());
LoD
res
;
...
...
paddle/fluid/operators/read_op.cc
浏览文件 @
b6dc3a59
...
...
@@ -66,9 +66,15 @@ class ReadOp : public framework::OperatorBase {
std
::
vector
<
std
::
string
>
out_arg_names
=
Outputs
(
"Out"
);
std
::
vector
<
framework
::
LoDTensor
>
ins
;
reader
->
ReadNext
(
&
ins
);
PADDLE_ENFORCE
(
!
ins
.
empty
(),
"There is no next data."
);
if
(
ins
.
empty
())
{
ins
.
resize
(
out_arg_names
.
size
());
for
(
auto
&
tensor
:
ins
)
{
// data type is not important for subsequent DataBalanceOpHandle
tensor
.
mutable_data
<
float
>
(
framework
::
make_ddim
({
0
}),
dev_place
);
}
}
PADDLE_ENFORCE_EQ
(
ins
.
size
(),
out_arg_names
.
size
());
for
(
size_t
i
=
0
;
i
<
in
s
.
size
();
++
i
)
{
for
(
size_t
i
=
0
;
i
<
out_arg_name
s
.
size
();
++
i
)
{
auto
*
out
=
scope
.
FindVar
(
out_arg_names
[
i
])
->
GetMutable
<
framework
::
LoDTensor
>
();
out
->
ShareDataWith
(
ins
[
i
]);
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录