Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
机器未来
Paddle
提交
63ff0b4b
P
Paddle
项目概览
机器未来
/
Paddle
与 Fork 源项目一致
Fork自
PaddlePaddle / Paddle
通知
1
Star
1
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
1
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
P
Paddle
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
1
Issue
1
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
63ff0b4b
编写于
1月 08, 2018
作者:
Y
Yang Yu
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
Refine get_places
上级
ed0cf3d6
变更
15
隐藏空白更改
内联
并排
Showing
15 changed file
with
100 addition
and
40 deletion
+100
-40
paddle/framework/block_desc.cc
paddle/framework/block_desc.cc
+2
-2
paddle/framework/block_desc.h
paddle/framework/block_desc.h
+1
-1
paddle/framework/executor.cc
paddle/framework/executor.cc
+5
-1
paddle/framework/framework.proto
paddle/framework/framework.proto
+1
-0
paddle/framework/op_desc.cc
paddle/framework/op_desc.cc
+1
-1
paddle/operators/get_places_op.cc
paddle/operators/get_places_op.cc
+57
-21
paddle/operators/lod_rank_table_op.cc
paddle/operators/lod_rank_table_op.cc
+1
-1
paddle/operators/sum_op.cc
paddle/operators/sum_op.cc
+5
-6
paddle/operators/tensor_array_read_write_op.cc
paddle/operators/tensor_array_read_write_op.cc
+1
-2
paddle/platform/place.h
paddle/platform/place.h
+2
-0
paddle/pybind/protobuf.cc
paddle/pybind/protobuf.cc
+2
-1
python/paddle/v2/fluid/framework.py
python/paddle/v2/fluid/framework.py
+1
-1
python/paddle/v2/fluid/layers/device.py
python/paddle/v2/fluid/layers/device.py
+3
-3
python/paddle/v2/fluid/tests/test_get_places_op.py
python/paddle/v2/fluid/tests/test_get_places_op.py
+17
-0
python/paddle/v2/fluid/tests/test_layers.py
python/paddle/v2/fluid/tests/test_layers.py
+1
-0
未找到文件。
paddle/framework/block_desc.cc
浏览文件 @
63ff0b4b
...
...
@@ -53,12 +53,12 @@ VarDesc *BlockDesc::FindVarRecursive(const std::string &name) const {
return
it
->
second
.
get
();
}
VarDesc
*
BlockDesc
::
FindRecursiveOrCreateVar
(
const
std
::
string
&
name_bytes
)
{
VarDesc
&
BlockDesc
::
FindRecursiveOrCreateVar
(
const
std
::
string
&
name_bytes
)
{
VarDesc
*
res
=
FindVarRecursive
(
name_bytes
);
if
(
res
==
nullptr
)
{
res
=
Var
(
name_bytes
);
}
return
res
;
return
*
res
;
}
bool
BlockDesc
::
HasVarRecursive
(
const
std
::
string
&
name
)
const
{
...
...
paddle/framework/block_desc.h
浏览文件 @
63ff0b4b
...
...
@@ -57,7 +57,7 @@ class BlockDesc {
VarDesc
*
FindVarRecursive
(
const
std
::
string
&
name_bytes
)
const
;
VarDesc
*
FindRecursiveOrCreateVar
(
const
std
::
string
&
name_bytes
);
VarDesc
&
FindRecursiveOrCreateVar
(
const
std
::
string
&
name_bytes
);
bool
HasVarRecursive
(
const
std
::
string
&
var_name
)
const
;
...
...
paddle/framework/executor.cc
浏览文件 @
63ff0b4b
...
...
@@ -21,6 +21,7 @@ limitations under the License. */
#include "paddle/framework/lod_rank_table.h"
#include "paddle/framework/lod_tensor_array.h"
#include "paddle/framework/op_registry.h"
#include "paddle/platform/place.h"
DEFINE_bool
(
check_nan_inf
,
false
,
"Checking whether operator produce NAN/INF or not. It will be "
...
...
@@ -49,10 +50,13 @@ static void CreateTensor(Variable* var, proto::VarDesc::VarType var_type) {
var
->
GetMutable
<
LoDRankTable
>
();
}
else
if
(
var_type
==
proto
::
VarDesc
::
LOD_TENSOR_ARRAY
)
{
var
->
GetMutable
<
LoDTensorArray
>
();
}
else
if
(
var_type
==
proto
::
VarDesc
::
PLACE_LIST
)
{
var
->
GetMutable
<
platform
::
PlaceList
>
();
}
else
{
PADDLE_THROW
(
"Variable type %d is not in "
"[LoDTensor, SelectedRows, FEED_MINIBATCH, FETCH_LIST, LOD_RANK_TABLE]"
,
"[LoDTensor, SelectedRows, FEED_MINIBATCH, FETCH_LIST, LOD_RANK_TABLE,"
" PLACE_LIST]"
,
var_type
);
}
}
...
...
paddle/framework/framework.proto
浏览文件 @
63ff0b4b
...
...
@@ -123,6 +123,7 @@ message VarDesc {
STEP_SCOPES
=
5
;
LOD_RANK_TABLE
=
6
;
LOD_TENSOR_ARRAY
=
7
;
PLACE_LIST
=
8
;
}
required
string
name
=
1
;
required
VarType
type
=
2
;
...
...
paddle/framework/op_desc.cc
浏览文件 @
63ff0b4b
...
...
@@ -383,7 +383,7 @@ void OpDesc::InferVarType(BlockDesc *block) const {
for
(
auto
&
out_pair
:
this
->
outputs_
)
{
for
(
auto
&
out_var_name
:
out_pair
.
second
)
{
block
->
FindRecursiveOrCreateVar
(
out_var_name
)
->
SetType
(
proto
::
VarDesc
::
LOD_TENSOR
);
.
SetType
(
proto
::
VarDesc
::
LOD_TENSOR
);
}
}
}
...
...
paddle/operators/get_places_op.cc
浏览文件 @
63ff0b4b
...
...
@@ -12,7 +12,9 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */
#include <thread>
#include "paddle/framework/op_registry.h"
#include "paddle/operators/detail/safe_ref.h"
#include "paddle/platform/place.h"
#ifdef PADDLE_WITH_CUDA
#include "paddle/platform/gpu_info.h"
...
...
@@ -21,6 +23,14 @@ limitations under the License. */
namespace
paddle
{
namespace
operators
{
static
size_t
CUDADevCount
()
{
#ifdef PADDLE_WITH_CUDA
return
platform
::
GetCUDADeviceCount
();
#else
return
0UL
;
#endif
}
class
GetPlacesOp
:
public
framework
::
OperatorBase
{
public:
GetPlacesOp
(
const
std
::
string
&
type
,
const
framework
::
VariableNameMap
&
inputs
,
...
...
@@ -28,28 +38,34 @@ class GetPlacesOp : public framework::OperatorBase {
const
framework
::
AttributeMap
&
attrs
)
:
OperatorBase
(
type
,
inputs
,
outputs
,
attrs
)
{}
void
Run
(
const
framework
::
Scope
&
scope
,
const
platform
::
DeviceContext
&
dev_ctx
)
const
override
{
const
platform
::
Place
&
place
)
const
override
{
std
::
string
device_type
=
Attr
<
std
::
string
>
(
"device_type"
);
auto
device_count
=
Attr
<
int
>
(
"device_count"
);
auto
device_count
=
static_cast
<
size_t
>
(
Attr
<
int
>
(
"device_count"
));
if
(
device_count
==
0
)
{
if
(
device_type
==
"CUDA"
)
{
device_count
=
CUDADevCount
();
}
else
if
(
device_type
==
"CPU"
)
{
device_count
=
std
::
thread
::
hardware_concurrency
();
}
}
PADDLE_ENFORCE_NE
(
device_count
,
0
,
"Cannot indicate %s device count"
,
device_type
);
auto
out_var_name
=
Output
(
"Out"
);
auto
*
out_var
=
scope
.
FindVar
(
out_var_name
);
PADDLE_ENFORCE
(
out_var
!=
nullptr
,
"Output variable %s cannot be found"
,
out_var_name
);
auto
&
places
=
*
(
out_var
->
GetMutable
<
std
::
vector
<
platform
::
Place
>>
());
places
.
resize
(
device_count
);
auto
&
places
=
*
(
detail
::
Ref
(
scope
.
FindVar
(
out_var_name
),
"Output variable %s cannot be found"
,
out_var_name
)
.
GetMutable
<
platform
::
PlaceList
>
());
places
.
reserve
(
device_count
);
if
(
device_type
==
"CUDA"
)
{
#ifdef PADDLE_WITH_CUDA
PADDLE_ENFORCE_LT
(
device_count
,
platform
::
GetCUDADeviceCount
());
for
(
int
i
=
0
;
i
<
device_count
;
i
++
)
{
places
.
emplace_back
(
platform
::
GPUPlace
(
i
));
PADDLE_ENFORCE_LE
(
device_count
,
CUDADevCount
(),
"Only %d CUDA devices found, cannot set to %d"
,
CUDADevCount
(),
device_count
);
for
(
size_t
i
=
0
;
i
<
device_count
;
++
i
)
{
places
.
emplace_back
(
platform
::
CUDAPlace
(
i
));
}
#else
PADDLE_THROW
(
"'GPUPlace' is not supported in CPU only device."
);
#endif
}
else
if
(
device_type
==
"CPU"
)
{
for
(
int
i
=
0
;
i
<
device_count
;
i
++
)
{
for
(
size_t
i
=
0
;
i
<
device_count
;
++
i
)
{
places
.
emplace_back
(
platform
::
CPUPlace
());
}
}
...
...
@@ -61,18 +77,38 @@ class GetPlacesOpProtoMaker : public framework::OpProtoAndCheckerMaker {
GetPlacesOpProtoMaker
(
OpProto
*
proto
,
OpAttrChecker
*
op_checker
)
:
OpProtoAndCheckerMaker
(
proto
,
op_checker
)
{
AddOutput
(
"Out"
,
"vector of Place"
);
AddAttr
<
int
>
(
"device_count"
,
"
(int)
device count"
).
SetDefault
(
1
);
AddAttr
<
int
>
(
"device_count"
,
"device count"
).
SetDefault
(
1
);
AddAttr
<
std
::
string
>
(
"device_type"
,
"(string), deivce type can be
\"
CPU
\"
and
\"
CUDA
\"
"
)
R"(device type must be in ["CPU", "CUDA"])
"
)
.
InEnum
({
"CPU"
,
"CUDA"
});
AddComment
(
R"DOC(
Returns a list of places based on flags. The list will be used for parallel
execution.
Returns a list of places based on flags. The list will be used for parallel
execution.
)DOC"
);
}
};
class
GetPlacesInferVarType
:
public
framework
::
VarTypeInference
{
public:
void
operator
()(
const
framework
::
OpDesc
&
op_desc
,
framework
::
BlockDesc
*
block
)
const
override
{
for
(
auto
&
o_name
:
op_desc
.
Output
(
"Out"
))
{
block
->
FindRecursiveOrCreateVar
(
o_name
).
SetType
(
framework
::
proto
::
VarDesc
::
PLACE_LIST
);
}
}
};
class
GetPlacesInferShape
:
public
framework
::
InferShapeBase
{
public:
void
operator
()(
framework
::
InferShapeContext
*
context
)
const
override
{
// Do nothing
}
};
}
// namespace operators
}
// namespace paddle
namespace
ops
=
paddle
::
operators
;
REGISTER_OPERATOR
(
get_places
,
ops
::
GetPlacesOp
,
ops
::
GetPlacesOpProtoMaker
);
REGISTER_OPERATOR
(
get_places
,
ops
::
GetPlacesOp
,
ops
::
GetPlacesOpProtoMaker
,
ops
::
GetPlacesInferVarType
,
ops
::
GetPlacesInferShape
);
paddle/operators/lod_rank_table_op.cc
浏览文件 @
63ff0b4b
...
...
@@ -66,7 +66,7 @@ class LoDRankTableInferVarType : public framework::VarTypeInference {
void
operator
()(
const
framework
::
OpDesc
&
op_desc
,
framework
::
BlockDesc
*
block
)
const
override
{
for
(
auto
&
o
:
op_desc
.
Output
(
"Out"
))
{
block
->
FindRecursiveOrCreateVar
(
o
)
->
SetType
(
block
->
FindRecursiveOrCreateVar
(
o
)
.
SetType
(
framework
::
proto
::
VarDesc
::
LOD_RANK_TABLE
);
}
}
...
...
paddle/operators/sum_op.cc
浏览文件 @
63ff0b4b
...
...
@@ -122,17 +122,17 @@ class SumOpVarTypeInference : public framework::VarTypeInference {
for
(
auto
&
name
:
op_desc
.
Input
(
"X"
))
{
VLOG
(
10
)
<<
name
<<
" "
<<
block
->
FindRecursiveOrCreateVar
(
name
)
->
GetType
();
<<
block
->
FindRecursiveOrCreateVar
(
name
)
.
GetType
();
}
bool
any_input_is_lod_tensor
=
std
::
any_of
(
inputs
.
begin
(),
inputs
.
end
(),
[
block
](
const
std
::
string
&
name
)
{
return
block
->
FindRecursiveOrCreateVar
(
name
)
->
GetType
()
==
return
block
->
FindRecursiveOrCreateVar
(
name
)
.
GetType
()
==
framework
::
proto
::
VarDesc
::
LOD_TENSOR
;
});
auto
is_tensor_array
=
[
block
](
const
std
::
string
&
name
)
{
return
detail
::
Ref
(
block
->
FindRecursiveOrCreateVar
(
name
)
).
GetType
()
==
return
block
->
FindRecursiveOrCreateVar
(
name
).
GetType
()
==
framework
::
proto
::
VarDesc
::
LOD_TENSOR_ARRAY
;
};
...
...
@@ -146,8 +146,7 @@ class SumOpVarTypeInference : public framework::VarTypeInference {
std
::
ostringstream
os
;
for
(
auto
&
each
:
inputs
)
{
os
<<
" "
<<
each
<<
" type is "
<<
detail
::
Ref
(
block
->
FindRecursiveOrCreateVar
(
each
)).
GetType
()
<<
"
\n
"
;
<<
block
->
FindRecursiveOrCreateVar
(
each
).
GetType
()
<<
"
\n
"
;
}
PADDLE_ENFORCE
(
all_inputs_are_tensor_array
,
"Not all inputs are tensor array:
\n
%s"
,
os
.
str
());
...
...
@@ -158,7 +157,7 @@ class SumOpVarTypeInference : public framework::VarTypeInference {
}
auto
out_var_name
=
op_desc
.
Output
(
"Out"
).
front
();
auto
&
out_var
=
detail
::
Ref
(
block
->
FindRecursiveOrCreateVar
(
out_var_name
)
);
auto
&
out_var
=
block
->
FindRecursiveOrCreateVar
(
out_var_name
);
out_var
.
SetType
(
var_type
);
auto
&
in_var
=
detail
::
Ref
(
block
->
FindVarRecursive
(
inputs
.
front
()));
out_var
.
SetDataType
(
in_var
.
GetDataType
());
...
...
paddle/operators/tensor_array_read_write_op.cc
浏览文件 @
63ff0b4b
...
...
@@ -106,8 +106,7 @@ class WriteToArrayInferVarType : public framework::VarTypeInference {
auto
x_name
=
op_desc
.
Input
(
"X"
)[
0
];
auto
out_name
=
op_desc
.
Output
(
"Out"
)[
0
];
VLOG
(
10
)
<<
"Set Variable "
<<
out_name
<<
" as LOD_TENSOR_ARRAY"
;
auto
&
out
=
detail
::
Ref
(
block
->
FindRecursiveOrCreateVar
(
out_name
),
"Cannot found %s"
,
out_name
);
auto
&
out
=
block
->
FindRecursiveOrCreateVar
(
out_name
);
out
.
SetType
(
framework
::
proto
::
VarDesc
::
LOD_TENSOR_ARRAY
);
auto
*
x
=
block
->
FindVarRecursive
(
x_name
);
if
(
x
!=
nullptr
)
{
...
...
paddle/platform/place.h
浏览文件 @
63ff0b4b
...
...
@@ -52,6 +52,8 @@ struct IsCUDAPlace : public boost::static_visitor<bool> {
typedef
boost
::
variant
<
CUDAPlace
,
CPUPlace
>
Place
;
using
PlaceList
=
std
::
vector
<
Place
>
;
void
set_place
(
const
Place
&
);
const
Place
&
get_place
();
...
...
paddle/pybind/protobuf.cc
浏览文件 @
63ff0b4b
...
...
@@ -231,7 +231,8 @@ void BindVarDsec(py::module &m) {
.
value
(
"FETCH_LIST"
,
proto
::
VarDesc
::
FETCH_LIST
)
.
value
(
"STEP_SCOPES"
,
proto
::
VarDesc
::
STEP_SCOPES
)
.
value
(
"LOD_RANK_TABLE"
,
proto
::
VarDesc
::
LOD_RANK_TABLE
)
.
value
(
"LOD_TENSOR_ARRAY"
,
proto
::
VarDesc
::
LOD_TENSOR_ARRAY
);
.
value
(
"LOD_TENSOR_ARRAY"
,
proto
::
VarDesc
::
LOD_TENSOR_ARRAY
)
.
value
(
"PLACE_LIST"
,
proto
::
VarDesc
::
PLACE_LIST
);
}
void
BindOpDesc
(
py
::
module
&
m
)
{
...
...
python/paddle/v2/fluid/framework.py
浏览文件 @
63ff0b4b
...
...
@@ -448,7 +448,7 @@ class Operator(object):
no_kernel_op_set
=
{
'feed'
,
'fetch'
,
'save'
,
'load'
,
'recurrent'
,
'rnn_memory_helper_grad'
,
'conditional_block'
,
'while'
,
'send'
,
'recv'
,
'
get_places'
,
'
parallel_do'
'recv'
,
'parallel_do'
}
if
type
not
in
no_kernel_op_set
:
self
.
desc
.
infer_var_type
(
self
.
block
.
desc
)
...
...
python/paddle/v2/fluid/layers/devie.py
→
python/paddle/v2/fluid/layers/devi
c
e.py
浏览文件 @
63ff0b4b
...
...
@@ -3,14 +3,14 @@ All util layers.
"""
from
..layer_helper
import
LayerHelper
from
..framework
import
Variabl
e
from
..framework
import
unique_nam
e
__all__
=
[
'get_places'
]
def
get_places
(
device_count
,
device_type
=
"CPU"
):
def
get_places
(
device_count
=
0
,
device_type
=
"CPU"
):
helper
=
LayerHelper
(
'get_places'
,
**
locals
())
out_places
=
helper
.
create_
tmp_variable
(
dtype
=
helper
.
input_dtype
(
))
out_places
=
helper
.
create_
variable
(
name
=
unique_name
(
helper
.
name
+
".out"
))
helper
.
append_op
(
type
=
'get_places'
,
outputs
=
{
"Out"
:
[
out_places
]},
...
...
python/paddle/v2/fluid/tests/test_get_places_op.py
0 → 100644
浏览文件 @
63ff0b4b
import
paddle.v2.fluid
as
fluid
import
decorators
import
unittest
class
TestGetPlaces
(
unittest
.
TestCase
):
@
decorators
.
prog_scope
()
def
test_get_places
(
self
):
places
=
fluid
.
layers
.
get_places
()
cpu
=
fluid
.
CPUPlace
()
exe
=
fluid
.
Executor
(
cpu
)
exe
.
run
(
fluid
.
default_main_program
())
self
.
assertEqual
(
places
.
type
,
fluid
.
core
.
VarDesc
.
VarType
.
PLACE_LIST
)
if
__name__
==
'__main__'
:
unittest
.
main
()
python/paddle/v2/fluid/tests/test_layers.py
浏览文件 @
63ff0b4b
...
...
@@ -200,6 +200,7 @@ class TestBook(unittest.TestCase):
program
=
Program
()
with
program_guard
(
program
):
x
=
layers
.
get_places
(
device_count
=
4
)
self
.
assertIsNotNone
(
x
)
print
(
str
(
program
))
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录