Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
兽拳
Paddle
提交
d84ddcf1
P
Paddle
项目概览
兽拳
/
Paddle
与 Fork 源项目一致
Fork自
PaddlePaddle / Paddle
通知
1
Star
1
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
1
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
P
Paddle
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
1
Issue
1
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
d84ddcf1
编写于
3月 15, 2018
作者:
Y
Yu Yang
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
Stash
上级
193c0a7e
变更
5
隐藏空白更改
内联
并排
Showing
5 changed file
with
31 addition
and
11 deletion
+31
-11
paddle/fluid/framework/executor.cc
paddle/fluid/framework/executor.cc
+4
-4
paddle/fluid/framework/executor.h
paddle/fluid/framework/executor.h
+2
-0
paddle/fluid/framework/parallel_executor.cc
paddle/fluid/framework/parallel_executor.cc
+4
-5
paddle/fluid/operators/reader/create_recordio_file_reader_op.cc
.../fluid/operators/reader/create_recordio_file_reader_op.cc
+3
-1
python/paddle/fluid/tests/unittests/test_parallel_executor.py
...on/paddle/fluid/tests/unittests/test_parallel_executor.py
+18
-1
未找到文件。
paddle/fluid/framework/executor.cc
浏览文件 @
d84ddcf1
...
@@ -45,7 +45,7 @@ struct ExecutorPrepareContext {
...
@@ -45,7 +45,7 @@ struct ExecutorPrepareContext {
Executor
::
Executor
(
const
platform
::
Place
&
place
)
:
place_
(
place
)
{}
Executor
::
Executor
(
const
platform
::
Place
&
place
)
:
place_
(
place
)
{}
static
void
CreateTensor
(
Variable
*
var
,
proto
::
VarType
::
Type
var_type
)
{
void
InitializeVariable
(
Variable
*
var
,
proto
::
VarType
::
Type
var_type
)
{
if
(
var_type
==
proto
::
VarType
::
LOD_TENSOR
)
{
if
(
var_type
==
proto
::
VarType
::
LOD_TENSOR
)
{
var
->
GetMutable
<
LoDTensor
>
();
var
->
GetMutable
<
LoDTensor
>
();
}
else
if
(
var_type
==
proto
::
VarType
::
SELECTED_ROWS
)
{
}
else
if
(
var_type
==
proto
::
VarType
::
SELECTED_ROWS
)
{
...
@@ -284,12 +284,12 @@ void Executor::RunPreparedContext(ExecutorPrepareContext* ctx, Scope* scope,
...
@@ -284,12 +284,12 @@ void Executor::RunPreparedContext(ExecutorPrepareContext* ctx, Scope* scope,
if
(
var
->
Persistable
())
{
if
(
var
->
Persistable
())
{
auto
*
ptr
=
scope
->
Var
(
var
->
Name
());
auto
*
ptr
=
scope
->
Var
(
var
->
Name
());
CreateTensor
(
ptr
,
var
->
GetType
());
InitializeVariable
(
ptr
,
var
->
GetType
());
VLOG
(
3
)
<<
"Create Variable "
<<
var
->
Name
()
VLOG
(
3
)
<<
"Create Variable "
<<
var
->
Name
()
<<
" global, which pointer is "
<<
ptr
;
<<
" global, which pointer is "
<<
ptr
;
}
else
{
}
else
{
auto
*
ptr
=
local_scope
->
Var
(
var
->
Name
());
auto
*
ptr
=
local_scope
->
Var
(
var
->
Name
());
CreateTensor
(
ptr
,
var
->
GetType
());
InitializeVariable
(
ptr
,
var
->
GetType
());
VLOG
(
3
)
<<
"Create Variable "
<<
var
->
Name
()
VLOG
(
3
)
<<
"Create Variable "
<<
var
->
Name
()
<<
" locally, which pointer is "
<<
ptr
;
<<
" locally, which pointer is "
<<
ptr
;
}
}
...
@@ -297,7 +297,7 @@ void Executor::RunPreparedContext(ExecutorPrepareContext* ctx, Scope* scope,
...
@@ -297,7 +297,7 @@ void Executor::RunPreparedContext(ExecutorPrepareContext* ctx, Scope* scope,
}
else
{
}
else
{
for
(
auto
&
var
:
block
.
AllVars
())
{
for
(
auto
&
var
:
block
.
AllVars
())
{
auto
*
ptr
=
local_scope
->
Var
(
var
->
Name
());
auto
*
ptr
=
local_scope
->
Var
(
var
->
Name
());
CreateTensor
(
ptr
,
var
->
GetType
());
InitializeVariable
(
ptr
,
var
->
GetType
());
VLOG
(
3
)
<<
"Create variable "
<<
var
->
Name
()
<<
", which pointer is "
VLOG
(
3
)
<<
"Create variable "
<<
var
->
Name
()
<<
", which pointer is "
<<
ptr
;
<<
ptr
;
}
}
...
...
paddle/fluid/framework/executor.h
浏览文件 @
d84ddcf1
...
@@ -59,5 +59,7 @@ class Executor {
...
@@ -59,5 +59,7 @@ class Executor {
const
platform
::
Place
place_
;
const
platform
::
Place
place_
;
};
};
extern
void
InitializeVariable
(
Variable
*
var
,
proto
::
VarType
::
Type
var_type
);
}
// namespace framework
}
// namespace framework
}
// namespace paddle
}
// namespace paddle
paddle/fluid/framework/parallel_executor.cc
浏览文件 @
d84ddcf1
...
@@ -84,14 +84,14 @@ struct ComputationOpHandle : public OpHandle {
...
@@ -84,14 +84,14 @@ struct ComputationOpHandle : public OpHandle {
Scope
*
scope_
;
Scope
*
scope_
;
platform
::
Place
place_
;
platform
::
Place
place_
;
explicit
ComputationOpHandle
(
const
OpDesc
&
op_desc
,
Scope
*
scope
,
explicit
ComputationOpHandle
(
const
OpDesc
&
op_desc
,
platform
::
Place
place
)
platform
::
Place
place
)
:
op_
(
framework
::
OpRegistry
::
CreateOp
(
op_desc
)),
:
op_
(
framework
::
OpRegistry
::
CreateOp
(
op_desc
)),
scope_
(
scope
),
scope_
(
nullptr
),
place_
(
place
)
{}
place_
(
place
)
{}
void
Run
()
override
{
void
Run
()
override
{
// Wait other op if necessary
// Wait other op if necessary
LOG
(
INFO
)
<<
DebugString
();
auto
*
cur_ctx
=
dev_ctx_
[
place_
];
auto
*
cur_ctx
=
dev_ctx_
[
place_
];
for
(
auto
*
in
:
inputs_
)
{
for
(
auto
*
in
:
inputs_
)
{
if
(
in
->
generated_op_
&&
in
->
generated_op_
->
dev_ctx_
[
place_
]
!=
cur_ctx
)
{
if
(
in
->
generated_op_
&&
in
->
generated_op_
->
dev_ctx_
[
place_
]
!=
cur_ctx
)
{
...
@@ -240,8 +240,7 @@ void ParallelExecutor::ConstructDependencyGraph(
...
@@ -240,8 +240,7 @@ void ParallelExecutor::ConstructDependencyGraph(
}
}
for
(
auto
&
pair
:
member_
->
local_scopes_
)
{
for
(
auto
&
pair
:
member_
->
local_scopes_
)
{
member_
->
ops_
.
emplace_back
(
member_
->
ops_
.
emplace_back
(
new
ComputationOpHandle
(
*
op
,
pair
.
first
));
new
ComputationOpHandle
(
*
op
,
pair
.
second
,
pair
.
first
));
auto
*
op_handle
=
member_
->
ops_
.
back
().
get
();
auto
*
op_handle
=
member_
->
ops_
.
back
().
get
();
op_handle
->
dev_ctx_
[
pair
.
first
]
=
const_cast
<
platform
::
DeviceContext
*>
(
op_handle
->
dev_ctx_
[
pair
.
first
]
=
const_cast
<
platform
::
DeviceContext
*>
(
platform
::
DeviceContextPool
::
Instance
().
Get
(
pair
.
first
));
platform
::
DeviceContextPool
::
Instance
().
Get
(
pair
.
first
));
...
...
paddle/fluid/operators/reader/create_recordio_file_reader_op.cc
浏览文件 @
d84ddcf1
...
@@ -25,7 +25,9 @@ class RecordIOFileReader : public framework::FileReader {
...
@@ -25,7 +25,9 @@ class RecordIOFileReader : public framework::FileReader {
:
FileReader
(
shapes
),
:
FileReader
(
shapes
),
scanner_
(
filename
),
scanner_
(
filename
),
dev_ctx_
(
*
platform
::
DeviceContextPool
::
Instance
().
Get
(
dev_ctx_
(
*
platform
::
DeviceContextPool
::
Instance
().
Get
(
platform
::
CPUPlace
()))
{}
platform
::
CPUPlace
()))
{
LOG
(
INFO
)
<<
"Creating file reader"
<<
filename
;
}
void
ReadNext
(
std
::
vector
<
framework
::
LoDTensor
>*
out
)
override
{
void
ReadNext
(
std
::
vector
<
framework
::
LoDTensor
>*
out
)
override
{
*
out
=
framework
::
ReadFromRecordIO
(
scanner_
,
dev_ctx_
);
*
out
=
framework
::
ReadFromRecordIO
(
scanner_
,
dev_ctx_
);
...
...
python/paddle/fluid/tests/unittests/test_parallel_executor.py
浏览文件 @
d84ddcf1
...
@@ -14,16 +14,33 @@
...
@@ -14,16 +14,33 @@
import
unittest
import
unittest
import
paddle.fluid
as
fluid
import
paddle.fluid
as
fluid
import
paddle.v2
as
paddle
import
paddle.v2.dataset.mnist
as
mnist
class
ParallelExecutor
(
unittest
.
TestCase
):
class
ParallelExecutor
(
unittest
.
TestCase
):
def
setUp
(
self
):
# Convert mnist to recordio file
with
fluid
.
program_guard
(
fluid
.
Program
(),
fluid
.
Program
()):
reader
=
paddle
.
batch
(
mnist
.
train
(),
batch_size
=
32
)
feeder
=
fluid
.
DataFeeder
(
feed_list
=
[
# order is image and label
fluid
.
layers
.
data
(
name
=
'image'
,
shape
=
[
784
]),
fluid
.
layers
.
data
(
name
=
'label'
,
shape
=
[
1
],
dtype
=
'int64'
),
],
place
=
fluid
.
CPUPlace
())
fluid
.
recordio_writer
.
convert_reader_to_recordio_file
(
'./mnist.recordio'
,
reader
,
feeder
)
def
test_main
(
self
):
def
test_main
(
self
):
main
=
fluid
.
Program
()
main
=
fluid
.
Program
()
startup
=
fluid
.
Program
()
startup
=
fluid
.
Program
()
with
fluid
.
program_guard
(
main
,
startup
):
with
fluid
.
program_guard
(
main
,
startup
):
reader
=
fluid
.
layers
.
open_recordio_file
(
reader
=
fluid
.
layers
.
open_recordio_file
(
filename
=
'
tmp
'
,
filename
=
'
./mnist.recordio
'
,
shapes
=
[[
-
1
,
784
],
[
-
1
,
1
]],
shapes
=
[[
-
1
,
784
],
[
-
1
,
1
]],
lod_levels
=
[
0
,
0
],
lod_levels
=
[
0
,
0
],
dtypes
=
[
'float32'
,
'int64'
])
dtypes
=
[
'float32'
,
'int64'
])
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录