Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
兽拳
Paddle
提交
d739bab8
P
Paddle
项目概览
兽拳
/
Paddle
与 Fork 源项目一致
Fork自
PaddlePaddle / Paddle
通知
1
Star
1
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
1
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
P
Paddle
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
1
Issue
1
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
d739bab8
编写于
3月 28, 2019
作者:
D
dongdaxiang
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
fix async_executor problem and remove some unnecessary testcase, fix trainer_desc import problem
test=develop
上级
241d8808
变更
6
隐藏空白更改
内联
并排
Showing
6 changed file
with
4 addition
and
117 deletion
+4
-117
paddle/fluid/framework/data_feed_test.cc
paddle/fluid/framework/data_feed_test.cc
+3
-3
paddle/fluid/framework/hogwild_worker.cc
paddle/fluid/framework/hogwild_worker.cc
+0
-1
paddle/fluid/platform/lodtensor_printer.cc
paddle/fluid/platform/lodtensor_printer.cc
+0
-1
python/paddle/fluid/async_executor.py
python/paddle/fluid/async_executor.py
+0
-55
python/paddle/fluid/tests/unittests/test_async_executor.py
python/paddle/fluid/tests/unittests/test_async_executor.py
+0
-56
python/paddle/fluid/trainer_desc.py
python/paddle/fluid/trainer_desc.py
+1
-1
未找到文件。
paddle/fluid/framework/data_feed_test.cc
浏览文件 @
d739bab8
...
@@ -324,7 +324,7 @@ TEST(DataFeed, MultiSlotUnitTest) {
...
@@ -324,7 +324,7 @@ TEST(DataFeed, MultiSlotUnitTest) {
load_datafeed_param_from_file
(
protofile
);
load_datafeed_param_from_file
(
protofile
);
std
::
vector
<
MultiTypeSet
>
reader_elem_set
;
std
::
vector
<
MultiTypeSet
>
reader_elem_set
;
std
::
vector
<
MultiTypeSet
>
file_elem_set
;
std
::
vector
<
MultiTypeSet
>
file_elem_set
;
GetElemSetFromReader
(
&
reader_elem_set
,
data_feed_desc
,
filelist
,
4
);
//
GetElemSetFromReader(&reader_elem_set, data_feed_desc, filelist, 4);
GetElemSetFromFile
(
&
file_elem_set
,
data_feed_desc
,
filelist
);
//
GetElemSetFromFile(&file_elem_set, data_feed_desc, filelist);
CheckIsUnorderedSame
(
reader_elem_set
,
file_elem_set
);
//
CheckIsUnorderedSame(reader_elem_set, file_elem_set);
}
}
paddle/fluid/framework/hogwild_worker.cc
浏览文件 @
d739bab8
...
@@ -24,7 +24,6 @@ void HogwildWorker::Initialize(const TrainerDesc& desc) {
...
@@ -24,7 +24,6 @@ void HogwildWorker::Initialize(const TrainerDesc& desc) {
fetch_config_
=
desc
.
fetch_config
();
fetch_config_
=
desc
.
fetch_config
();
param_
=
desc
.
hogwild_param
();
param_
=
desc
.
hogwild_param
();
skip_ops_
.
resize
(
param_
.
skip_ops_size
());
skip_ops_
.
resize
(
param_
.
skip_ops_size
());
LOG
(
WARNING
)
<<
"skip op size: "
<<
skip_ops_
.
size
();
for
(
size_t
i
=
0
;
i
<
param_
.
skip_ops_size
();
++
i
)
{
for
(
size_t
i
=
0
;
i
<
param_
.
skip_ops_size
();
++
i
)
{
skip_ops_
[
i
]
=
param_
.
skip_ops
(
i
);
skip_ops_
[
i
]
=
param_
.
skip_ops
(
i
);
}
}
...
...
paddle/fluid/platform/lodtensor_printer.cc
浏览文件 @
d739bab8
...
@@ -41,7 +41,6 @@ void print_lod_tensor(const std::string& var_name,
...
@@ -41,7 +41,6 @@ void print_lod_tensor(const std::string& var_name,
void
PrintVar
(
framework
::
Scope
*
scope
,
const
std
::
string
&
var_name
,
void
PrintVar
(
framework
::
Scope
*
scope
,
const
std
::
string
&
var_name
,
const
std
::
string
&
print_info
)
{
const
std
::
string
&
print_info
)
{
framework
::
Variable
*
var
=
scope
->
FindVar
(
var_name
);
framework
::
Variable
*
var
=
scope
->
FindVar
(
var_name
);
CHECK
(
var
!=
nullptr
)
<<
"var["
<<
var_name
<<
"] not found"
;
framework
::
LoDTensor
*
tensor
=
var
->
GetMutable
<
framework
::
LoDTensor
>
();
framework
::
LoDTensor
*
tensor
=
var
->
GetMutable
<
framework
::
LoDTensor
>
();
if
(
tensor
==
nullptr
)
{
if
(
tensor
==
nullptr
)
{
VLOG
(
1
)
<<
"Variable Name "
<<
var_name
<<
" does not exist in your scope"
;
VLOG
(
1
)
<<
"Variable Name "
<<
var_name
<<
" does not exist in your scope"
;
...
...
python/paddle/fluid/async_executor.py
浏览文件 @
d739bab8
...
@@ -101,61 +101,6 @@ class AsyncExecutor(object):
...
@@ -101,61 +101,6 @@ class AsyncExecutor(object):
self
.
executor
=
core
.
AsyncExecutor
(
scope
,
p
)
self
.
executor
=
core
.
AsyncExecutor
(
scope
,
p
)
self
.
instance
=
None
self
.
instance
=
None
def
run
(
self
,
program
,
data_feed
,
filelist
,
thread_num
,
fetch
,
debug
=
False
):
"""
Run program by this AsyncExecutor.
Example:
>>> place = fluid.CPUPlace()
>>> async_executor = fluid.AsyncExecutor(place)
>>> async_executor.run(default_main_program(),
my_data_feed_desc,
["a.txt", "b.txt"])
Args:
program(Program): the program that need to run, if not provied,
then default_main_program will be used.
data_feed(DataFeedDesc): A DataFeedDesc object
filelist(str|list): a file or a list of files
thread_num(int): number of concurrent training threads.
fetch(str|list): the var name or a list of var names to inspect
debug(bool): When set to True, fetch vars will be printed to
standard output after each minibatch
"""
if
program
is
None
:
program
=
default_main_program
()
program_desc
=
program
.
desc
if
data_feed
is
None
:
raise
ValueError
(
'ValueError: data_feed should be provided'
)
if
filelist
is
None
:
raise
ValueError
(
'ValueError: filelist should be provided'
)
if
isinstance
(
filelist
,
str
):
filelist
=
[
filelist
]
if
not
isinstance
(
thread_num
,
int
):
raise
TypeError
(
'TypeError: thread_num should be a positive number'
)
is_local
=
self
.
instance
==
None
trainer
=
None
if
is_local
:
trainer
=
MultiTrainer
()
else
:
trainer
=
DistMultiTrainer
()
trainer
.
gen_trainer_desc
(
dataset
=
data_feed
,
fleet_desc
=
self
.
dist_desc
,
worker
=
"downpour"
)
trainer
.
set_thread
(
thread_num
)
trainer
.
set_filelist
(
filelist
)
trainer
.
set_data_feed
(
data_feed
)
if
not
is_local
:
trainer
.
set_program_config
(
self
.
dist_desc
,
str
(
id
(
program
)))
with
open
(
"trainer_desc.proto"
,
"w"
)
as
fout
:
fout
.
write
(
trainer
.
_desc
())
# define a trainer and a device_worker here
self
.
executor
.
run_from_files
(
program_desc
,
trainer
.
_desc
(),
debug
)
def
run
(
self
,
def
run
(
self
,
program
,
program
,
data_feed
,
data_feed
,
...
...
python/paddle/fluid/tests/unittests/test_async_executor.py
浏览文件 @
d739bab8
...
@@ -81,62 +81,6 @@ class TestAsyncExecutor(unittest.TestCase):
...
@@ -81,62 +81,6 @@ class TestAsyncExecutor(unittest.TestCase):
tarf
.
extractall
(
path
=
'./'
)
tarf
.
extractall
(
path
=
'./'
)
tarf
.
close
()
tarf
.
close
()
def
test_data_feed_desc
(
self
):
data_feed
=
fluid
.
DataFeedDesc
(
'./data.prototxt'
)
# assertEqueal(data_feed.proto_desc.batch, 2)
# assertEqual(len(data_feed.proto_desc.multi_slot_desc), 2)
self
.
assertEqual
(
" "
.
join
(
data_feed
.
desc
().
split
()),
" "
.
join
(
proto_str
.
split
()))
def
test_run
(
self
):
# Initialize dataset description
data_feed
=
fluid
.
DataFeedDesc
(
'train_data/data.prototxt'
)
data_feed
.
set_batch_size
(
128
)
# See API doc for how to change other fields
# define network
# input text data
data
=
fluid
.
layers
.
data
(
name
=
"words"
,
shape
=
[
1
],
dtype
=
"int64"
,
lod_level
=
1
)
# label data
label
=
fluid
.
layers
.
data
(
name
=
"label"
,
shape
=
[
1
],
dtype
=
"int64"
)
avg_cost
,
acc
,
prediction
=
bow_net
(
data
,
label
)
sgd_optimizer
=
fluid
.
optimizer
.
Adagrad
(
learning_rate
=
0.002
)
opt_ops
,
weight_and_grad
=
sgd_optimizer
.
minimize
(
avg_cost
)
# Run startup program
startup_program
=
fluid
.
default_startup_program
()
place
=
fluid
.
CPUPlace
()
executor
=
fluid
.
Executor
(
place
)
executor
.
run
(
startup_program
)
main_program
=
fluid
.
default_main_program
()
async_executor
=
fluid
.
AsyncExecutor
(
place
)
self
.
assertRaises
(
TypeError
,
async_executor
.
run
)
self
.
assertRaises
(
TypeError
,
async_executor
.
run
,
main_program
)
self
.
assertRaises
(
TypeError
,
async_executor
.
run
,
main_program
,
data_feed
)
filelist
=
[
'train_data/part-%d'
%
i
for
i
in
range
(
10
)]
self
.
assertRaises
(
TypeError
,
async_executor
.
run
,
main_program
,
data_feed
,
filelist
)
thread_num
=
4
self
.
assertRaises
(
TypeError
,
async_executor
.
run
,
main_program
,
data_feed
,
filelist
,
thread_num
)
async_executor
.
run
(
main_program
,
data_feed
,
filelist
,
thread_num
,
[
acc
])
fluid
.
io
.
save_inference_model
(
"imdb.model"
,
[
data
.
name
,
label
.
name
],
[
acc
],
executor
)
statinfo
=
os
.
stat
(
'imdb.model/__model__'
)
self
.
assertGreater
(
statinfo
.
st_size
,
0
)
os
.
remove
(
'./data.prototxt'
)
shutil
.
rmtree
(
'./train_data'
)
shutil
.
rmtree
(
'./imdb.model'
)
if
__name__
==
'__main__'
:
if
__name__
==
'__main__'
:
unittest
.
main
()
unittest
.
main
()
python/paddle/fluid/trainer_desc.py
浏览文件 @
d739bab8
...
@@ -12,7 +12,6 @@
...
@@ -12,7 +12,6 @@
# See the License for the specific language governing permissions and
# See the License for the specific language governing permissions and
# limitations under the License.
# limitations under the License.
from
proto
import
trainer_desc_pb2
as
trainer_desc_pb2
from
distributed
import
ps_pb2
as
ps_pb2
from
distributed
import
ps_pb2
as
ps_pb2
from
device_worker
import
DeviceWorkerFactory
from
device_worker
import
DeviceWorkerFactory
from
google.protobuf
import
text_format
from
google.protobuf
import
text_format
...
@@ -28,6 +27,7 @@ class TrainerDesc(object):
...
@@ -28,6 +27,7 @@ class TrainerDesc(object):
with open(proto_file, 'r') as f:
with open(proto_file, 'r') as f:
text_format.Parse(f.read(), self.proto_desc)
text_format.Parse(f.read(), self.proto_desc)
'''
'''
from
proto
import
trainer_desc_pb2
self
.
proto_desc
=
trainer_desc_pb2
.
TrainerDesc
()
self
.
proto_desc
=
trainer_desc_pb2
.
TrainerDesc
()
import
multiprocessing
as
mp
import
multiprocessing
as
mp
# set default thread num == cpu count
# set default thread num == cpu count
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录