Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
urhero
Paddle
提交
05464e7c
P
Paddle
项目概览
urhero
/
Paddle
与 Fork 源项目一致
Fork自
PaddlePaddle / Paddle
通知
1
Star
0
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
P
Paddle
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
05464e7c
编写于
4月 10, 2019
作者:
D
dongdaxiang
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
add gpu training for Executor.train_from_dataset
test=develop
上级
674aed6a
变更
6
隐藏空白更改
内联
并排
Showing
6 changed file
with
30 addition
and
12 deletion
+30
-12
paddle/fluid/framework/data_feed.cc
paddle/fluid/framework/data_feed.cc
+25
-4
paddle/fluid/framework/data_feed.h
paddle/fluid/framework/data_feed.h
+1
-0
paddle/fluid/framework/data_feed.proto
paddle/fluid/framework/data_feed.proto
+1
-0
paddle/fluid/imperative/nccl_context.h
paddle/fluid/imperative/nccl_context.h
+1
-0
python/paddle/fluid/dataset.py
python/paddle/fluid/dataset.py
+2
-0
python/paddle/fluid/executor.py
python/paddle/fluid/executor.py
+0
-8
未找到文件。
paddle/fluid/framework/data_feed.cc
浏览文件 @
05464e7c
...
...
@@ -466,6 +466,17 @@ void MultiSlotDataFeed::Init(
if
(
slot
.
is_used
())
{
use_slots_
.
push_back
(
all_slots_
[
i
]);
use_slots_is_dense_
.
push_back
(
slot
.
is_dense
());
std
::
vector
<
int
>
local_shape
;
if
(
slot
.
is_dense
())
{
// for batch size holder if is_dense
if
(
slot
.
shape
(
0
)
>
0
)
{
local_shape
.
push_back
(
0
);
}
}
for
(
size_t
i
=
0
;
i
<
slot
.
shape_size
();
++
i
)
{
local_shape
.
push_back
(
slot
.
shape
(
i
));
}
use_slots_shape_
.
push_back
(
local_shape
);
}
}
feed_vec_
.
resize
(
use_slots_
.
size
());
...
...
@@ -752,8 +763,8 @@ void MultiSlotDataFeed::PutToFeedVec(
LoD
data_lod
{
offset
};
feed_vec_
[
i
]
->
set_lod
(
data_lod
);
if
(
use_slots_is_dense_
[
i
])
{
int
dim
=
total_instance
/
batch_size_
;
feed_vec_
[
i
]
->
Resize
(
{
batch_size_
,
dim
}
);
use_slots_shape_
[
i
][
0
]
=
batch_size_
;
feed_vec_
[
i
]
->
Resize
(
framework
::
make_ddim
(
use_slots_shape_
[
i
])
);
}
}
#endif
...
...
@@ -785,6 +796,16 @@ void MultiSlotInMemoryDataFeed::Init(
if
(
slot
.
is_used
())
{
use_slots_
.
push_back
(
all_slots_
[
i
]);
use_slots_is_dense_
.
push_back
(
slot
.
is_dense
());
std
::
vector
<
int
>
local_shape
;
if
(
slot
.
is_dense
())
{
if
(
slot
.
shape
(
0
)
>
0
)
{
local_shape
.
push_back
(
0
);
}
}
for
(
size_t
i
=
0
;
i
<
slot
.
shape_size
();
++
i
)
{
local_shape
.
push_back
(
slot
.
shape
(
i
));
}
use_slots_shape_
.
push_back
(
local_shape
);
}
}
feed_vec_
.
resize
(
use_slots_
.
size
());
...
...
@@ -940,8 +961,8 @@ void MultiSlotInMemoryDataFeed::PutToFeedVec(
LoD
data_lod
{
offset
};
feed_vec_
[
i
]
->
set_lod
(
data_lod
);
if
(
use_slots_is_dense_
[
i
])
{
int
dim
=
total_instance
/
batch_size_
;
feed_vec_
[
i
]
->
Resize
(
{
batch_size_
,
dim
}
);
use_slots_shape_
[
i
][
0
]
=
batch_size_
;
feed_vec_
[
i
]
->
Resize
(
framework
::
make_ddim
(
use_slots_shape_
[
i
])
);
}
}
#endif
...
...
paddle/fluid/framework/data_feed.h
浏览文件 @
05464e7c
...
...
@@ -142,6 +142,7 @@ class DataFeed {
// object)
std
::
vector
<
std
::
string
>
all_slots_
;
std
::
vector
<
std
::
string
>
all_slots_type_
;
std
::
vector
<
std
::
vector
<
int
>>
use_slots_shape_
;
std
::
vector
<
int
>
use_slots_index_
;
// -1: not used; >=0: the index of use_slots_
...
...
paddle/fluid/framework/data_feed.proto
浏览文件 @
05464e7c
...
...
@@ -19,6 +19,7 @@ message Slot {
required
string
type
=
2
;
optional
bool
is_dense
=
3
[
default
=
false
];
optional
bool
is_used
=
4
[
default
=
false
];
repeated
int32
shape
=
5
;
// we can define N-D Tensor
}
message
MultiSlotDesc
{
repeated
Slot
slots
=
1
;
}
...
...
paddle/fluid/imperative/nccl_context.h
浏览文件 @
05464e7c
...
...
@@ -16,6 +16,7 @@
// network header files
#if defined(PADDLE_WITH_CUDA) && !defined(_WIN32)
#include <arpa/inet.h>
#include <asm-generic/socket.h>
#include <netinet/in.h>
#include <stdlib.h>
#include <sys/socket.h>
...
...
python/paddle/fluid/dataset.py
浏览文件 @
05464e7c
...
...
@@ -136,6 +136,8 @@ class DatasetBase(object):
slot_var
.
name
=
var
.
name
if
var
.
lod_level
==
0
:
slot_var
.
is_dense
=
True
print
(
var
.
shape
)
slot_var
.
shape
.
extend
(
var
.
shape
)
if
var
.
dtype
==
core
.
VarDesc
.
VarType
.
FP32
:
slot_var
.
type
=
"float"
elif
var
.
dtype
==
core
.
VarDesc
.
VarType
.
INT64
:
...
...
python/paddle/fluid/executor.py
浏览文件 @
05464e7c
...
...
@@ -712,10 +712,6 @@ class Executor(object):
if
dataset
==
None
:
raise
RuntimeError
(
"dataset is needed and should be initialized"
)
if
not
isinstance
(
self
.
place
,
core
.
CPUPlace
):
raise
RuntimeError
(
"infer_from_dataset is verified on CPUPlace"
"We will open CUDAPlace in the future"
)
scope
,
trainer
=
self
.
_prepare_trainer
(
program
=
program
,
dataset
=
dataset
,
...
...
@@ -796,10 +792,6 @@ class Executor(object):
if
dataset
==
None
:
raise
RuntimeError
(
"dataset is need and should be initialized"
)
if
not
isinstance
(
self
.
place
,
core
.
CPUPlace
):
raise
RuntimeError
(
"train_from_dataset is verified on CPUPlace"
"We will open CUDAPlace in the future"
)
scope
,
trainer
=
self
.
_prepare_trainer
(
program
=
program
,
dataset
=
dataset
,
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录