Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
TonyTonyFun
Paddle
提交
0f99d240
P
Paddle
项目概览
TonyTonyFun
/
Paddle
与 Fork 源项目一致
Fork自
PaddlePaddle / Paddle
通知
1
Star
0
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
P
Paddle
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
未验证
提交
0f99d240
编写于
3月 05, 2019
作者:
W
whs
提交者:
GitHub
3月 05, 2019
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
Make sequence_erase op support for input with multi-level LoD. (#15982)
test=develop
上级
02c106c7
变更
3
隐藏空白更改
内联
并排
Showing
3 changed file
with
35 addition
and
17 deletion
+35
-17
paddle/fluid/operators/sequence_ops/sequence_erase_op.cu
paddle/fluid/operators/sequence_ops/sequence_erase_op.cu
+10
-9
paddle/fluid/operators/sequence_ops/sequence_erase_op.h
paddle/fluid/operators/sequence_ops/sequence_erase_op.h
+10
-8
python/paddle/fluid/tests/unittests/test_sequence_erase_op.py
...on/paddle/fluid/tests/unittests/test_sequence_erase_op.py
+15
-0
未找到文件。
paddle/fluid/operators/sequence_ops/sequence_erase_op.cu
浏览文件 @
0f99d240
...
...
@@ -64,8 +64,7 @@ class SequenceEraseOpCUDAKernel : public framework::OpKernel<T> {
auto
*
out
=
ctx
.
Output
<
LoDTensor
>
(
"Out"
);
auto
lod
=
in
->
lod
();
PADDLE_ENFORCE_EQ
(
lod
.
size
(),
1UL
,
"Only support one level sequence now."
);
PADDLE_ENFORCE_EQ
(
lod
[
0
].
back
(),
(
size_t
)
in
->
numel
(),
PADDLE_ENFORCE_EQ
(
lod
[
lod
.
size
()
-
1
].
back
(),
(
size_t
)
in
->
numel
(),
"The actual size mismatches with the LoD information."
);
auto
tokens
=
ctx
.
Attr
<
std
::
vector
<
int
>>
(
"tokens"
);
auto
in_len
=
in
->
numel
();
...
...
@@ -85,10 +84,9 @@ class SequenceEraseOpCUDAKernel : public framework::OpKernel<T> {
num_erased
.
begin
()
+
1
);
// Copy LoD to GPU
auto
lod0
=
lod
[
0
];
auto
lod_len
=
lod0
.
size
();
const
size_t
*
dev_in_lod_ptr
=
lod0
.
CUDAData
(
ctx
.
GetPlace
());
auto
last_lod
=
lod
[
lod
.
size
()
-
1
];
auto
lod_len
=
last_lod
.
size
();
const
size_t
*
dev_in_lod_ptr
=
last_lod
.
CUDAData
(
ctx
.
GetPlace
());
// Calc output LoD
thrust
::
device_vector
<
size_t
>
dev_out_lod
(
lod_len
);
size_t
*
dev_out_lod_ptr
=
thrust
::
raw_pointer_cast
(
dev_out_lod
.
data
());
...
...
@@ -96,13 +94,16 @@ class SequenceEraseOpCUDAKernel : public framework::OpKernel<T> {
PADDLE_CUDA_NUM_THREADS
,
0
,
stream
>>>
(
num_erased_ptr
,
dev_in_lod_ptr
,
lod_len
,
dev_out_lod_ptr
);
// Set LoD for output
std
::
vector
<
size_t
>
out_l
od0
(
dev_out_lod
.
begin
(),
dev_out_lod
.
end
());
std
::
vector
<
size_t
>
out_l
ast_lod
(
dev_out_lod
.
begin
(),
dev_out_lod
.
end
());
framework
::
LoD
out_lod
;
out_lod
.
push_back
(
out_lod0
);
for
(
size_t
i
=
0
;
i
<
lod
.
size
()
-
1
;
++
i
)
{
out_lod
.
push_back
(
lod
[
i
]);
}
out_lod
.
push_back
(
out_last_lod
);
out
->
set_lod
(
out_lod
);
// Set output
out
->
Resize
({
static_cast
<
int64_t
>
(
out_l
od0
.
back
()),
1
});
out
->
Resize
({
static_cast
<
int64_t
>
(
out_l
ast_lod
.
back
()),
1
});
auto
out_dat
=
out
->
mutable_data
<
T
>
(
ctx
.
GetPlace
());
SetOutput
<<<
(
in_len
-
1
)
/
PADDLE_CUDA_NUM_THREADS
+
1
,
PADDLE_CUDA_NUM_THREADS
,
0
,
stream
>>>
(
in_dat
,
in_len
,
...
...
paddle/fluid/operators/sequence_ops/sequence_erase_op.h
浏览文件 @
0f99d240
...
...
@@ -28,19 +28,18 @@ class SequenceEraseKernel : public framework::OpKernel<T> {
auto
*
out
=
ctx
.
Output
<
framework
::
LoDTensor
>
(
"Out"
);
auto
lod
=
in
->
lod
();
PADDLE_ENFORCE_EQ
(
lod
.
size
(),
1UL
,
"Only support one level sequence now."
);
PADDLE_ENFORCE_EQ
(
lod
[
0
].
back
(),
(
size_t
)
in
->
numel
(),
PADDLE_ENFORCE_EQ
(
lod
[
lod
.
size
()
-
1
].
back
(),
(
size_t
)
in
->
numel
(),
"The actual size mismatches with the LoD information."
);
auto
tokens
=
ctx
.
Attr
<
std
::
vector
<
int
>>
(
"tokens"
);
auto
in_len
=
in
->
numel
();
auto
in_dat
=
in
->
data
<
T
>
();
auto
l
od0
=
lod
[
0
];
auto
l
ast_lod
=
lod
[
lod
.
size
()
-
1
];
std
::
vector
<
size_t
>
num_erased
(
in_len
+
1
,
0
);
std
::
vector
<
size_t
>
out_l
od0
(
1
,
0
);
for
(
size_t
i
=
0
;
i
<
l
od0
.
size
()
-
1
;
++
i
)
{
std
::
vector
<
size_t
>
out_l
ast_lod
(
1
,
0
);
for
(
size_t
i
=
0
;
i
<
l
ast_lod
.
size
()
-
1
;
++
i
)
{
size_t
num_out
=
0
;
for
(
auto
j
=
l
od0
[
i
]
+
1
;
j
<=
lod0
[
i
+
1
];
++
j
)
{
for
(
auto
j
=
l
ast_lod
[
i
]
+
1
;
j
<=
last_lod
[
i
+
1
];
++
j
)
{
num_erased
[
j
]
=
num_erased
[
j
-
1
];
if
(
std
::
find
(
tokens
.
begin
(),
tokens
.
end
(),
in_dat
[
j
-
1
])
!=
tokens
.
end
())
{
...
...
@@ -49,7 +48,7 @@ class SequenceEraseKernel : public framework::OpKernel<T> {
num_out
+=
1
;
}
}
out_l
od0
.
push_back
(
out_lod0
.
back
()
+
num_out
);
out_l
ast_lod
.
push_back
(
out_last_lod
.
back
()
+
num_out
);
}
auto
out_len
=
in_len
-
num_erased
[
in_len
];
...
...
@@ -62,7 +61,10 @@ class SequenceEraseKernel : public framework::OpKernel<T> {
}
}
framework
::
LoD
out_lod
;
out_lod
.
push_back
(
out_lod0
);
for
(
size_t
i
=
0
;
i
<
lod
.
size
()
-
1
;
++
i
)
{
out_lod
.
push_back
(
lod
[
i
]);
}
out_lod
.
push_back
(
out_last_lod
);
out
->
set_lod
(
out_lod
);
}
};
...
...
python/paddle/fluid/tests/unittests/test_sequence_erase_op.py
浏览文件 @
0f99d240
...
...
@@ -49,6 +49,21 @@ class TestSequenceEraseOpInt32(OpTest):
self
.
check_output
()
class
TestSequenceEraseOpInt32LoD2
(
OpTest
):
def
setUp
(
self
):
self
.
op_type
=
"sequence_erase"
in_seq
=
np
.
random
.
randint
(
0
,
10
,
(
30
,
1
)).
astype
(
"int32"
)
lod
=
[[
1
,
3
],
[
9
,
4
,
11
,
6
]]
tokens
=
[
2
,
3
,
5
]
out_seq
,
new_lod0
=
sequence_erase
(
in_seq
,
lod
[
-
1
],
tokens
)
self
.
attrs
=
{
'tokens'
:
tokens
}
self
.
inputs
=
{
'X'
:
(
in_seq
,
lod
)}
self
.
outputs
=
{
'Out'
:
(
out_seq
,
lod
[:
-
1
]
+
[
new_lod0
])}
def
test_check_output
(
self
):
self
.
check_output
()
class
TestSequenceEraseOpInt64
(
OpTest
):
def
setUp
(
self
):
self
.
op_type
=
"sequence_erase"
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录