Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
机器未来
Paddle
提交
ac739009
P
Paddle
项目概览
机器未来
/
Paddle
与 Fork 源项目一致
Fork自
PaddlePaddle / Paddle
通知
1
Star
1
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
1
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
P
Paddle
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
1
Issue
1
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
未验证
提交
ac739009
编写于
1月 17, 2018
作者:
Y
Yan Chunwei
提交者:
GitHub
1月 17, 2018
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
enhance/add lod check (#7439)
上级
3dc72a26
变更
3
显示空白变更内容
内联
并排
Showing
3 changed file
with
139 addition
and
30 deletion
+139
-30
paddle/framework/lod_tensor.cc
paddle/framework/lod_tensor.cc
+59
-0
paddle/framework/lod_tensor.h
paddle/framework/lod_tensor.h
+32
-0
paddle/framework/lod_tensor_test.cc
paddle/framework/lod_tensor_test.cc
+48
-30
未找到文件。
paddle/framework/lod_tensor.cc
浏览文件 @
ac739009
...
...
@@ -135,6 +135,65 @@ bool operator==(const LoD &a, const LoD &b) {
return
true
;
}
bool
CheckLoD
(
const
LoD
&
in
,
int
tensor_height
)
{
if
(
in
.
empty
())
return
true
;
for
(
const
auto
&
level
:
in
)
{
// check: there should be more than 2 offsets existing in each level.
if
(
level
.
size
()
<
2
)
return
false
;
// check: the first offset(the begin offset) of each level should be 0.
if
(
level
.
front
()
!=
0
)
return
false
;
// check: all the offsets in a level should be ascending(no same items
// allows).
if
(
!
std
::
is_sorted
(
level
.
begin
(),
level
.
begin
(),
[](
size_t
a
,
size_t
b
)
{
if
(
a
<
b
)
return
true
;
return
false
;
}))
{
LOG
(
INFO
)
<<
"ascending error"
;
return
false
;
}
}
// check: the lowest level's last offset should equals `tensor_height` if
// tensor_height>0.
if
(
tensor_height
>
0
&&
(
size_t
)
tensor_height
!=
in
.
back
().
back
())
return
false
;
// check: the higher level's last offset should equals the lower level's
// size-1.
// NOTE LoD store the levels from top to bottom, so the higher level goes
// first.
for
(
size_t
level
=
0
;
level
<
in
.
size
()
-
1
;
level
++
)
{
if
(
in
[
level
].
back
()
!=
in
[
level
+
1
].
size
()
-
1
)
return
false
;
}
return
true
;
}
bool
CheckAbsLoD
(
const
LoD
&
in
,
int
tensor_height
)
{
if
(
in
.
empty
())
return
true
;
for
(
const
auto
&
level
:
in
)
{
// check: all the offsets in a level should be ascending(no same items
// allows).
if
(
!
std
::
is_sorted
(
level
.
begin
(),
level
.
begin
(),
[](
size_t
a
,
size_t
b
)
{
if
(
a
<
b
)
return
true
;
return
false
;
}))
{
return
false
;
}
// check: there should be more than 2 offsets existing in each level.
if
(
level
.
size
()
<
2
)
return
false
;
// check: the first offset of each level should be 0, and the last should be
// the same(the height of underlying tensor).
if
(
level
.
front
()
!=
0
)
return
false
;
if
(
tensor_height
<
0
)
{
tensor_height
=
level
.
back
();
}
else
if
((
size_t
)
tensor_height
!=
level
.
back
())
{
return
false
;
}
}
return
true
;
}
using
LoDAndOffset
=
std
::
pair
<
LoD
,
std
::
pair
<
size_t
,
size_t
>>
;
LoDAndOffset
GetSubLoDAndAbsoluteOffset
(
const
LoD
&
lod
,
size_t
start_idx
,
size_t
end_idx
,
size_t
start_level
)
{
...
...
paddle/framework/lod_tensor.h
浏览文件 @
ac739009
...
...
@@ -71,6 +71,38 @@ LoD ToAbsOffset(const LoD& in);
bool
operator
==
(
const
LoD
&
a
,
const
LoD
&
b
);
/*
* Check whether this lod's format is valid.
*
* ATTENTION:
* - Empty lod is treated as valid.
*
* It will check two things:
*
* 1. all the offsets in a level should be ascending(no same items allows).
* 2. there should be more than 2 offsets existing in each level.
* 3. the higher level's last offset should equals the lower level's size-1.
* 4. the first offset(the begin offset) of each level should be 0.
* 5. the lowest level's last offset should equals `tensor_height` if
* tensor_height>0.
*/
bool
CheckLoD
(
const
LoD
&
in
,
int
tensor_height
=
-
1
);
/*
* Check whether this absolute lod's format is valid.
*
* ATTENTION:
* - Empty lod is treated as valid.
*
* It will check two things:
* 1. all the offsets in a level should be ascending(no same items allows)
* 2. there should be more than 2 offsets existing in each level.
* 3. the first offset of each level should be 0, and the last should be the
* same(the height of underlying tensor) or `tensor_height` if
* tensor_height>0.
*/
bool
CheckAbsLoD
(
const
LoD
&
in
,
int
tensor_height
=
-
1
);
/*
* LoDTensor (Level of details Tensor)
* see https://en.wikipedia.org/wiki/Level_of_details for reference.
...
...
paddle/framework/lod_tensor_test.cc
浏览文件 @
ac739009
...
...
@@ -37,36 +37,6 @@ namespace framework {
const
int
kLodTensorSize
=
20
*
128
;
class
LoDTensorTester
:
public
::
testing
::
Test
{
public:
virtual
void
SetUp
()
override
{
// tensor's batch_size: 30
// 3 levels
// 0 10 20
// 0 5 10 15 20
// 0 2 5 7 10 12 15 20
LoD
lod
;
lod
.
push_back
(
std
::
vector
<
size_t
>
{
0
,
2
,
3
});
lod
.
push_back
(
std
::
vector
<
size_t
>
{
0
,
2
,
5
,
8
});
lod
.
push_back
(
std
::
vector
<
size_t
>
{
0
,
2
,
5
,
7
,
10
,
12
,
15
,
17
,
20
});
ASSERT_EQ
(
lod
.
size
(),
3UL
);
lod_tensor_
.
Resize
({
20
/*batch size*/
,
128
/*dim*/
});
// malloc memory
float
*
dst_ptr
=
lod_tensor_
.
mutable_data
<
float
>
(
place
);
for
(
int
i
=
0
;
i
<
kLodTensorSize
;
++
i
)
{
dst_ptr
[
i
]
=
i
;
}
lod_tensor_
.
set_lod
(
lod
);
}
protected:
platform
::
CPUPlace
place
;
LoDTensor
lod_tensor_
;
};
TEST
(
LodExpand
,
test
)
{
LoD
lod
{{
0
,
2
}};
LoDTensor
tensor
;
...
...
@@ -144,5 +114,53 @@ TEST(LoD, ToAbsOffset) {
EXPECT_EQ
(
abs_lod
,
expected
);
}
TEST
(
LoD
,
CheckLoD
)
{
LoD
relative_lod
;
relative_lod
.
push_back
(
std
::
vector
<
size_t
>
({
0
,
2
}));
relative_lod
.
push_back
(
std
::
vector
<
size_t
>
({
0
,
1
,
3
}));
relative_lod
.
push_back
(
std
::
vector
<
size_t
>
({
0
,
2
,
4
,
5
}));
// check compatible
ASSERT_TRUE
(
CheckLoD
(
relative_lod
));
relative_lod
[
1
].
back
()
++
;
ASSERT_FALSE
(
CheckLoD
(
relative_lod
));
relative_lod
[
1
].
back
()
--
;
// recover it
// check empty
LoD
empty_lod
;
ASSERT_TRUE
(
CheckLoD
(
empty_lod
));
// check less than 2 offsets in a level
LoD
some_lod0
;
some_lod0
.
push_back
(
std
::
vector
<
size_t
>
({
0
}));
ASSERT_FALSE
(
CheckLoD
(
some_lod0
));
// check with underlying tensor storage.
ASSERT_TRUE
(
CheckLoD
(
relative_lod
,
5
));
ASSERT_FALSE
(
CheckLoD
(
relative_lod
,
9
));
}
TEST
(
LoD
,
CheckAbsLoD
)
{
LoD
relative_lod
;
relative_lod
.
push_back
(
std
::
vector
<
size_t
>
({
0
,
2
}));
relative_lod
.
push_back
(
std
::
vector
<
size_t
>
({
0
,
1
,
3
}));
relative_lod
.
push_back
(
std
::
vector
<
size_t
>
({
0
,
2
,
4
,
5
}));
auto
abs_lod
=
ToAbsOffset
(
relative_lod
);
ASSERT_TRUE
(
CheckAbsLoD
(
abs_lod
));
// check less than 2 offsets in a level.
// check the last item should be compatible with tensor height.
abs_lod
.
back
().
back
()
++
;
ASSERT_FALSE
(
CheckAbsLoD
(
abs_lod
));
abs_lod
.
back
().
back
()
--
;
// restore
// check less than 2 offsets in a lod.
LoD
abs_lod0
;
abs_lod0
.
push_back
(
std
::
vector
<
size_t
>
({
0
}));
ASSERT_FALSE
(
CheckAbsLoD
(
abs_lod0
));
}
}
// namespace framework
}
// namespace paddle
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录