Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
wzpei9
Paddle
提交
db7b1171
P
Paddle
项目概览
wzpei9
/
Paddle
与 Fork 源项目一致
Fork自
PaddlePaddle / Paddle
通知
1
Star
1
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
1
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
P
Paddle
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
1
Issue
1
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
db7b1171
编写于
10月 20, 2017
作者:
Y
Yan Chunwei
提交者:
GitHub
10月 20, 2017
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
change lod tensor to absolute offsets (#4952)
上级
1680903d
变更
3
隐藏空白更改
内联
并排
Showing
3 changed file
with
73 addition
and
52 deletion
+73
-52
paddle/framework/lod_tensor.cc
paddle/framework/lod_tensor.cc
+39
-30
paddle/framework/lod_tensor.h
paddle/framework/lod_tensor.h
+19
-6
paddle/framework/lod_tensor_test.cc
paddle/framework/lod_tensor_test.cc
+15
-16
未找到文件。
paddle/framework/lod_tensor.cc
浏览文件 @
db7b1171
...
...
@@ -25,31 +25,50 @@ LoD SliceLevels(const LoD& in, size_t level_begin, size_t level_end) {
for
(
size_t
i
=
level_begin
;
i
<
level_end
;
i
++
)
{
new_lod
.
emplace_back
(
in
.
at
(
i
));
}
// transform the lowest level to absolute offset.
LoD
abs_offset_lod
=
ToAbsOffset
(
in
);
new_lod
.
back
()
=
abs_offset_lod
[
level_end
-
1
];
return
new_lod
;
}
LoD
SliceInLevel
(
const
LoD
&
in
,
size_t
level
,
size_t
elem_begin
,
size_t
elem_end
)
{
// slice the lod.
LoD
new_lod
;
new_lod
.
reserve
(
in
.
size
()
-
level
);
auto
start
=
in
.
at
(
level
)[
elem_begin
];
auto
end
=
in
.
at
(
level
)[
elem_end
];
for
(
auto
it
=
in
.
begin
()
+
level
;
it
!=
in
.
end
();
it
++
)
{
auto
it_begin
=
std
::
find
(
it
->
begin
(),
it
->
end
(),
start
);
auto
it_end
=
std
::
find
(
it_begin
,
it
->
end
(),
end
);
PADDLE_ENFORCE
(
it_begin
!=
it
->
end
(),
"error in parsing lod info"
);
PADDLE_ENFORCE
(
it_end
!=
it
->
end
(),
"error in parsing lod info"
);
new_lod
.
emplace_back
(
it_begin
,
it_end
+
1
);
// reset offset if tensor is copyed and sliced.
std
::
transform
(
new_lod
.
back
().
begin
(),
new_lod
.
back
().
end
(),
new_lod
.
back
().
begin
(),
[
start
](
int
v
)
{
return
v
-
start
;
});
PADDLE_ENFORCE_EQ
(
new_lod
.
back
().
front
(),
0
,
"error in slice LoD"
);
PADDLE_ENFORCE_LT
(
level
,
in
.
size
());
PADDLE_ENFORCE_LT
(
elem_end
,
in
[
level
].
size
());
LoD
res
;
res
.
resize
(
in
.
size
()
-
level
);
// copy the first level
res
[
0
].
assign
(
in
[
level
].
begin
()
+
elem_begin
,
in
[
level
].
begin
()
+
elem_end
+
1
);
for
(
size_t
lvl
=
1
;
lvl
<
res
.
size
();
lvl
++
)
{
const
auto
&
in_level
=
in
[
level
+
lvl
];
const
auto
&
above_level
=
res
[
lvl
-
1
];
auto
&
out_level
=
res
[
lvl
];
out_level
.
assign
(
in_level
.
begin
()
+
above_level
.
front
(),
in_level
.
begin
()
+
above_level
.
back
()
+
1
);
}
PADDLE_ENFORCE_LE
(
new_lod
.
size
(),
in
.
size
());
return
new_lod
;
for
(
size_t
lvl
=
0
;
lvl
<
res
.
size
();
lvl
++
)
{
// to make the first offset equals 0, all the elements minus the first
// element
size_t
front
=
res
[
lvl
].
front
();
for
(
auto
&
ele
:
res
[
lvl
])
{
ele
-=
front
;
}
}
return
res
;
}
LoD
ToAbsOffset
(
const
LoD
&
in
)
{
// the lowest level stores relative offsets
if
(
in
.
empty
()
||
in
.
size
()
==
1
)
return
in
;
LoD
result
=
in
;
for
(
int
level
=
result
.
size
()
-
2
;
level
>=
0
;
level
--
)
{
for
(
auto
&
ele
:
result
[
level
])
{
ele
=
result
[
level
+
1
][
ele
];
}
}
return
result
;
}
bool
operator
==
(
const
LoD
&
a
,
const
LoD
&
b
)
{
...
...
@@ -75,17 +94,7 @@ bool operator==(const LoD& a, const LoD& b) {
size_t
LoDTensor
::
NumElements
(
size_t
level
,
size_t
idx
)
const
{
PADDLE_ENFORCE_LT
(
level
,
NumLevels
());
PADDLE_ENFORCE_LT
(
idx
,
NumElements
(
level
));
// the last level of LoD, just return number of records in Tensor
if
(
level
==
NumLevels
()
-
1
)
{
return
lod_
[
level
][
idx
+
1
]
-
lod_
[
level
][
idx
];
}
// high level of LoD, and there is another lower level, return number of
// lower-level elements
auto
tmp
=
SliceInLevel
(
lod_
,
level
,
idx
,
idx
+
1
);
PADDLE_ENFORCE_GE
(
tmp
.
size
(),
2
);
// there is a 0 as a placeholder stored in LoD, so the number of elements
// equals lod.size() - 1
return
tmp
[
1
].
size
()
-
1
;
return
lod_
[
level
][
idx
+
1
]
-
lod_
[
level
][
idx
];
}
void
LoDTensor
::
ShrinkLevels
(
size_t
level_begin
,
size_t
level_end
)
{
...
...
paddle/framework/lod_tensor.h
浏览文件 @
db7b1171
...
...
@@ -39,23 +39,36 @@ using Vector = thrust::host_vector<
#endif
/*
*
3-level LoD stores
*
LoD is short for Level of Details.
*
* 0 10 20
* 0 5 10 15 20
* 0 2 5 7 10 12 15 20
*
* - in a level, each element indicates offset in the underlying Tensor
* - in a level, each element indicates relative offset of the lower level
* - the first element should be 0 and that indicates that this sequence start
* from 0
* - each sequence's begin and end(no-inclusive) is level[id, id+1]
*
* For example:
* 3-level LoD stores
*
* 0 2 3
* 0 2 4 7
* 0 2 5 7 10 12 15 20
*/
using
LoD
=
std
::
vector
<
Vector
<
size_t
>>
;
/*
* Slice levels from a LoD.
* NOTE the lowest level should always be the absolute offsets of the underlying
* tensor instances. So if higher layers are sliced without the lowest level,
* the lower level of the sliced LoD will be transformed to the absolute offset.
*/
LoD
SliceLevels
(
const
LoD
&
in
,
size_t
level_begin
,
size_t
level_end
);
LoD
SliceInLevel
(
const
LoD
&
in
,
size_t
level
,
size_t
elem_begin
,
size_t
elem_end
);
/*
* Transform an LoD from relative offsets to absolute offsets.
*/
LoD
ToAbsOffset
(
const
LoD
&
in
);
bool
operator
==
(
const
LoD
&
a
,
const
LoD
&
b
);
...
...
paddle/framework/lod_tensor_test.cc
浏览文件 @
db7b1171
...
...
@@ -30,8 +30,8 @@ class LoDTensorTester : public ::testing::Test {
// 0 5 10 15 20
// 0 2 5 7 10 12 15 20
LoD
lod
;
lod
.
push_back
(
std
::
vector
<
size_t
>
{
0
,
10
,
20
});
lod
.
push_back
(
std
::
vector
<
size_t
>
{
0
,
5
,
10
,
15
,
20
});
lod
.
push_back
(
std
::
vector
<
size_t
>
{
0
,
2
,
3
});
lod
.
push_back
(
std
::
vector
<
size_t
>
{
0
,
2
,
5
,
8
});
lod
.
push_back
(
std
::
vector
<
size_t
>
{
0
,
2
,
5
,
7
,
10
,
12
,
15
,
17
,
20
});
ASSERT_EQ
(
lod
.
size
(),
3UL
);
...
...
@@ -52,14 +52,14 @@ TEST_F(LoDTensorTester, NumLevels) { ASSERT_EQ(lod_tensor_.NumLevels(), 3UL); }
TEST_F
(
LoDTensorTester
,
NumElements
)
{
ASSERT_EQ
(
lod_tensor_
.
NumElements
(
0
),
2UL
);
ASSERT_EQ
(
lod_tensor_
.
NumElements
(
1
),
4
UL
);
ASSERT_EQ
(
lod_tensor_
.
NumElements
(
1
),
3
UL
);
ASSERT_EQ
(
lod_tensor_
.
NumElements
(
2
),
8UL
);
}
TEST_F
(
LoDTensorTester
,
NumElements2
)
{
ASSERT_EQ
(
lod_tensor_
.
NumElements
(
0
,
0
),
2UL
);
ASSERT_EQ
(
lod_tensor_
.
NumElements
(
0
,
1
),
2
UL
);
ASSERT_EQ
(
lod_tensor_
.
NumElements
(
1
,
1
),
2
UL
);
ASSERT_EQ
(
lod_tensor_
.
NumElements
(
0
,
1
),
1
UL
);
ASSERT_EQ
(
lod_tensor_
.
NumElements
(
1
,
1
),
3
UL
);
}
TEST_F
(
LoDTensorTester
,
ShrinkLevels
)
{
...
...
@@ -68,17 +68,16 @@ TEST_F(LoDTensorTester, ShrinkLevels) {
LoDTensor
new_lod_tensor
=
lod_tensor_
;
new_lod_tensor
.
ShrinkLevels
(
level
,
level
+
1
);
ASSERT_EQ
(
new_lod_tensor
.
NumLevels
(),
1UL
);
ASSERT_EQ
(
new_lod_tensor
.
NumElements
(
0
),
lod_tensor_
.
NumElements
(
level
));
ASSERT_EQ
(
new_lod_tensor
.
data
<
float
>
(),
lod_tensor_
.
data
<
float
>
());
}
// shrink 2 level
for
(
size_t
level
=
0
;
level
<
2UL
;
++
level
)
{
LoDTensor
new_lod_tensor
=
lod_tensor_
;
new_lod_tensor
.
ShrinkLevels
(
level
,
level
+
2
);
// the lowest level's last element should be the tensor's batch_size.
ASSERT_EQ
(
new_lod_tensor
.
lod
().
back
().
back
(),
lod_tensor_
.
lod
().
back
().
back
());
ASSERT_EQ
(
new_lod_tensor
.
NumLevels
(),
2UL
);
ASSERT_EQ
(
new_lod_tensor
.
NumElements
(
0
),
lod_tensor_
.
NumElements
(
level
));
ASSERT_EQ
(
new_lod_tensor
.
NumElements
(
1
),
lod_tensor_
.
NumElements
(
level
+
1
));
ASSERT_EQ
(
new_lod_tensor
.
data
<
float
>
(),
lod_tensor_
.
data
<
float
>
());
}
}
...
...
@@ -86,19 +85,19 @@ TEST_F(LoDTensorTester, ShrinkLevels) {
TEST_F
(
LoDTensorTester
,
ShrinkInLevel
)
{
size_t
level
=
0
;
LoDTensor
new_lod_tensor
=
lod_tensor_
;
new_lod_tensor
.
ShrinkInLevel
(
level
,
0
,
2
);
new_lod_tensor
.
ShrinkInLevel
(
level
,
0
,
1
);
EXPECT_EQ
(
new_lod_tensor
.
NumLevels
(),
3UL
);
EXPECT_EQ
(
new_lod_tensor
.
NumElements
(
0
),
2
UL
);
EXPECT_EQ
(
new_lod_tensor
.
NumElements
(
1
),
4
UL
);
EXPECT_EQ
(
new_lod_tensor
.
NumElements
(
2
),
8
UL
);
EXPECT_EQ
(
new_lod_tensor
.
NumElements
(
0
),
1
UL
);
EXPECT_EQ
(
new_lod_tensor
.
NumElements
(
1
),
2
UL
);
EXPECT_EQ
(
new_lod_tensor
.
NumElements
(
2
),
5
UL
);
ASSERT_EQ
(
new_lod_tensor
.
data
<
float
>
(),
lod_tensor_
.
data
<
float
>
());
level
=
1
;
new_lod_tensor
=
lod_tensor_
;
new_lod_tensor
.
ShrinkInLevel
(
level
,
0
,
2
);
new_lod_tensor
.
ShrinkInLevel
(
level
,
1
,
2
);
ASSERT_EQ
(
new_lod_tensor
.
NumLevels
(),
2UL
);
ASSERT_EQ
(
new_lod_tensor
.
NumElements
(
0
),
2
UL
);
ASSERT_EQ
(
new_lod_tensor
.
NumElements
(
1
),
4
UL
);
ASSERT_EQ
(
new_lod_tensor
.
NumElements
(
0
),
1
UL
);
ASSERT_EQ
(
new_lod_tensor
.
NumElements
(
1
),
3
UL
);
ASSERT_EQ
(
new_lod_tensor
.
data
<
float
>
(),
lod_tensor_
.
data
<
float
>
());
}
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录