Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
s920243400
PaddleDetection
提交
e6778337
P
PaddleDetection
项目概览
s920243400
/
PaddleDetection
与 Fork 源项目一致
Fork自
PaddlePaddle / PaddleDetection
通知
2
Star
0
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
P
PaddleDetection
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
提交
e6778337
编写于
10月 21, 2018
作者:
Q
Qiao Longfei
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
code clean
上级
5c65eff6
变更
3
隐藏空白更改
内联
并排
Showing
3 changed file
with
16 addition
and
50 deletion
+16
-50
paddle/fluid/operators/reader/ctr_reader.cc
paddle/fluid/operators/reader/ctr_reader.cc
+1
-27
paddle/fluid/operators/reader/ctr_reader.h
paddle/fluid/operators/reader/ctr_reader.h
+0
-9
paddle/fluid/operators/reader/ctr_reader_test.cc
paddle/fluid/operators/reader/ctr_reader_test.cc
+15
-14
未找到文件。
paddle/fluid/operators/reader/ctr_reader.cc
浏览文件 @
e6778337
...
...
@@ -73,18 +73,6 @@ static inline void parse_line(
}
}
static
void
print_map
(
std
::
unordered_map
<
std
::
string
,
std
::
vector
<
int64_t
>>*
map
)
{
for
(
auto
it
=
map
->
begin
();
it
!=
map
->
end
();
++
it
)
{
std
::
cout
<<
it
->
first
<<
" -> "
;
std
::
cout
<<
"["
;
for
(
auto
&
i
:
it
->
second
)
{
std
::
cout
<<
i
<<
" "
;
}
std
::
cout
<<
"]
\n
"
;
}
}
class
Reader
{
public:
virtual
~
Reader
()
{}
...
...
@@ -162,10 +150,6 @@ void ReadThread(const std::vector<std::string>& file_list,
VLOG
(
3
)
<<
"reader inited"
;
uint64_t
t0
=
GetTimeInSec
();
int
i
=
0
;
while
(
reader
.
HasNext
())
{
batch_data
.
clear
();
batch_data
.
reserve
(
batch_size
);
...
...
@@ -186,7 +170,6 @@ void ReadThread(const std::vector<std::string>& file_list,
break
;
}
}
// print_map(&batch_data[0]);
std
::
vector
<
framework
::
LoDTensor
>
lod_datas
;
...
...
@@ -224,19 +207,10 @@ void ReadThread(const std::vector<std::string>& file_list,
queue
->
Push
(
lod_datas
);
VLOG
(
4
)
<<
"push one data, queue_size="
<<
queue
->
Size
();
if
(
i
!=
0
&&
i
%
100
==
0
)
{
uint64_t
t1
=
GetTimeInSec
();
float
line_per_s
=
100
*
batch_size
*
1000000
/
(
t1
-
t0
);
VLOG
(
3
)
<<
"["
<<
thread_id
<<
"]"
<<
" line_per_second = "
<<
line_per_s
;
t0
=
t1
;
}
i
++
;
}
(
*
thread_status
)[
thread_id
]
=
Stopped
;
VLOG
(
3
)
<<
"thread "
<<
thread_id
<<
" exited"
;
VLOG
(
3
)
<<
"
set status to stopped,
thread "
<<
thread_id
<<
" exited"
;
}
}
// namespace reader
...
...
paddle/fluid/operators/reader/ctr_reader.h
浏览文件 @
e6778337
...
...
@@ -39,15 +39,6 @@ void ReadThread(const std::vector<std::string>& file_list,
int
thread_id
,
std
::
vector
<
ReaderThreadStatus
>*
thread_status
,
std
::
shared_ptr
<
LoDTensorBlockingQueue
>
queue
);
inline
uint64_t
GetTimeInSec
()
{
using
clock
=
std
::
conditional
<
std
::
chrono
::
high_resolution_clock
::
is_steady
,
std
::
chrono
::
high_resolution_clock
,
std
::
chrono
::
steady_clock
>::
type
;
return
std
::
chrono
::
duration_cast
<
std
::
chrono
::
microseconds
>
(
clock
::
now
().
time_since_epoch
())
.
count
();
}
class
CTRReader
:
public
framework
::
FileReader
{
public:
explicit
CTRReader
(
const
std
::
shared_ptr
<
LoDTensorBlockingQueue
>&
queue
,
...
...
paddle/fluid/operators/reader/ctr_reader_test.cc
浏览文件 @
e6778337
...
...
@@ -33,6 +33,7 @@ using paddle::operators::reader::LoDTensorBlockingQueueHolder;
using
paddle
::
operators
::
reader
::
CTRReader
;
using
paddle
::
framework
::
LoDTensor
;
using
paddle
::
framework
::
LoD
;
using
paddle
::
framework
::
DDim
;
using
paddle
::
platform
::
CPUPlace
;
static
void
generatedata
(
const
std
::
vector
<
std
::
string
>&
data
,
...
...
@@ -73,17 +74,17 @@ TEST(CTR_READER, read_data) {
std
::
vector
<
int64_t
>
label_value
=
{
0
,
0
,
1
,
0
,
1
,
1
,
0
,
1
,
1
,
1
};
std
::
vector
<
std
::
tuple
<
LoD
,
std
::
vector
<
int64_t
>>>
data_slot_6002
{
{{{
0
,
1
,
2
}},
{
0
,
0
}},
{{{
0
,
5
,
6
}},
{
10
,
11
,
12
,
13
,
14
,
0
}},
{{{
0
,
1
,
2
}},
{
0
,
0
}},
{{{
0
,
1
,
2
}},
{
30
,
0
}},
{{{
0
,
1
,
2
}},
{
40
,
0
}}};
{{{
0
,
1
,
2
,
7
}},
{
0
,
0
,
10
,
11
,
12
,
13
,
14
}},
{{{
0
,
1
,
2
,
3
}},
{
0
,
0
,
0
}},
{{{
0
,
1
,
2
,
3
}},
{
30
,
0
,
40
}},
{{{
0
,
1
}},
{
0
}}};
std
::
vector
<
std
::
tuple
<
LoD
,
std
::
vector
<
int64_t
>>>
data_slot_6003
{
{{{
0
,
1
,
4
}},
{
1
,
5
,
6
,
7
}},
{{{
0
,
1
,
5
}},
{
0
,
15
,
16
,
17
,
18
}},
{{{
0
,
1
,
2
}},
{
0
,
0
}},
{{{
0
,
1
,
3
}},
{
31
,
35
,
36
}},
{{{
0
,
1
,
4
}},
{
41
,
47
,
48
,
49
}}};
{{{
0
,
1
,
4
,
5
}},
{
1
,
5
,
6
,
7
,
0
}},
{{{
0
,
4
,
5
,
6
}},
{
15
,
16
,
17
,
18
,
0
,
0
}},
{{{
0
,
1
,
3
,
4
}},
{
31
,
35
,
36
,
41
}},
{{{
0
,
3
}},
{
47
,
48
,
49
}}};
std
::
vector
<
DDim
>
label_dims
=
{{
1
,
3
},
{
1
,
3
},
{
1
,
3
},
{
1
,
1
}};
LoDTensorBlockingQueueHolder
queue_holder
;
int
capacity
=
64
;
...
...
@@ -91,7 +92,7 @@ TEST(CTR_READER, read_data) {
std
::
shared_ptr
<
LoDTensorBlockingQueue
>
queue
=
queue_holder
.
GetQueue
();
int
batch_size
=
2
;
int
batch_size
=
3
;
int
thread_num
=
1
;
std
::
vector
<
std
::
string
>
slots
=
{
"6002"
,
"6003"
};
std
::
vector
<
std
::
string
>
file_list
;
...
...
@@ -103,15 +104,15 @@ TEST(CTR_READER, read_data) {
reader
.
Start
();
size_t
batch_num
=
std
::
ceil
(
ctr_data
.
size
()
/
batch_size
)
*
thread_num
;
size_t
batch_num
=
std
::
ceil
(
static_cast
<
float
>
(
ctr_data
.
size
())
/
batch_size
)
*
thread_num
;
for
(
size_t
i
=
0
;
i
<
batch_num
;
++
i
)
{
std
::
vector
<
LoDTensor
>
out
;
reader
.
ReadNext
(
&
out
);
ASSERT_EQ
(
out
.
size
(),
slots
.
size
()
+
1
);
auto
&
label_tensor
=
out
.
back
();
ASSERT_EQ
(
label_tensor
.
dims
(),
paddle
::
framework
::
make_ddim
({
1
,
batch_size
}));
ASSERT_EQ
(
label_tensor
.
dims
(),
label_dims
[
i
]);
for
(
size_t
j
=
0
;
j
<
batch_size
&&
i
*
batch_num
+
j
<
ctr_data
.
size
();
++
j
)
{
auto
&
label
=
label_tensor
.
data
<
int64_t
>
()[
j
];
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录