Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
PaddlePaddle
VisualDL
提交
5f1d9984
V
VisualDL
项目概览
PaddlePaddle
/
VisualDL
大约 1 年 前同步成功
通知
88
Star
4655
Fork
642
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
10
列表
看板
标记
里程碑
合并请求
2
Wiki
5
Wiki
分析
仓库
DevOps
项目成员
Pages
V
VisualDL
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
10
Issue
10
列表
看板
标记
里程碑
合并请求
2
合并请求
2
Pages
分析
分析
仓库分析
DevOps
Wiki
5
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
未验证
提交
5f1d9984
编写于
1月 13, 2018
作者:
Q
Qiao Longfei
提交者:
GitHub
1月 13, 2018
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
init comment for sdk (#117)
* init comment for sdk * update comment
上级
dad31052
变更
3
隐藏空白更改
内联
并排
Showing
3 changed file
with
83 addition
and
20 deletion
+83
-20
visualdl/logic/sdk.h
visualdl/logic/sdk.h
+22
-3
visualdl/python/storage.py
visualdl/python/storage.py
+59
-13
visualdl/python/test_storage.py
visualdl/python/test_storage.py
+2
-4
未找到文件。
visualdl/logic/sdk.h
浏览文件 @
5f1d9984
...
...
@@ -10,6 +10,10 @@ namespace visualdl {
const
static
std
::
string
kDefaultMode
{
"default"
};
/**
* LogWriter is common Data Structure used to write data
* into a low level storage data structure.
*/
class
LogWriter
{
public:
LogWriter
(
const
std
::
string
&
dir
,
int
sync_cycle
)
{
...
...
@@ -34,6 +38,9 @@ public:
return
writer
;
}
/**
* create a new tablet
*/
Tablet
AddTablet
(
const
std
::
string
&
tag
)
{
// TODO(ChunweiYan) add string check here.
auto
tmp
=
mode_
+
"/"
+
tag
;
...
...
@@ -51,6 +58,10 @@ private:
std
::
string
mode_
{
kDefaultMode
};
};
/**
* LogReader is common Data Structure used to read data
* from a low level storage data structure.
*/
class
LogReader
{
public:
LogReader
(
const
std
::
string
&
dir
)
:
reader_
(
dir
)
{}
...
...
@@ -180,19 +191,21 @@ struct Image {
writer_
.
SetNumSamples
(
num_samples
);
SetCaption
(
tablet
.
reader
().
tag
());
}
void
SetCaption
(
const
std
::
string
&
c
)
{
writer_
.
SetCaptions
(
std
::
vector
<
std
::
string
>
({
c
}));
}
/*
* Start a sampl
e
period.
* Start a sampl
ing
period.
*/
void
StartSampling
();
/*
* Will this sample
will
be taken.
* Will this sample be taken.
*/
int
IsSampleTaken
();
/*
* End a sampl
e
period.
* End a sampl
ing
period.
*/
void
FinishSampling
();
...
...
@@ -265,6 +278,9 @@ private:
std
::
string
mode_
;
};
/*
* Histogram component writer.
*/
template
<
typename
T
>
struct
Histogram
{
Histogram
(
Tablet
tablet
,
int
num_buckets
)
...
...
@@ -279,6 +295,9 @@ private:
Tablet
writer_
;
};
/*
* Histogram reader.
*/
template
<
typename
T
>
struct
HistogramReader
{
HistogramReader
(
TabletReader
tablet
)
:
reader_
(
tablet
)
{}
...
...
visualdl/python/storage.py
浏览文件 @
5f1d9984
...
...
@@ -6,28 +6,60 @@ dtypes = ("float", "double", "int32", "int64")
class
LogReader
(
object
):
cur_mode
=
None
"""LogReader is a Python wrapper to read and analysis the data that
saved with data format defined in storage.proto. user can get
Scalar Reader/Image Reader/Histogram Reader from this module and use
them to reade the data you need.
"""
def
__init__
(
self
,
dir
,
reader
=
None
):
"""
create a LogReader
:param dir: the dir where log file is.
:param reader: create a new LogReader with a formal one
"""
self
.
dir
=
dir
self
.
reader
=
reader
if
reader
else
core
.
LogReader
(
dir
)
def
mode
(
self
,
mode
):
"""
Set the current mode of reader.
:param mode: the mode is something like a scope, it's used to
put some related data together. for example: train or test.
data generated during training can be marked mode train, and data
generated during testing can be marked test.
:return: the reader itself
"""
self
.
reader
.
set_mode
(
mode
)
return
self
def
as_mode
(
self
,
mode
):
"""
create a new LogReader with mode and return it to user.
"""
tmp
=
LogReader
(
dir
,
self
.
reader
.
as_mode
(
mode
))
return
tmp
def
modes
(
self
):
"""
Get all modes of the log file
:return:
"""
return
self
.
reader
.
modes
()
def
tags
(
self
,
kind
):
return
self
.
reader
.
tags
(
kind
)
def
tags
(
self
,
component
):
"""
Get all tags from the current log file for one kind of component
:param component: Scalar|Histogram|Images
:return: all the tags
"""
return
self
.
reader
.
tags
(
component
)
def
scalar
(
self
,
tag
,
type
=
'float'
):
"""
Get a scalar reader with tag and data type
"""
type2scalar
=
{
'float'
:
self
.
reader
.
get_scalar_float
,
'double'
:
self
.
reader
.
get_scalar_double
,
...
...
@@ -36,9 +68,15 @@ class LogReader(object):
return
type2scalar
[
type
](
tag
)
def
image
(
self
,
tag
):
"""
Get a image reader with tag
"""
return
self
.
reader
.
get_image
(
tag
)
def
histogram
(
self
,
tag
,
type
=
'float'
):
"""
Get a histogram reader with tag and data type
"""
type2scalar
=
{
'float'
:
self
.
reader
.
get_histogram_float
,
'double'
:
self
.
reader
.
get_histogram_double
,
...
...
@@ -54,6 +92,10 @@ class LogReader(object):
class
LogWriter
(
object
):
"""LogWriter is a Python wrapper to write data to log file with the data
format defined in storage.proto. user can get Scalar Reader/Image Reader/
Histogram Reader from this module and use them to write the data to log file.
"""
cur_mode
=
None
...
...
@@ -67,13 +109,16 @@ class LogWriter(object):
return
self
def
as_mode
(
self
,
mode
):
"""
create a new LogWriter with mode and return it.
"""
LogWriter
.
cur_mode
=
LogWriter
(
self
.
dir
,
self
.
sync_cycle
,
self
.
writer
.
as_mode
(
mode
))
return
LogWriter
.
cur_mode
def
scalar
(
self
,
tag
,
type
=
'float'
):
'''
Create a scalar
component
.
'''
"""
Create a scalar
writer with tag and type to write scalar data
.
"""
type2scalar
=
{
'float'
:
self
.
writer
.
new_scalar_float
,
'double'
:
self
.
writer
.
new_scalar_double
,
...
...
@@ -82,15 +127,16 @@ class LogWriter(object):
return
type2scalar
[
type
](
tag
)
def
image
(
self
,
tag
,
num_samples
,
step_cycle
):
'''
Create an image
component
.
'''
"""
Create an image
writer that used to write image data
.
"""
return
self
.
writer
.
new_image
(
tag
,
num_samples
,
step_cycle
)
def
histogram
(
self
,
tag
,
num_buckets
,
type
=
'float'
):
'''
Create a histogram component.
'''
"""
Create a histogram writer that used to write
histogram related data.
"""
types
=
{
'float'
:
self
.
writer
.
new_histogram_float
,
'double'
:
self
.
writer
.
new_histogram_double
,
...
...
visualdl/python/test_storage.py
浏览文件 @
5f1d9984
import
random
import
time
import
pprint
import
sys
import
unittest
import
numpy
as
np
from
PIL
import
Image
import
sys
,
pprint
pprint
.
pprint
(
sys
.
path
)
from
visualdl
import
LogWriter
,
LogReader
...
...
@@ -119,7 +118,6 @@ class StorageTest(unittest.TestCase):
self
.
assertEqual
(
scalar
.
caption
(),
"train"
)
def
test_modes
(
self
):
dir
=
"./tmp/storagetest0"
store
=
LogWriter
(
self
.
dir
,
sync_cycle
=
1
)
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录