Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
PaddlePaddle
VisualDL
提交
5f1d9984
V
VisualDL
项目概览
PaddlePaddle
/
VisualDL
大约 2 年 前同步成功
通知
89
Star
4655
Fork
642
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
10
列表
看板
标记
里程碑
合并请求
2
Wiki
5
Wiki
分析
仓库
DevOps
项目成员
Pages
V
VisualDL
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
10
Issue
10
列表
看板
标记
里程碑
合并请求
2
合并请求
2
Pages
分析
分析
仓库分析
DevOps
Wiki
5
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
未验证
提交
5f1d9984
编写于
1月 13, 2018
作者:
Q
Qiao Longfei
提交者:
GitHub
1月 13, 2018
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
init comment for sdk (#117)
* init comment for sdk * update comment
上级
dad31052
变更
3
显示空白变更内容
内联
并排
Showing
3 changed file
with
83 addition
and
20 deletion
+83
-20
visualdl/logic/sdk.h
visualdl/logic/sdk.h
+22
-3
visualdl/python/storage.py
visualdl/python/storage.py
+59
-13
visualdl/python/test_storage.py
visualdl/python/test_storage.py
+2
-4
未找到文件。
visualdl/logic/sdk.h
浏览文件 @
5f1d9984
...
@@ -10,6 +10,10 @@ namespace visualdl {
...
@@ -10,6 +10,10 @@ namespace visualdl {
const
static
std
::
string
kDefaultMode
{
"default"
};
const
static
std
::
string
kDefaultMode
{
"default"
};
/**
* LogWriter is common Data Structure used to write data
* into a low level storage data structure.
*/
class
LogWriter
{
class
LogWriter
{
public:
public:
LogWriter
(
const
std
::
string
&
dir
,
int
sync_cycle
)
{
LogWriter
(
const
std
::
string
&
dir
,
int
sync_cycle
)
{
...
@@ -34,6 +38,9 @@ public:
...
@@ -34,6 +38,9 @@ public:
return
writer
;
return
writer
;
}
}
/**
* create a new tablet
*/
Tablet
AddTablet
(
const
std
::
string
&
tag
)
{
Tablet
AddTablet
(
const
std
::
string
&
tag
)
{
// TODO(ChunweiYan) add string check here.
// TODO(ChunweiYan) add string check here.
auto
tmp
=
mode_
+
"/"
+
tag
;
auto
tmp
=
mode_
+
"/"
+
tag
;
...
@@ -51,6 +58,10 @@ private:
...
@@ -51,6 +58,10 @@ private:
std
::
string
mode_
{
kDefaultMode
};
std
::
string
mode_
{
kDefaultMode
};
};
};
/**
* LogReader is common Data Structure used to read data
* from a low level storage data structure.
*/
class
LogReader
{
class
LogReader
{
public:
public:
LogReader
(
const
std
::
string
&
dir
)
:
reader_
(
dir
)
{}
LogReader
(
const
std
::
string
&
dir
)
:
reader_
(
dir
)
{}
...
@@ -180,19 +191,21 @@ struct Image {
...
@@ -180,19 +191,21 @@ struct Image {
writer_
.
SetNumSamples
(
num_samples
);
writer_
.
SetNumSamples
(
num_samples
);
SetCaption
(
tablet
.
reader
().
tag
());
SetCaption
(
tablet
.
reader
().
tag
());
}
}
void
SetCaption
(
const
std
::
string
&
c
)
{
void
SetCaption
(
const
std
::
string
&
c
)
{
writer_
.
SetCaptions
(
std
::
vector
<
std
::
string
>
({
c
}));
writer_
.
SetCaptions
(
std
::
vector
<
std
::
string
>
({
c
}));
}
}
/*
/*
* Start a sampl
e
period.
* Start a sampl
ing
period.
*/
*/
void
StartSampling
();
void
StartSampling
();
/*
/*
* Will this sample
will
be taken.
* Will this sample be taken.
*/
*/
int
IsSampleTaken
();
int
IsSampleTaken
();
/*
/*
* End a sampl
e
period.
* End a sampl
ing
period.
*/
*/
void
FinishSampling
();
void
FinishSampling
();
...
@@ -265,6 +278,9 @@ private:
...
@@ -265,6 +278,9 @@ private:
std
::
string
mode_
;
std
::
string
mode_
;
};
};
/*
* Histogram component writer.
*/
template
<
typename
T
>
template
<
typename
T
>
struct
Histogram
{
struct
Histogram
{
Histogram
(
Tablet
tablet
,
int
num_buckets
)
Histogram
(
Tablet
tablet
,
int
num_buckets
)
...
@@ -279,6 +295,9 @@ private:
...
@@ -279,6 +295,9 @@ private:
Tablet
writer_
;
Tablet
writer_
;
};
};
/*
* Histogram reader.
*/
template
<
typename
T
>
template
<
typename
T
>
struct
HistogramReader
{
struct
HistogramReader
{
HistogramReader
(
TabletReader
tablet
)
:
reader_
(
tablet
)
{}
HistogramReader
(
TabletReader
tablet
)
:
reader_
(
tablet
)
{}
...
...
visualdl/python/storage.py
浏览文件 @
5f1d9984
...
@@ -6,28 +6,60 @@ dtypes = ("float", "double", "int32", "int64")
...
@@ -6,28 +6,60 @@ dtypes = ("float", "double", "int32", "int64")
class
LogReader
(
object
):
class
LogReader
(
object
):
"""LogReader is a Python wrapper to read and analysis the data that
cur_mode
=
None
saved with data format defined in storage.proto. user can get
Scalar Reader/Image Reader/Histogram Reader from this module and use
them to reade the data you need.
"""
def
__init__
(
self
,
dir
,
reader
=
None
):
def
__init__
(
self
,
dir
,
reader
=
None
):
"""
create a LogReader
:param dir: the dir where log file is.
:param reader: create a new LogReader with a formal one
"""
self
.
dir
=
dir
self
.
dir
=
dir
self
.
reader
=
reader
if
reader
else
core
.
LogReader
(
dir
)
self
.
reader
=
reader
if
reader
else
core
.
LogReader
(
dir
)
def
mode
(
self
,
mode
):
def
mode
(
self
,
mode
):
"""
Set the current mode of reader.
:param mode: the mode is something like a scope, it's used to
put some related data together. for example: train or test.
data generated during training can be marked mode train, and data
generated during testing can be marked test.
:return: the reader itself
"""
self
.
reader
.
set_mode
(
mode
)
self
.
reader
.
set_mode
(
mode
)
return
self
return
self
def
as_mode
(
self
,
mode
):
def
as_mode
(
self
,
mode
):
"""
create a new LogReader with mode and return it to user.
"""
tmp
=
LogReader
(
dir
,
self
.
reader
.
as_mode
(
mode
))
tmp
=
LogReader
(
dir
,
self
.
reader
.
as_mode
(
mode
))
return
tmp
return
tmp
def
modes
(
self
):
def
modes
(
self
):
"""
Get all modes of the log file
:return:
"""
return
self
.
reader
.
modes
()
return
self
.
reader
.
modes
()
def
tags
(
self
,
kind
):
def
tags
(
self
,
component
):
return
self
.
reader
.
tags
(
kind
)
"""
Get all tags from the current log file for one kind of component
:param component: Scalar|Histogram|Images
:return: all the tags
"""
return
self
.
reader
.
tags
(
component
)
def
scalar
(
self
,
tag
,
type
=
'float'
):
def
scalar
(
self
,
tag
,
type
=
'float'
):
"""
Get a scalar reader with tag and data type
"""
type2scalar
=
{
type2scalar
=
{
'float'
:
self
.
reader
.
get_scalar_float
,
'float'
:
self
.
reader
.
get_scalar_float
,
'double'
:
self
.
reader
.
get_scalar_double
,
'double'
:
self
.
reader
.
get_scalar_double
,
...
@@ -36,9 +68,15 @@ class LogReader(object):
...
@@ -36,9 +68,15 @@ class LogReader(object):
return
type2scalar
[
type
](
tag
)
return
type2scalar
[
type
](
tag
)
def
image
(
self
,
tag
):
def
image
(
self
,
tag
):
"""
Get a image reader with tag
"""
return
self
.
reader
.
get_image
(
tag
)
return
self
.
reader
.
get_image
(
tag
)
def
histogram
(
self
,
tag
,
type
=
'float'
):
def
histogram
(
self
,
tag
,
type
=
'float'
):
"""
Get a histogram reader with tag and data type
"""
type2scalar
=
{
type2scalar
=
{
'float'
:
self
.
reader
.
get_histogram_float
,
'float'
:
self
.
reader
.
get_histogram_float
,
'double'
:
self
.
reader
.
get_histogram_double
,
'double'
:
self
.
reader
.
get_histogram_double
,
...
@@ -54,6 +92,10 @@ class LogReader(object):
...
@@ -54,6 +92,10 @@ class LogReader(object):
class
LogWriter
(
object
):
class
LogWriter
(
object
):
"""LogWriter is a Python wrapper to write data to log file with the data
format defined in storage.proto. user can get Scalar Reader/Image Reader/
Histogram Reader from this module and use them to write the data to log file.
"""
cur_mode
=
None
cur_mode
=
None
...
@@ -67,13 +109,16 @@ class LogWriter(object):
...
@@ -67,13 +109,16 @@ class LogWriter(object):
return
self
return
self
def
as_mode
(
self
,
mode
):
def
as_mode
(
self
,
mode
):
"""
create a new LogWriter with mode and return it.
"""
LogWriter
.
cur_mode
=
LogWriter
(
self
.
dir
,
self
.
sync_cycle
,
self
.
writer
.
as_mode
(
mode
))
LogWriter
.
cur_mode
=
LogWriter
(
self
.
dir
,
self
.
sync_cycle
,
self
.
writer
.
as_mode
(
mode
))
return
LogWriter
.
cur_mode
return
LogWriter
.
cur_mode
def
scalar
(
self
,
tag
,
type
=
'float'
):
def
scalar
(
self
,
tag
,
type
=
'float'
):
'''
"""
Create a scalar
component
.
Create a scalar
writer with tag and type to write scalar data
.
'''
"""
type2scalar
=
{
type2scalar
=
{
'float'
:
self
.
writer
.
new_scalar_float
,
'float'
:
self
.
writer
.
new_scalar_float
,
'double'
:
self
.
writer
.
new_scalar_double
,
'double'
:
self
.
writer
.
new_scalar_double
,
...
@@ -82,15 +127,16 @@ class LogWriter(object):
...
@@ -82,15 +127,16 @@ class LogWriter(object):
return
type2scalar
[
type
](
tag
)
return
type2scalar
[
type
](
tag
)
def
image
(
self
,
tag
,
num_samples
,
step_cycle
):
def
image
(
self
,
tag
,
num_samples
,
step_cycle
):
'''
"""
Create an image
component
.
Create an image
writer that used to write image data
.
'''
"""
return
self
.
writer
.
new_image
(
tag
,
num_samples
,
step_cycle
)
return
self
.
writer
.
new_image
(
tag
,
num_samples
,
step_cycle
)
def
histogram
(
self
,
tag
,
num_buckets
,
type
=
'float'
):
def
histogram
(
self
,
tag
,
num_buckets
,
type
=
'float'
):
'''
"""
Create a histogram component.
Create a histogram writer that used to write
'''
histogram related data.
"""
types
=
{
types
=
{
'float'
:
self
.
writer
.
new_histogram_float
,
'float'
:
self
.
writer
.
new_histogram_float
,
'double'
:
self
.
writer
.
new_histogram_double
,
'double'
:
self
.
writer
.
new_histogram_double
,
...
...
visualdl/python/test_storage.py
浏览文件 @
5f1d9984
import
random
import
pprint
import
time
import
sys
import
unittest
import
unittest
import
numpy
as
np
import
numpy
as
np
from
PIL
import
Image
from
PIL
import
Image
import
sys
,
pprint
pprint
.
pprint
(
sys
.
path
)
pprint
.
pprint
(
sys
.
path
)
from
visualdl
import
LogWriter
,
LogReader
from
visualdl
import
LogWriter
,
LogReader
...
@@ -119,7 +118,6 @@ class StorageTest(unittest.TestCase):
...
@@ -119,7 +118,6 @@ class StorageTest(unittest.TestCase):
self
.
assertEqual
(
scalar
.
caption
(),
"train"
)
self
.
assertEqual
(
scalar
.
caption
(),
"train"
)
def
test_modes
(
self
):
def
test_modes
(
self
):
dir
=
"./tmp/storagetest0"
store
=
LogWriter
(
store
=
LogWriter
(
self
.
dir
,
sync_cycle
=
1
)
self
.
dir
,
sync_cycle
=
1
)
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录