Skip to content

V
VisualDL

PaddlePaddle / VisualDL
大约 2 年 前同步成功

通知 89
Star 4655
Fork 642
V
VisualDL
“52e8ebf246faa737a9553e1fe7df531a10234074”上不存在“develop/doc/design/file_manager/README.html”
未验证 提交 f7af2157 编写于 作者: C chenjian 提交者： GitHub
浏览文件
操作

Add scalars component (#1081) 

* backend of add_scalars

* fix function name bug of get_scalars_tags

* @zh794390558 and @ShenYuhan commented 0508

* fix a bug

* adapt to frontend runs need

* fix proto

* fix a bug

* add scalars test demo

* fix

* adapt scalars to scalar

* remove unused code

* fix demo code

* fix proto version

* fix proto version

* upgrade protoc to v3.19.0

* fix a bug in logreader
Co-authored-by: N伽利略 <jialiasus2@163.com>
Co-authored-by: Nwuzewu <wuzewu@baidu.com>
上级 59c86e52
显示空白变更内容
内联 并排
Showing with 344 addition and 774 deletion
demo/components/scalars_test.py 0 → 100644
浏览文件 @ f7af2157
# Copyright (c) 2022 VisualDL Authors. All Rights Reserve.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# =======================================================================
# coding=utf-8
import math
from visualdl import LogWriter
if __name__ == '__main__':
value = [i / 1000.0 for i in range(1000)]
with LogWriter(logdir="./log/scalars_test/") as writer:
for step in range(1000):
writer.add_scalars(
main_tag='math/formula1',
tag_scalar_dict={
'sinx': math.sin(value[step]),
'cosx': math.cos(value[step])
},
step=step)
writer.add_scalars(
main_tag='math/formula2',
tag_scalar_dict={
'sqrtx': math.sqrt(value[step]),
'squarex': value[step]**2
},
step=step)
with LogWriter(logdir="./log/scalar_test1/") as writer:
for step in range(1000):
writer.add_scalar(
tag='math/formula2', value=value[step], step=step)
visualdl/component/base_component.py
浏览文件 @ f7af2157
......@@ -18,6 +18,29 @@ from PIL import Image
from visualdl.proto.record_pb2 import Record
def scalars(main_tag, tag_scalar_dict, step, walltime=None):
"""Package data to scalars
Args:
main_tag (string): Data identifier
tag_scalar_dict (dict): A dict to provide multi-values with tags
step (int): Step of scalar
walltime (int): Wall time of scalar
Return:
Package with format of record_pb2.Record
"""
for sub_tag, value in tag_scalar_dict.items():
value = float(value)
yield Record(values=[
Record.Value(
id=step,
tag=main_tag,
timestamp=walltime,
tag_value=Record.TagValue(tag=sub_tag, value=value))
])
def scalar(tag, value, step, walltime=None):
"""Package data to one scalar.
Args:
......
visualdl/proto/record.proto
浏览文件 @ f7af2157
......@@ -75,6 +75,11 @@ message Record {
string display_name = 1;
}
message TagValue {
string tag = 1;
float value = 2;
}
message Value {
int64 id = 1;
string tag = 2;
......@@ -90,6 +95,7 @@ message Record {
ROC_Curve roc_curve = 11;
Text text = 12;
HParam hparam = 13;
TagValue tag_value = 14;
}
}
......
visualdl/proto/record_pb2.py
浏览文件 @ f7af2157
# -*- coding: utf-8 -*-
# Generated by the protocol buffer compiler. DO NOT EDIT!
# source: record.proto
"""Generated protocol buffer code."""
from google.protobuf import descriptor as _descriptor
from google.protobuf import descriptor_pool as _descriptor_pool
from google.protobuf import message as _message
from google.protobuf import reflection as _reflection
from google.protobuf import symbol_database as _symbol_database
......@@ -13,756 +14,25 @@ _sym_db = _symbol_database.Default()
DESCRIPTOR = _descriptor.FileDescriptor(
name='record.proto',
package='visualdl',
syntax='proto3',
serialized_options=None,
serialized_pb=b'\n\x0crecord.proto\x12\x08visualdl\"\xca\x0b\n\x06Record\x12&\n\x06values\x18\x01 \x03(\x0b\x32\x16.visualdl.Record.Value\x1a%\n\x05Image\x12\x1c\n\x14\x65ncoded_image_string\x18\x04 \x01(\x0c\x1a#\n\x04Text\x12\x1b\n\x13\x65ncoded_text_string\x18\x01 \x01(\t\x1a}\n\x05\x41udio\x12\x13\n\x0bsample_rate\x18\x01 \x01(\x02\x12\x14\n\x0cnum_channels\x18\x02 \x01(\x03\x12\x15\n\rlength_frames\x18\x03 \x01(\x03\x12\x1c\n\x14\x65ncoded_audio_string\x18\x04 \x01(\x0c\x12\x14\n\x0c\x63ontent_type\x18\x05 \x01(\t\x1a+\n\tEmbedding\x12\r\n\x05label\x18\x01 \x03(\t\x12\x0f\n\x07vectors\x18\x02 \x03(\x02\x1aP\n\nEmbeddings\x12.\n\nembeddings\x18\x01 \x03(\x0b\x32\x1a.visualdl.Record.Embedding\x12\x12\n\nlabel_meta\x18\x02 \x03(\t\x1a\x43\n\x10\x62ytes_embeddings\x12\x16\n\x0e\x65ncoded_labels\x18\x01 \x01(\x0c\x12\x17\n\x0f\x65ncoded_vectors\x18\x02 \x01(\x0c\x1a\x34\n\tHistogram\x12\x10\n\x04hist\x18\x01 \x03(\x01\x42\x02\x10\x01\x12\x15\n\tbin_edges\x18\x02 \x03(\x01\x42\x02\x10\x01\x1al\n\x07PRCurve\x12\x0e\n\x02TP\x18\x01 \x03(\x03\x42\x02\x10\x01\x12\x0e\n\x02\x46P\x18\x02 \x03(\x03\x42\x02\x10\x01\x12\x0e\n\x02TN\x18\x03 \x03(\x03\x42\x02\x10\x01\x12\x0e\n\x02\x46N\x18\x04 \x03(\x03\x42\x02\x10\x01\x12\x11\n\tprecision\x18\x05 \x03(\x01\x12\x0e\n\x06recall\x18\x06 \x03(\x01\x1a\x65\n\tROC_Curve\x12\x0e\n\x02TP\x18\x01 \x03(\x03\x42\x02\x10\x01\x12\x0e\n\x02\x46P\x18\x02 \x03(\x03\x42\x02\x10\x01\x12\x0e\n\x02TN\x18\x03 \x03(\x03\x42\x02\x10\x01\x12\x0e\n\x02\x46N\x18\x04 \x03(\x03\x42\x02\x10\x01\x12\x0b\n\x03tpr\x18\x05 \x03(\x01\x12\x0b\n\x03\x66pr\x18\x06 \x03(\x01\x1a\xf0\x01\n\x06HParam\x12\x37\n\x0bhparamInfos\x18\x01 \x03(\x0b\x32\".visualdl.Record.HParam.HparamInfo\x12\x37\n\x0bmetricInfos\x18\x02 \x03(\x0b\x32\".visualdl.Record.HParam.HparamInfo\x12\x0c\n\x04name\x18\x03 \x01(\t\x1a\x66\n\nHparamInfo\x12\x13\n\tint_value\x18\x01 \x01(\x03H\x00\x12\x15\n\x0b\x66loat_value\x18\x02 \x01(\x01H\x00\x12\x16\n\x0cstring_value\x18\x03 \x01(\tH\x00\x12\x0c\n\x04name\x18\x04 \x01(\tB\x06\n\x04type\x1a \n\x08MetaData\x12\x14\n\x0c\x64isplay_name\x18\x01 \x01(\t\x1a\xe8\x03\n\x05Value\x12\n\n\x02id\x18\x01 \x01(\x03\x12\x0b\n\x03tag\x18\x02 \x01(\t\x12\x11\n\ttimestamp\x18\x03 \x01(\x03\x12\x0f\n\x05value\x18\x04 \x01(\x02H\x00\x12\'\n\x05image\x18\x05 \x01(\x0b\x32\x16.visualdl.Record.ImageH\x00\x12\'\n\x05\x61udio\x18\x06 \x01(\x0b\x32\x16.visualdl.Record.AudioH\x00\x12\x31\n\nembeddings\x18\x07 \x01(\x0b\x32\x1b.visualdl.Record.EmbeddingsH\x00\x12/\n\thistogram\x18\x08 \x01(\x0b\x32\x1a.visualdl.Record.HistogramH\x00\x12,\n\x08pr_curve\x18\t \x01(\x0b\x32\x18.visualdl.Record.PRCurveH\x00\x12.\n\tmeta_data\x18\n \x01(\x0b\x32\x19.visualdl.Record.MetaDataH\x00\x12/\n\troc_curve\x18\x0b \x01(\x0b\x32\x1a.visualdl.Record.ROC_CurveH\x00\x12%\n\x04text\x18\x0c \x01(\x0b\x32\x15.visualdl.Record.TextH\x00\x12)\n\x06hparam\x18\r \x01(\x0b\x32\x17.visualdl.Record.HParamH\x00\x42\x0b\n\tone_valueb\x06proto3'
)
_RECORD_IMAGE = _descriptor.Descriptor(
name='Image',
full_name='visualdl.Record.Image',
filename=None,
file=DESCRIPTOR,
containing_type=None,
fields=[
_descriptor.FieldDescriptor(
name='encoded_image_string', full_name='visualdl.Record.Image.encoded_image_string', index=0,
number=4, type=12, cpp_type=9, label=1,
has_default_value=False, default_value=b"",
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
],
extensions=[
],
nested_types=[],
enum_types=[
],
serialized_options=None,
is_extendable=False,
syntax='proto3',
extension_ranges=[],
oneofs=[
],
serialized_start=77,
serialized_end=114,
)
_RECORD_TEXT = _descriptor.Descriptor(
name='Text',
full_name='visualdl.Record.Text',
filename=None,
file=DESCRIPTOR,
containing_type=None,
fields=[
_descriptor.FieldDescriptor(
name='encoded_text_string', full_name='visualdl.Record.Text.encoded_text_string', index=0,
number=1, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=b"".decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
],
extensions=[
],
nested_types=[],
enum_types=[
],
serialized_options=None,
is_extendable=False,
syntax='proto3',
extension_ranges=[],
oneofs=[
],
serialized_start=116,
serialized_end=151,
)
_RECORD_AUDIO = _descriptor.Descriptor(
name='Audio',
full_name='visualdl.Record.Audio',
filename=None,
file=DESCRIPTOR,
containing_type=None,
fields=[
_descriptor.FieldDescriptor(
name='sample_rate', full_name='visualdl.Record.Audio.sample_rate', index=0,
number=1, type=2, cpp_type=6, label=1,
has_default_value=False, default_value=float(0),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='num_channels', full_name='visualdl.Record.Audio.num_channels', index=1,
number=2, type=3, cpp_type=2, label=1,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='length_frames', full_name='visualdl.Record.Audio.length_frames', index=2,
number=3, type=3, cpp_type=2, label=1,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='encoded_audio_string', full_name='visualdl.Record.Audio.encoded_audio_string', index=3,
number=4, type=12, cpp_type=9, label=1,
has_default_value=False, default_value=b"",
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='content_type', full_name='visualdl.Record.Audio.content_type', index=4,
number=5, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=b"".decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
],
extensions=[
],
nested_types=[],
enum_types=[
],
serialized_options=None,
is_extendable=False,
syntax='proto3',
extension_ranges=[],
oneofs=[
],
serialized_start=153,
serialized_end=278,
)
_RECORD_EMBEDDING = _descriptor.Descriptor(
name='Embedding',
full_name='visualdl.Record.Embedding',
filename=None,
file=DESCRIPTOR,
containing_type=None,
fields=[
_descriptor.FieldDescriptor(
name='label', full_name='visualdl.Record.Embedding.label', index=0,
number=1, type=9, cpp_type=9, label=3,
has_default_value=False, default_value=[],
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='vectors', full_name='visualdl.Record.Embedding.vectors', index=1,
number=2, type=2, cpp_type=6, label=3,
has_default_value=False, default_value=[],
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
],
extensions=[
],
nested_types=[],
enum_types=[
],
serialized_options=None,
is_extendable=False,
syntax='proto3',
extension_ranges=[],
oneofs=[
],
serialized_start=280,
serialized_end=323,
)
_RECORD_EMBEDDINGS = _descriptor.Descriptor(
name='Embeddings',
full_name='visualdl.Record.Embeddings',
filename=None,
file=DESCRIPTOR,
containing_type=None,
fields=[
_descriptor.FieldDescriptor(
name='embeddings', full_name='visualdl.Record.Embeddings.embeddings', index=0,
number=1, type=11, cpp_type=10, label=3,
has_default_value=False, default_value=[],
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='label_meta', full_name='visualdl.Record.Embeddings.label_meta', index=1,
number=2, type=9, cpp_type=9, label=3,
has_default_value=False, default_value=[],
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
],
extensions=[
],
nested_types=[],
enum_types=[
],
serialized_options=None,
is_extendable=False,
syntax='proto3',
extension_ranges=[],
oneofs=[
],
serialized_start=325,
serialized_end=405,
)
_RECORD_BYTES_EMBEDDINGS = _descriptor.Descriptor(
name='bytes_embeddings',
full_name='visualdl.Record.bytes_embeddings',
filename=None,
file=DESCRIPTOR,
containing_type=None,
fields=[
_descriptor.FieldDescriptor(
name='encoded_labels', full_name='visualdl.Record.bytes_embeddings.encoded_labels', index=0,
number=1, type=12, cpp_type=9, label=1,
has_default_value=False, default_value=b"",
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='encoded_vectors', full_name='visualdl.Record.bytes_embeddings.encoded_vectors', index=1,
number=2, type=12, cpp_type=9, label=1,
has_default_value=False, default_value=b"",
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
],
extensions=[
],
nested_types=[],
enum_types=[
],
serialized_options=None,
is_extendable=False,
syntax='proto3',
extension_ranges=[],
oneofs=[
],
serialized_start=407,
serialized_end=474,
)
_RECORD_HISTOGRAM = _descriptor.Descriptor(
name='Histogram',
full_name='visualdl.Record.Histogram',
filename=None,
file=DESCRIPTOR,
containing_type=None,
fields=[
_descriptor.FieldDescriptor(
name='hist', full_name='visualdl.Record.Histogram.hist', index=0,
number=1, type=1, cpp_type=5, label=3,
has_default_value=False, default_value=[],
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=b'\020\001', file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='bin_edges', full_name='visualdl.Record.Histogram.bin_edges', index=1,
number=2, type=1, cpp_type=5, label=3,
has_default_value=False, default_value=[],
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=b'\020\001', file=DESCRIPTOR),
],
extensions=[
],
nested_types=[],
enum_types=[
],
serialized_options=None,
is_extendable=False,
syntax='proto3',
extension_ranges=[],
oneofs=[
],
serialized_start=476,
serialized_end=528,
)
DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0crecord.proto\x12\x08visualdl\"\xa2\x0c\n\x06Record\x12&\n\x06values\x18\x01 \x03(\x0b\x32\x16.visualdl.Record.Value\x1a%\n\x05Image\x12\x1c\n\x14\x65ncoded_image_string\x18\x04 \x01(\x0c\x1a#\n\x04Text\x12\x1b\n\x13\x65ncoded_text_string\x18\x01 \x01(\t\x1a}\n\x05\x41udio\x12\x13\n\x0bsample_rate\x18\x01 \x01(\x02\x12\x14\n\x0cnum_channels\x18\x02 \x01(\x03\x12\x15\n\rlength_frames\x18\x03 \x01(\x03\x12\x1c\n\x14\x65ncoded_audio_string\x18\x04 \x01(\x0c\x12\x14\n\x0c\x63ontent_type\x18\x05 \x01(\t\x1a+\n\tEmbedding\x12\r\n\x05label\x18\x01 \x03(\t\x12\x0f\n\x07vectors\x18\x02 \x03(\x02\x1aP\n\nEmbeddings\x12.\n\nembeddings\x18\x01 \x03(\x0b\x32\x1a.visualdl.Record.Embedding\x12\x12\n\nlabel_meta\x18\x02 \x03(\t\x1a\x43\n\x10\x62ytes_embeddings\x12\x16\n\x0e\x65ncoded_labels\x18\x01 \x01(\x0c\x12\x17\n\x0f\x65ncoded_vectors\x18\x02 \x01(\x0c\x1a\x34\n\tHistogram\x12\x10\n\x04hist\x18\x01 \x03(\x01\x42\x02\x10\x01\x12\x15\n\tbin_edges\x18\x02 \x03(\x01\x42\x02\x10\x01\x1al\n\x07PRCurve\x12\x0e\n\x02TP\x18\x01 \x03(\x03\x42\x02\x10\x01\x12\x0e\n\x02\x46P\x18\x02 \x03(\x03\x42\x02\x10\x01\x12\x0e\n\x02TN\x18\x03 \x03(\x03\x42\x02\x10\x01\x12\x0e\n\x02\x46N\x18\x04 \x03(\x03\x42\x02\x10\x01\x12\x11\n\tprecision\x18\x05 \x03(\x01\x12\x0e\n\x06recall\x18\x06 \x03(\x01\x1a\x65\n\tROC_Curve\x12\x0e\n\x02TP\x18\x01 \x03(\x03\x42\x02\x10\x01\x12\x0e\n\x02\x46P\x18\x02 \x03(\x03\x42\x02\x10\x01\x12\x0e\n\x02TN\x18\x03 \x03(\x03\x42\x02\x10\x01\x12\x0e\n\x02\x46N\x18\x04 \x03(\x03\x42\x02\x10\x01\x12\x0b\n\x03tpr\x18\x05 \x03(\x01\x12\x0b\n\x03\x66pr\x18\x06 \x03(\x01\x1a\xf0\x01\n\x06HParam\x12\x37\n\x0bhparamInfos\x18\x01 \x03(\x0b\x32\".visualdl.Record.HParam.HparamInfo\x12\x37\n\x0bmetricInfos\x18\x02 \x03(\x0b\x32\".visualdl.Record.HParam.HparamInfo\x12\x0c\n\x04name\x18\x03 \x01(\t\x1a\x66\n\nHparamInfo\x12\x13\n\tint_value\x18\x01 \x01(\x03H\x00\x12\x15\n\x0b\x66loat_value\x18\x02 \x01(\x01H\x00\x12\x16\n\x0cstring_value\x18\x03 \x01(\tH\x00\x12\x0c\n\x04name\x18\x04 \x01(\tB\x06\n\x04type\x1a \n\x08MetaData\x12\x14\n\x0c\x64isplay_name\x18\x01 \x01(\t\x1a&\n\x08TagValue\x12\x0b\n\x03tag\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x02\x1a\x98\x04\n\x05Value\x12\n\n\x02id\x18\x01 \x01(\x03\x12\x0b\n\x03tag\x18\x02 \x01(\t\x12\x11\n\ttimestamp\x18\x03 \x01(\x03\x12\x0f\n\x05value\x18\x04 \x01(\x02H\x00\x12\'\n\x05image\x18\x05 \x01(\x0b\x32\x16.visualdl.Record.ImageH\x00\x12\'\n\x05\x61udio\x18\x06 \x01(\x0b\x32\x16.visualdl.Record.AudioH\x00\x12\x31\n\nembeddings\x18\x07 \x01(\x0b\x32\x1b.visualdl.Record.EmbeddingsH\x00\x12/\n\thistogram\x18\x08 \x01(\x0b\x32\x1a.visualdl.Record.HistogramH\x00\x12,\n\x08pr_curve\x18\t \x01(\x0b\x32\x18.visualdl.Record.PRCurveH\x00\x12.\n\tmeta_data\x18\n \x01(\x0b\x32\x19.visualdl.Record.MetaDataH\x00\x12/\n\troc_curve\x18\x0b \x01(\x0b\x32\x1a.visualdl.Record.ROC_CurveH\x00\x12%\n\x04text\x18\x0c \x01(\x0b\x32\x15.visualdl.Record.TextH\x00\x12)\n\x06hparam\x18\r \x01(\x0b\x32\x17.visualdl.Record.HParamH\x00\x12.\n\ttag_value\x18\x0e \x01(\x0b\x32\x19.visualdl.Record.TagValueH\x00\x42\x0b\n\tone_valueb\x06proto3')
_RECORD_PRCURVE = _descriptor.Descriptor(
name='PRCurve',
full_name='visualdl.Record.PRCurve',
filename=None,
file=DESCRIPTOR,
containing_type=None,
fields=[
_descriptor.FieldDescriptor(
name='TP', full_name='visualdl.Record.PRCurve.TP', index=0,
number=1, type=3, cpp_type=2, label=3,
has_default_value=False, default_value=[],
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=b'\020\001', file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='FP', full_name='visualdl.Record.PRCurve.FP', index=1,
number=2, type=3, cpp_type=2, label=3,
has_default_value=False, default_value=[],
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=b'\020\001', file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='TN', full_name='visualdl.Record.PRCurve.TN', index=2,
number=3, type=3, cpp_type=2, label=3,
has_default_value=False, default_value=[],
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=b'\020\001', file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='FN', full_name='visualdl.Record.PRCurve.FN', index=3,
number=4, type=3, cpp_type=2, label=3,
has_default_value=False, default_value=[],
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=b'\020\001', file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='precision', full_name='visualdl.Record.PRCurve.precision', index=4,
number=5, type=1, cpp_type=5, label=3,
has_default_value=False, default_value=[],
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='recall', full_name='visualdl.Record.PRCurve.recall', index=5,
number=6, type=1, cpp_type=5, label=3,
has_default_value=False, default_value=[],
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
],
extensions=[
],
nested_types=[],
enum_types=[
],
serialized_options=None,
is_extendable=False,
syntax='proto3',
extension_ranges=[],
oneofs=[
],
serialized_start=530,
serialized_end=638,
)
_RECORD_ROC_CURVE = _descriptor.Descriptor(
name='ROC_Curve',
full_name='visualdl.Record.ROC_Curve',
filename=None,
file=DESCRIPTOR,
containing_type=None,
fields=[
_descriptor.FieldDescriptor(
name='TP', full_name='visualdl.Record.ROC_Curve.TP', index=0,
number=1, type=3, cpp_type=2, label=3,
has_default_value=False, default_value=[],
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=b'\020\001', file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='FP', full_name='visualdl.Record.ROC_Curve.FP', index=1,
number=2, type=3, cpp_type=2, label=3,
has_default_value=False, default_value=[],
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=b'\020\001', file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='TN', full_name='visualdl.Record.ROC_Curve.TN', index=2,
number=3, type=3, cpp_type=2, label=3,
has_default_value=False, default_value=[],
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=b'\020\001', file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='FN', full_name='visualdl.Record.ROC_Curve.FN', index=3,
number=4, type=3, cpp_type=2, label=3,
has_default_value=False, default_value=[],
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=b'\020\001', file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='tpr', full_name='visualdl.Record.ROC_Curve.tpr', index=4,
number=5, type=1, cpp_type=5, label=3,
has_default_value=False, default_value=[],
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='fpr', full_name='visualdl.Record.ROC_Curve.fpr', index=5,
number=6, type=1, cpp_type=5, label=3,
has_default_value=False, default_value=[],
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
],
extensions=[
],
nested_types=[],
enum_types=[
],
serialized_options=None,
is_extendable=False,
syntax='proto3',
extension_ranges=[],
oneofs=[
],
serialized_start=640,
serialized_end=741,
)
_RECORD_HPARAM_HPARAMINFO = _descriptor.Descriptor(
name='HparamInfo',
full_name='visualdl.Record.HParam.HparamInfo',
filename=None,
file=DESCRIPTOR,
containing_type=None,
fields=[
_descriptor.FieldDescriptor(
name='int_value', full_name='visualdl.Record.HParam.HparamInfo.int_value', index=0,
number=1, type=3, cpp_type=2, label=1,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='float_value', full_name='visualdl.Record.HParam.HparamInfo.float_value', index=1,
number=2, type=1, cpp_type=5, label=1,
has_default_value=False, default_value=float(0),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='string_value', full_name='visualdl.Record.HParam.HparamInfo.string_value', index=2,
number=3, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=b"".decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='name', full_name='visualdl.Record.HParam.HparamInfo.name', index=3,
number=4, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=b"".decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
],
extensions=[
],
nested_types=[],
enum_types=[
],
serialized_options=None,
is_extendable=False,
syntax='proto3',
extension_ranges=[],
oneofs=[
_descriptor.OneofDescriptor(
name='type', full_name='visualdl.Record.HParam.HparamInfo.type',
index=0, containing_type=None, fields=[]),
],
serialized_start=882,
serialized_end=984,
)
_RECORD_HPARAM = _descriptor.Descriptor(
name='HParam',
full_name='visualdl.Record.HParam',
filename=None,
file=DESCRIPTOR,
containing_type=None,
fields=[
_descriptor.FieldDescriptor(
name='hparamInfos', full_name='visualdl.Record.HParam.hparamInfos', index=0,
number=1, type=11, cpp_type=10, label=3,
has_default_value=False, default_value=[],
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='metricInfos', full_name='visualdl.Record.HParam.metricInfos', index=1,
number=2, type=11, cpp_type=10, label=3,
has_default_value=False, default_value=[],
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='name', full_name='visualdl.Record.HParam.name', index=2,
number=3, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=b"".decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
],
extensions=[
],
nested_types=[_RECORD_HPARAM_HPARAMINFO, ],
enum_types=[
],
serialized_options=None,
is_extendable=False,
syntax='proto3',
extension_ranges=[],
oneofs=[
],
serialized_start=744,
serialized_end=984,
)
_RECORD_METADATA = _descriptor.Descriptor(
name='MetaData',
full_name='visualdl.Record.MetaData',
filename=None,
file=DESCRIPTOR,
containing_type=None,
fields=[
_descriptor.FieldDescriptor(
name='display_name', full_name='visualdl.Record.MetaData.display_name', index=0,
number=1, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=b"".decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
],
extensions=[
],
nested_types=[],
enum_types=[
],
serialized_options=None,
is_extendable=False,
syntax='proto3',
extension_ranges=[],
oneofs=[
],
serialized_start=986,
serialized_end=1018,
)
_RECORD_VALUE = _descriptor.Descriptor(
name='Value',
full_name='visualdl.Record.Value',
filename=None,
file=DESCRIPTOR,
containing_type=None,
fields=[
_descriptor.FieldDescriptor(
name='id', full_name='visualdl.Record.Value.id', index=0,
number=1, type=3, cpp_type=2, label=1,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='tag', full_name='visualdl.Record.Value.tag', index=1,
number=2, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=b"".decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='timestamp', full_name='visualdl.Record.Value.timestamp', index=2,
number=3, type=3, cpp_type=2, label=1,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='value', full_name='visualdl.Record.Value.value', index=3,
number=4, type=2, cpp_type=6, label=1,
has_default_value=False, default_value=float(0),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='image', full_name='visualdl.Record.Value.image', index=4,
number=5, type=11, cpp_type=10, label=1,
has_default_value=False, default_value=None,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='audio', full_name='visualdl.Record.Value.audio', index=5,
number=6, type=11, cpp_type=10, label=1,
has_default_value=False, default_value=None,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='embeddings', full_name='visualdl.Record.Value.embeddings', index=6,
number=7, type=11, cpp_type=10, label=1,
has_default_value=False, default_value=None,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='histogram', full_name='visualdl.Record.Value.histogram', index=7,
number=8, type=11, cpp_type=10, label=1,
has_default_value=False, default_value=None,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='pr_curve', full_name='visualdl.Record.Value.pr_curve', index=8,
number=9, type=11, cpp_type=10, label=1,
has_default_value=False, default_value=None,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='meta_data', full_name='visualdl.Record.Value.meta_data', index=9,
number=10, type=11, cpp_type=10, label=1,
has_default_value=False, default_value=None,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='roc_curve', full_name='visualdl.Record.Value.roc_curve', index=10,
number=11, type=11, cpp_type=10, label=1,
has_default_value=False, default_value=None,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='text', full_name='visualdl.Record.Value.text', index=11,
number=12, type=11, cpp_type=10, label=1,
has_default_value=False, default_value=None,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='hparam', full_name='visualdl.Record.Value.hparam', index=12,
number=13, type=11, cpp_type=10, label=1,
has_default_value=False, default_value=None,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
],
extensions=[
],
nested_types=[],
enum_types=[
],
serialized_options=None,
is_extendable=False,
syntax='proto3',
extension_ranges=[],
oneofs=[
_descriptor.OneofDescriptor(
name='one_value', full_name='visualdl.Record.Value.one_value',
index=0, containing_type=None, fields=[]),
],
serialized_start=1021,
serialized_end=1509,
)
_RECORD = _descriptor.Descriptor(
name='Record',
full_name='visualdl.Record',
filename=None,
file=DESCRIPTOR,
containing_type=None,
fields=[
_descriptor.FieldDescriptor(
name='values', full_name='visualdl.Record.values', index=0,
number=1, type=11, cpp_type=10, label=3,
has_default_value=False, default_value=[],
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
],
extensions=[
],
nested_types=[_RECORD_IMAGE, _RECORD_TEXT, _RECORD_AUDIO, _RECORD_EMBEDDING, _RECORD_EMBEDDINGS, _RECORD_BYTES_EMBEDDINGS, _RECORD_HISTOGRAM, _RECORD_PRCURVE, _RECORD_ROC_CURVE, _RECORD_HPARAM, _RECORD_METADATA, _RECORD_VALUE, ],
enum_types=[
],
serialized_options=None,
is_extendable=False,
syntax='proto3',
extension_ranges=[],
oneofs=[
],
serialized_start=27,
serialized_end=1509,
)
_RECORD_IMAGE.containing_type = _RECORD
_RECORD_TEXT.containing_type = _RECORD
_RECORD_AUDIO.containing_type = _RECORD
_RECORD_EMBEDDING.containing_type = _RECORD
_RECORD_EMBEDDINGS.fields_by_name['embeddings'].message_type = _RECORD_EMBEDDING
_RECORD_EMBEDDINGS.containing_type = _RECORD
_RECORD_BYTES_EMBEDDINGS.containing_type = _RECORD
_RECORD_HISTOGRAM.containing_type = _RECORD
_RECORD_PRCURVE.containing_type = _RECORD
_RECORD_ROC_CURVE.containing_type = _RECORD
_RECORD_HPARAM_HPARAMINFO.containing_type = _RECORD_HPARAM
_RECORD_HPARAM_HPARAMINFO.oneofs_by_name['type'].fields.append(
_RECORD_HPARAM_HPARAMINFO.fields_by_name['int_value'])
_RECORD_HPARAM_HPARAMINFO.fields_by_name['int_value'].containing_oneof = _RECORD_HPARAM_HPARAMINFO.oneofs_by_name['type']
_RECORD_HPARAM_HPARAMINFO.oneofs_by_name['type'].fields.append(
_RECORD_HPARAM_HPARAMINFO.fields_by_name['float_value'])
_RECORD_HPARAM_HPARAMINFO.fields_by_name['float_value'].containing_oneof = _RECORD_HPARAM_HPARAMINFO.oneofs_by_name['type']
_RECORD_HPARAM_HPARAMINFO.oneofs_by_name['type'].fields.append(
_RECORD_HPARAM_HPARAMINFO.fields_by_name['string_value'])
_RECORD_HPARAM_HPARAMINFO.fields_by_name['string_value'].containing_oneof = _RECORD_HPARAM_HPARAMINFO.oneofs_by_name['type']
_RECORD_HPARAM.fields_by_name['hparamInfos'].message_type = _RECORD_HPARAM_HPARAMINFO
_RECORD_HPARAM.fields_by_name['metricInfos'].message_type = _RECORD_HPARAM_HPARAMINFO
_RECORD_HPARAM.containing_type = _RECORD
_RECORD_METADATA.containing_type = _RECORD
_RECORD_VALUE.fields_by_name['image'].message_type = _RECORD_IMAGE
_RECORD_VALUE.fields_by_name['audio'].message_type = _RECORD_AUDIO
_RECORD_VALUE.fields_by_name['embeddings'].message_type = _RECORD_EMBEDDINGS
_RECORD_VALUE.fields_by_name['histogram'].message_type = _RECORD_HISTOGRAM
_RECORD_VALUE.fields_by_name['pr_curve'].message_type = _RECORD_PRCURVE
_RECORD_VALUE.fields_by_name['meta_data'].message_type = _RECORD_METADATA
_RECORD_VALUE.fields_by_name['roc_curve'].message_type = _RECORD_ROC_CURVE
_RECORD_VALUE.fields_by_name['text'].message_type = _RECORD_TEXT
_RECORD_VALUE.fields_by_name['hparam'].message_type = _RECORD_HPARAM
_RECORD_VALUE.containing_type = _RECORD
_RECORD_VALUE.oneofs_by_name['one_value'].fields.append(
_RECORD_VALUE.fields_by_name['value'])
_RECORD_VALUE.fields_by_name['value'].containing_oneof = _RECORD_VALUE.oneofs_by_name['one_value']
_RECORD_VALUE.oneofs_by_name['one_value'].fields.append(
_RECORD_VALUE.fields_by_name['image'])
_RECORD_VALUE.fields_by_name['image'].containing_oneof = _RECORD_VALUE.oneofs_by_name['one_value']
_RECORD_VALUE.oneofs_by_name['one_value'].fields.append(
_RECORD_VALUE.fields_by_name['audio'])
_RECORD_VALUE.fields_by_name['audio'].containing_oneof = _RECORD_VALUE.oneofs_by_name['one_value']
_RECORD_VALUE.oneofs_by_name['one_value'].fields.append(
_RECORD_VALUE.fields_by_name['embeddings'])
_RECORD_VALUE.fields_by_name['embeddings'].containing_oneof = _RECORD_VALUE.oneofs_by_name['one_value']
_RECORD_VALUE.oneofs_by_name['one_value'].fields.append(
_RECORD_VALUE.fields_by_name['histogram'])
_RECORD_VALUE.fields_by_name['histogram'].containing_oneof = _RECORD_VALUE.oneofs_by_name['one_value']
_RECORD_VALUE.oneofs_by_name['one_value'].fields.append(
_RECORD_VALUE.fields_by_name['pr_curve'])
_RECORD_VALUE.fields_by_name['pr_curve'].containing_oneof = _RECORD_VALUE.oneofs_by_name['one_value']
_RECORD_VALUE.oneofs_by_name['one_value'].fields.append(
_RECORD_VALUE.fields_by_name['meta_data'])
_RECORD_VALUE.fields_by_name['meta_data'].containing_oneof = _RECORD_VALUE.oneofs_by_name['one_value']
_RECORD_VALUE.oneofs_by_name['one_value'].fields.append(
_RECORD_VALUE.fields_by_name['roc_curve'])
_RECORD_VALUE.fields_by_name['roc_curve'].containing_oneof = _RECORD_VALUE.oneofs_by_name['one_value']
_RECORD_VALUE.oneofs_by_name['one_value'].fields.append(
_RECORD_VALUE.fields_by_name['text'])
_RECORD_VALUE.fields_by_name['text'].containing_oneof = _RECORD_VALUE.oneofs_by_name['one_value']
_RECORD_VALUE.oneofs_by_name['one_value'].fields.append(
_RECORD_VALUE.fields_by_name['hparam'])
_RECORD_VALUE.fields_by_name['hparam'].containing_oneof = _RECORD_VALUE.oneofs_by_name['one_value']
_RECORD.fields_by_name['values'].message_type = _RECORD_VALUE
DESCRIPTOR.message_types_by_name['Record'] = _RECORD
_sym_db.RegisterFileDescriptor(DESCRIPTOR)
_RECORD = DESCRIPTOR.message_types_by_name['Record']
_RECORD_IMAGE = _RECORD.nested_types_by_name['Image']
_RECORD_TEXT = _RECORD.nested_types_by_name['Text']
_RECORD_AUDIO = _RECORD.nested_types_by_name['Audio']
_RECORD_EMBEDDING = _RECORD.nested_types_by_name['Embedding']
_RECORD_EMBEDDINGS = _RECORD.nested_types_by_name['Embeddings']
_RECORD_BYTES_EMBEDDINGS = _RECORD.nested_types_by_name['bytes_embeddings']
_RECORD_HISTOGRAM = _RECORD.nested_types_by_name['Histogram']
_RECORD_PRCURVE = _RECORD.nested_types_by_name['PRCurve']
_RECORD_ROC_CURVE = _RECORD.nested_types_by_name['ROC_Curve']
_RECORD_HPARAM = _RECORD.nested_types_by_name['HParam']
_RECORD_HPARAM_HPARAMINFO = _RECORD_HPARAM.nested_types_by_name['HparamInfo']
_RECORD_METADATA = _RECORD.nested_types_by_name['MetaData']
_RECORD_TAGVALUE = _RECORD.nested_types_by_name['TagValue']
_RECORD_VALUE = _RECORD.nested_types_by_name['Value']
Record = _reflection.GeneratedProtocolMessageType('Record', (_message.Message,), {
'Image' : _reflection.GeneratedProtocolMessageType('Image', (_message.Message,), {
......@@ -849,6 +119,13 @@ Record = _reflection.GeneratedProtocolMessageType('Record', (_message.Message,),
})
,
'TagValue' : _reflection.GeneratedProtocolMessageType('TagValue', (_message.Message,), {
'DESCRIPTOR' : _RECORD_TAGVALUE,
'__module__' : 'record_pb2'
# @@protoc_insertion_point(class_scope:visualdl.Record.TagValue)
})
,
'Value' : _reflection.GeneratedProtocolMessageType('Value', (_message.Message,), {
'DESCRIPTOR' : _RECORD_VALUE,
'__module__' : 'record_pb2'
......@@ -872,17 +149,60 @@ _sym_db.RegisterMessage(Record.ROC_Curve)
_sym_db.RegisterMessage(Record.HParam)
_sym_db.RegisterMessage(Record.HParam.HparamInfo)
_sym_db.RegisterMessage(Record.MetaData)
_sym_db.RegisterMessage(Record.TagValue)
_sym_db.RegisterMessage(Record.Value)
_RECORD_HISTOGRAM.fields_by_name['hist']._options = None
_RECORD_HISTOGRAM.fields_by_name['bin_edges']._options = None
_RECORD_PRCURVE.fields_by_name['TP']._options = None
_RECORD_PRCURVE.fields_by_name['FP']._options = None
_RECORD_PRCURVE.fields_by_name['TN']._options = None
_RECORD_PRCURVE.fields_by_name['FN']._options = None
_RECORD_ROC_CURVE.fields_by_name['TP']._options = None
_RECORD_ROC_CURVE.fields_by_name['FP']._options = None
_RECORD_ROC_CURVE.fields_by_name['TN']._options = None
_RECORD_ROC_CURVE.fields_by_name['FN']._options = None
if _descriptor._USE_C_DESCRIPTORS == False:
DESCRIPTOR._options = None
_RECORD_HISTOGRAM.fields_by_name['hist']._options = None
_RECORD_HISTOGRAM.fields_by_name['hist']._serialized_options = b'\020\001'
_RECORD_HISTOGRAM.fields_by_name['bin_edges']._options = None
_RECORD_HISTOGRAM.fields_by_name['bin_edges']._serialized_options = b'\020\001'
_RECORD_PRCURVE.fields_by_name['TP']._options = None
_RECORD_PRCURVE.fields_by_name['TP']._serialized_options = b'\020\001'
_RECORD_PRCURVE.fields_by_name['FP']._options = None
_RECORD_PRCURVE.fields_by_name['FP']._serialized_options = b'\020\001'
_RECORD_PRCURVE.fields_by_name['TN']._options = None
_RECORD_PRCURVE.fields_by_name['TN']._serialized_options = b'\020\001'
_RECORD_PRCURVE.fields_by_name['FN']._options = None
_RECORD_PRCURVE.fields_by_name['FN']._serialized_options = b'\020\001'
_RECORD_ROC_CURVE.fields_by_name['TP']._options = None
_RECORD_ROC_CURVE.fields_by_name['TP']._serialized_options = b'\020\001'
_RECORD_ROC_CURVE.fields_by_name['FP']._options = None
_RECORD_ROC_CURVE.fields_by_name['FP']._serialized_options = b'\020\001'
_RECORD_ROC_CURVE.fields_by_name['TN']._options = None
_RECORD_ROC_CURVE.fields_by_name['TN']._serialized_options = b'\020\001'
_RECORD_ROC_CURVE.fields_by_name['FN']._options = None
_RECORD_ROC_CURVE.fields_by_name['FN']._serialized_options = b'\020\001'
_RECORD._serialized_start=27
_RECORD._serialized_end=1597
_RECORD_IMAGE._serialized_start=77
_RECORD_IMAGE._serialized_end=114
_RECORD_TEXT._serialized_start=116
_RECORD_TEXT._serialized_end=151
_RECORD_AUDIO._serialized_start=153
_RECORD_AUDIO._serialized_end=278
_RECORD_EMBEDDING._serialized_start=280
_RECORD_EMBEDDING._serialized_end=323
_RECORD_EMBEDDINGS._serialized_start=325
_RECORD_EMBEDDINGS._serialized_end=405
_RECORD_BYTES_EMBEDDINGS._serialized_start=407
_RECORD_BYTES_EMBEDDINGS._serialized_end=474
_RECORD_HISTOGRAM._serialized_start=476
_RECORD_HISTOGRAM._serialized_end=528
_RECORD_PRCURVE._serialized_start=530
_RECORD_PRCURVE._serialized_end=638
_RECORD_ROC_CURVE._serialized_start=640
_RECORD_ROC_CURVE._serialized_end=741
_RECORD_HPARAM._serialized_start=744
_RECORD_HPARAM._serialized_end=984
_RECORD_HPARAM_HPARAMINFO._serialized_start=882
_RECORD_HPARAM_HPARAMINFO._serialized_end=984
_RECORD_METADATA._serialized_start=986
_RECORD_METADATA._serialized_end=1018
_RECORD_TAGVALUE._serialized_start=1020
_RECORD_TAGVALUE._serialized_end=1058
_RECORD_VALUE._serialized_start=1061
_RECORD_VALUE._serialized_end=1597
# @@protoc_insertion_point(module_scope)
visualdl/reader/reader.py
浏览文件 @ f7af2157
......@@ -123,6 +123,13 @@ class LogReader(object):
for item in proto_datas:
data.append([item.id, item.tag, item.timestamp, item.value])
return data
elif 'scalars' == component:
for item in proto_datas:
data.append([
item.id, item.tag, item.tag_value.tag, item.timestamp,
item.tag_value.value
])
return data
return proto_datas
def _get_log_tags(self):
......@@ -143,6 +150,8 @@ class LogReader(object):
else:
file_path = bfile.join(run, self.walks[run])
reader = self._get_file_reader(file_path=file_path, update=False)
reader.dir = run
self.reader = reader
remain = self.get_remain(reader=reader)
data = self.read_log_data(
remain=remain, update=False)[component][tag]
......@@ -191,6 +200,8 @@ class LogReader(object):
component = "text"
elif "hparam" == value_type:
component = "hyper_parameters"
elif "tag_value" == value_type:
component = "scalars"
else:
raise TypeError("Invalid value type `%s`." % value_type)
self._tags[path] = component
......@@ -265,7 +276,6 @@ class LogReader(object):
if update:
self.register_reader(file_path)
self.reader = self.readers[file_path]
self.reader.dir = file_path
return self.reader
else:
reader = RecordReader(filepath=file_path)
......@@ -275,6 +285,7 @@ class LogReader(object):
if update:
if path not in list(self.readers.keys()):
reader = RecordReader(filepath=path, dir=dir)
reader.dir = dir
self.readers[path] = reader
else:
pass
......
visualdl/server/api.py
浏览文件 @ f7af2157
......@@ -135,6 +135,11 @@ class Api(object):
return self._get_with_retry('data/plugin/scalars/tags',
lib.get_scalar_tags)
@result()
def scalars_tags(self):
return self._get_with_retry('data/plugin/multiscalars/tags',
lib.get_scalars_tags)
@result()
def image_tags(self):
return self._get_with_retry('data/plugin/images/tags',
......@@ -194,11 +199,24 @@ class Api(object):
key = os.path.join('data/plugin/scalars/scalars', run, tag)
return self._get_with_retry(key, lib.get_scalar, run, tag)
@result()
def scalars_list(self, run, tag, sub_tag):
key = os.path.join('data/plugin/multiscalars/scalars', run, tag,
sub_tag)
return self._get_with_retry(key, lib.get_scalars, run, tag, sub_tag)
@result('text/csv')
def scalar_data(self, run, tag, type='tsv'):
key = os.path.join('data/plugin/scalars/data', run, tag, type)
return self._get_with_retry(key, lib.get_scalar_data, run, tag, type)
@result('text/csv')
def scalars_data(self, run, tag, sub_tag, type='tsv'):
key = os.path.join('data/plugin/multiscalars/data', run, tag, sub_tag,
type)
return self._get_with_retry(key, lib.get_scalars_data, run, tag,
sub_tag, type)
@result()
def image_list(self, mode, tag):
key = os.path.join('data/plugin/images/images', mode, tag)
......@@ -412,6 +430,7 @@ def create_api_call(logdir, model, cache_timeout):
'tags': (api.tags, []),
'logs': (api.logs, []),
'scalar/tags': (api.scalar_tags, []),
'scalars/tags': (api.scalars_tags, []),
'image/tags': (api.image_tags, []),
'text/tags': (api.text_tags, []),
'audio/tags': (api.audio_tags, []),
......@@ -420,7 +439,9 @@ def create_api_call(logdir, model, cache_timeout):
'pr-curve/tags': (api.pr_curve_tags, []),
'roc-curve/tags': (api.roc_curve_tags, []),
'scalar/list': (api.scalar_list, ['run', 'tag']),
'scalars/list': (api.scalars_list, ['run', 'tag', 'sub_tag']),
'scalar/data': (api.scalar_data, ['run', 'tag', 'type']),
'scalars/data': (api.scalars_data, ['run', 'tag', 'sub_tag', 'type']),
'image/list': (api.image_list, ['run', 'tag']),
'image/image': (api.image_image, ['run', 'tag', 'index']),
'text/list': (api.text_list, ['run', 'tag']),
......
visualdl/server/data_manager.py
浏览文件 @ f7af2157
......@@ -12,10 +12,9 @@
# See the License for the specific language governing permissions and
# limitations under the License.
# =======================================================================
import threading
import random
import collections
import random
import threading
DEFAULT_PLUGIN_MAXSIZE = {
"scalar": 1000,
......@@ -31,6 +30,10 @@ DEFAULT_PLUGIN_MAXSIZE = {
}
def add_sub_tag(tag, sub_tag):
return tag.replace('%', '_') + '_' + sub_tag
class Reservoir(object):
"""A map-to-arrays dict, with deterministic Reservoir Sampling.
......@@ -53,10 +56,8 @@ class Reservoir(object):
if max_size < 0 or max_size != round(max_size):
raise ValueError("Max_size must be nonnegative integer.")
self._max_size = max_size
self._buckets = collections.defaultdict(
lambda: _ReservoirBucket(max_size=self._max_size,
random_instance=random.Random(seed))
)
self._buckets = collections.defaultdict(lambda: _ReservoirBucket(
max_size=self._max_size, random_instance=random.Random(seed)))
self._mutex = threading.Lock()
@property
......@@ -174,6 +175,25 @@ class Reservoir(object):
with self._mutex:
self._buckets[key].add_scalar_item(item)
def _add_scalars_item(self, key, item):
"""Add a new scalar item to reservoir buckets with given tag as key.
If bucket with key has not yet reached full size, each item will be
added.
If bucket with key is full, each item will be added with same
probability.
Add new item to buckets will always valid because self._buckets is a
collection.defaultdict.
Args:
key: Tag of one bucket to add new item.
item: New item to add to bucket.
"""
with self._mutex:
self._buckets[key].add_scalars_item(item)
def add_item(self, run, tag, item):
"""Add a new item to reservoir buckets with given tag as key.
......@@ -197,8 +217,14 @@ class Reservoir(object):
tag: Identity of one record in tablet.
item: New item to add to bucket.
"""
if item.WhichOneof("one_value") == "value":
key = run + "/" + tag
self._add_scalar_item(key, item)
elif item.WhichOneof("one_value") == "tag_value":
key = run + "/" + add_sub_tag(tag, item.tag_value.tag) + "/" + tag
self._add_scalars_item(key, item)
else:
raise ValueError("Not scalar type:" + item.WhichOneof("one_value"))
def _cut_tail(self, key):
with self._mutex:
......@@ -247,6 +273,9 @@ class _ReservoirBucket(object):
self.max_scalar = None
self.min_scalar = None
# improve performance when data is monotonous
self._last_special = False
def add_item(self, item):
""" Add an item to bucket, replacing an old item with probability.
......@@ -274,24 +303,101 @@ class _ReservoirBucket(object):
Use reservoir sampling to add a new item to sampling bucket,
each item in a steam has same probability stay in the bucket.
use _last_special mark to improve performance when data is monotonous
Args:
item: The item to add to reservoir bucket.
"""
with self._mutex:
# save max and min value
if not self.max_scalar or self.max_scalar.value < item.value:
self.max_scalar = item
if not self.min_scalar or self.min_scalar.value > item.value:
self.min_scalar = item
if len(self._items) < self._max_size or self._max_size == 0:
# capacity is valid, append directly
self._items.append(item)
else:
if self._last_special:
if self._items[-1].id == self.min_scalar.id or self._items[
-1].id == self.max_scalar.id:
# data is not monotonous, set special to False
self._last_special = False
else:
# data is monotonous, drop last item by reservoir algorithm
r = self._random.randint(1, self._num_items_index)
if r >= self._max_size:
self._items.pop(-1)
self._items.append(item)
self._num_items_index += 1
return
if item.id == self.min_scalar.id or item.id == self.max_scalar.id:
# this item is max or min, should be reserved
r = self._random.randint(1, self._max_size - 1)
self._last_special = True
else:
# drop by reservoir algorithm
r = self._random.randint(1, self._num_items_index)
self._last_special = False
if r < self._max_size:
if self._items[r].id == self.min_scalar.id or self._items[
r].id == self.max_scalar.id:
# reserve max and min point
if r - 1 > 0:
r = r - 1
elif r + 1 < self._max_size:
r = r + 1
self._items.pop(r)
self._items.append(item)
self._num_items_index += 1
def add_scalars_item(self, item):
""" Add an scalar item to bucket, replacing an old item with probability.
Use reservoir sampling to add a new item to sampling bucket,
each item in a steam has same probability stay in the bucket.
Args:
item: The item to add to reservoir bucket.
"""
with self._mutex:
# save max and min value
if not self.max_scalar or self.max_scalar.tag_value.value < item.tag_value.value:
self.max_scalar = item
if not self.min_scalar or self.min_scalar.tag_value.value > item.tag_value.value:
self.min_scalar = item
if len(self._items) < self._max_size or self._max_size == 0:
# capacity is valid, append directly
self._items.append(item)
else:
if self._last_special:
if self._items[-1].id == self.min_scalar.id or self._items[
-1].id == self.max_scalar.id:
# data is not monotic, set special to False
self._last_special = False
else:
# data is monotic, drop last item by reservoir algorithm
r = self._random.randint(1, self._num_items_index)
if r >= self._max_size:
self._items.pop(-1)
self._items.append(item)
self._num_items_index += 1
return
if item.id == self.min_scalar.id or item.id == self.max_scalar.id:
# this item is max or min, should be reserved
r = self._random.randint(1, self._max_size - 1)
self._last_special = True
else:
# drop by reservoir algorithm
r = self._random.randint(1, self._num_items_index)
self._last_special = False
if r < self._max_size:
if self._items[r].id == self.min_scalar.id or self._items[r].id == self.max_scalar.id:
if self._items[r].id == self.min_scalar.id or self._items[
r].id == self.max_scalar.id:
# reserve max and min point
if r - 1 > 0:
r = r - 1
elif r + 1 < self._max_size:
......@@ -405,8 +511,8 @@ class DataManager(object):
item: The item to add to reservoir bucket.
"""
with self._mutex:
if 'scalar' == plugin:
self._reservoirs[plugin].add_scalar_item(run, tag, item)
if 'scalar' == plugin or 'scalars' == plugin: # We adapt scalars data to be saved in scalar reservoir.
self._reservoirs['scalar'].add_scalar_item(run, tag, item)
else:
self._reservoirs[plugin].add_item(run, tag, item)
......
visualdl/server/lib.py
浏览文件 @ f7af2157
......@@ -335,20 +335,37 @@ def get_scalar(log_reader, run, tag):
log_reader.load_new_data()
records = log_reader.data_manager.get_reservoir("scalar").get_items(
run, decode_tag(tag))
results = [[
s2ms(item.timestamp), item.id,
transfer_abnomal_scalar_value(item.value)
] if item.WhichOneof("one_value") == "value" else [
s2ms(item.timestamp), item.id,
transfer_abnomal_scalar_value(item.tag_value.value)
] for item in records]
return results
def get_scalar_data(log_reader, run, tag, type='tsv'):
is_scalars = False
if os.path.basename(run).startswith(decode_tag(tag).replace('%', '_')) and \
log_reader.tags().get(bfile.join(os.path.dirname(run), decode_tag(tag)), None) == 'scalars':
run = os.path.dirname(run)
is_scalars = True
run = log_reader.name2tags[run] if run in log_reader.name2tags else run
log_reader.load_new_data()
if is_scalars:
result = log_reader.get_log_data('scalars', run, decode_tag(tag))
else:
result = log_reader.get_log_data('scalar', run, decode_tag(tag))
print('scalar', result, 'run', run, 'tag', decode_tag(tag))
delimeter = '\t' if 'tsv' == type else ','
with io.StringIO() as fp:
csv_writer = csv.writer(fp, delimiter=delimeter)
if is_scalars:
csv_writer.writerow(['id', 'tag', 'sub_tag', 'timestamp', 'value'])
else:
csv_writer.writerow(['id', 'tag', 'timestamp', 'value'])
csv_writer.writerows(result)
result = fp.getvalue()
......
visualdl/writer/writer.py
浏览文件 @ f7af2157
......@@ -26,6 +26,7 @@ from visualdl.component.base_component import meta_data
from visualdl.component.base_component import pr_curve
from visualdl.component.base_component import roc_curve
from visualdl.component.base_component import scalar
from visualdl.component.base_component import scalars
from visualdl.component.base_component import text
from visualdl.component.graph import translate_graph
from visualdl.io import bfile
......@@ -187,6 +188,30 @@ class LogWriter(object):
self._get_file_writer().add_record(
scalar(tag=tag, value=value, step=step, walltime=walltime))
def add_scalars(self, main_tag, tag_scalar_dict, step, walltime=None):
"""Add a group of scalars to vdl record file.
Args:
main_tag (string): Data identifier
tag_scalar_dict (float): A dict to provide multi-values with tags
step (int): Step of scalar
walltime (int): Wall time of scalar
Example:
import math
for index in range(1, 101):
alpha = index*2*math.pi/100
tval = {'sin':math.sin(alpha), 'cos':math.cos(alpha)}
writer.add_scalars(tag="sin_and_cos", tag_value=tval, step=index)
"""
if '%' in main_tag:
raise RuntimeError("% can't appear in tag!")
if not isinstance(tag_scalar_dict, dict):
raise RuntimeError("tag_value must be a dict!")
walltime = round(time.time() * 1000) if walltime is None else walltime
for record in scalars(main_tag, tag_scalar_dict, step, walltime):
self._get_file_writer().add_record(record)
def add_image(self, tag, img, step, walltime=None, dataformats="HWC"):
"""Add an image to vdl record file.
......
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或