From f7af2157f7254a3c14fcc9d21bc25eee27f63d09 Mon Sep 17 00:00:00 2001 From: chenjian Date: Mon, 28 Nov 2022 20:27:58 +0800 Subject: [PATCH] Add scalars component (#1081) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * backend of add_scalars * fix function name bug of get_scalars_tags * @zh794390558 and @ShenYuhan commented 0508 * fix a bug * adapt to frontend runs need * fix proto * fix a bug * add scalars test demo * fix * adapt scalars to scalar * remove unused code * fix demo code * fix proto version * fix proto version * upgrade protoc to v3.19.0 * fix a bug in logreader Co-authored-by: 伽利略 Co-authored-by: wuzewu --- demo/components/scalars_test.py | 41 ++ visualdl/component/base_component.py | 23 + visualdl/proto/record.proto | 6 + visualdl/proto/record_pb2.py | 838 +++------------------------ visualdl/reader/reader.py | 13 +- visualdl/server/api.py | 21 + visualdl/server/data_manager.py | 130 ++++- visualdl/server/lib.py | 21 +- visualdl/writer/writer.py | 25 + 9 files changed, 344 insertions(+), 774 deletions(-) create mode 100644 demo/components/scalars_test.py diff --git a/demo/components/scalars_test.py b/demo/components/scalars_test.py new file mode 100644 index 00000000..6cd93414 --- /dev/null +++ b/demo/components/scalars_test.py @@ -0,0 +1,41 @@ +# Copyright (c) 2022 VisualDL Authors. All Rights Reserve. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ======================================================================= +# coding=utf-8 +import math + +from visualdl import LogWriter + +if __name__ == '__main__': + value = [i / 1000.0 for i in range(1000)] + with LogWriter(logdir="./log/scalars_test/") as writer: + for step in range(1000): + writer.add_scalars( + main_tag='math/formula1', + tag_scalar_dict={ + 'sinx': math.sin(value[step]), + 'cosx': math.cos(value[step]) + }, + step=step) + writer.add_scalars( + main_tag='math/formula2', + tag_scalar_dict={ + 'sqrtx': math.sqrt(value[step]), + 'squarex': value[step]**2 + }, + step=step) + with LogWriter(logdir="./log/scalar_test1/") as writer: + for step in range(1000): + writer.add_scalar( + tag='math/formula2', value=value[step], step=step) diff --git a/visualdl/component/base_component.py b/visualdl/component/base_component.py index b6ea1ef9..d1f7de9b 100644 --- a/visualdl/component/base_component.py +++ b/visualdl/component/base_component.py @@ -18,6 +18,29 @@ from PIL import Image from visualdl.proto.record_pb2 import Record +def scalars(main_tag, tag_scalar_dict, step, walltime=None): + """Package data to scalars + + Args: + main_tag (string): Data identifier + tag_scalar_dict (dict): A dict to provide multi-values with tags + step (int): Step of scalar + walltime (int): Wall time of scalar + + Return: + Package with format of record_pb2.Record + """ + for sub_tag, value in tag_scalar_dict.items(): + value = float(value) + yield Record(values=[ + Record.Value( + id=step, + tag=main_tag, + timestamp=walltime, + tag_value=Record.TagValue(tag=sub_tag, value=value)) + ]) + + def scalar(tag, value, step, walltime=None): """Package data to one scalar. Args: diff --git a/visualdl/proto/record.proto b/visualdl/proto/record.proto index 680d0adb..63073fdf 100644 --- a/visualdl/proto/record.proto +++ b/visualdl/proto/record.proto @@ -75,6 +75,11 @@ message Record { string display_name = 1; } + message TagValue { + string tag = 1; + float value = 2; + } + message Value { int64 id = 1; string tag = 2; @@ -90,6 +95,7 @@ message Record { ROC_Curve roc_curve = 11; Text text = 12; HParam hparam = 13; + TagValue tag_value = 14; } } diff --git a/visualdl/proto/record_pb2.py b/visualdl/proto/record_pb2.py index 1dc3fd22..baac5bd1 100644 --- a/visualdl/proto/record_pb2.py +++ b/visualdl/proto/record_pb2.py @@ -1,8 +1,9 @@ # -*- coding: utf-8 -*- # Generated by the protocol buffer compiler. DO NOT EDIT! # source: record.proto - +"""Generated protocol buffer code.""" from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool from google.protobuf import message as _message from google.protobuf import reflection as _reflection from google.protobuf import symbol_database as _symbol_database @@ -13,756 +14,25 @@ _sym_db = _symbol_database.Default() -DESCRIPTOR = _descriptor.FileDescriptor( - name='record.proto', - package='visualdl', - syntax='proto3', - serialized_options=None, - serialized_pb=b'\n\x0crecord.proto\x12\x08visualdl\"\xca\x0b\n\x06Record\x12&\n\x06values\x18\x01 \x03(\x0b\x32\x16.visualdl.Record.Value\x1a%\n\x05Image\x12\x1c\n\x14\x65ncoded_image_string\x18\x04 \x01(\x0c\x1a#\n\x04Text\x12\x1b\n\x13\x65ncoded_text_string\x18\x01 \x01(\t\x1a}\n\x05\x41udio\x12\x13\n\x0bsample_rate\x18\x01 \x01(\x02\x12\x14\n\x0cnum_channels\x18\x02 \x01(\x03\x12\x15\n\rlength_frames\x18\x03 \x01(\x03\x12\x1c\n\x14\x65ncoded_audio_string\x18\x04 \x01(\x0c\x12\x14\n\x0c\x63ontent_type\x18\x05 \x01(\t\x1a+\n\tEmbedding\x12\r\n\x05label\x18\x01 \x03(\t\x12\x0f\n\x07vectors\x18\x02 \x03(\x02\x1aP\n\nEmbeddings\x12.\n\nembeddings\x18\x01 \x03(\x0b\x32\x1a.visualdl.Record.Embedding\x12\x12\n\nlabel_meta\x18\x02 \x03(\t\x1a\x43\n\x10\x62ytes_embeddings\x12\x16\n\x0e\x65ncoded_labels\x18\x01 \x01(\x0c\x12\x17\n\x0f\x65ncoded_vectors\x18\x02 \x01(\x0c\x1a\x34\n\tHistogram\x12\x10\n\x04hist\x18\x01 \x03(\x01\x42\x02\x10\x01\x12\x15\n\tbin_edges\x18\x02 \x03(\x01\x42\x02\x10\x01\x1al\n\x07PRCurve\x12\x0e\n\x02TP\x18\x01 \x03(\x03\x42\x02\x10\x01\x12\x0e\n\x02\x46P\x18\x02 \x03(\x03\x42\x02\x10\x01\x12\x0e\n\x02TN\x18\x03 \x03(\x03\x42\x02\x10\x01\x12\x0e\n\x02\x46N\x18\x04 \x03(\x03\x42\x02\x10\x01\x12\x11\n\tprecision\x18\x05 \x03(\x01\x12\x0e\n\x06recall\x18\x06 \x03(\x01\x1a\x65\n\tROC_Curve\x12\x0e\n\x02TP\x18\x01 \x03(\x03\x42\x02\x10\x01\x12\x0e\n\x02\x46P\x18\x02 \x03(\x03\x42\x02\x10\x01\x12\x0e\n\x02TN\x18\x03 \x03(\x03\x42\x02\x10\x01\x12\x0e\n\x02\x46N\x18\x04 \x03(\x03\x42\x02\x10\x01\x12\x0b\n\x03tpr\x18\x05 \x03(\x01\x12\x0b\n\x03\x66pr\x18\x06 \x03(\x01\x1a\xf0\x01\n\x06HParam\x12\x37\n\x0bhparamInfos\x18\x01 \x03(\x0b\x32\".visualdl.Record.HParam.HparamInfo\x12\x37\n\x0bmetricInfos\x18\x02 \x03(\x0b\x32\".visualdl.Record.HParam.HparamInfo\x12\x0c\n\x04name\x18\x03 \x01(\t\x1a\x66\n\nHparamInfo\x12\x13\n\tint_value\x18\x01 \x01(\x03H\x00\x12\x15\n\x0b\x66loat_value\x18\x02 \x01(\x01H\x00\x12\x16\n\x0cstring_value\x18\x03 \x01(\tH\x00\x12\x0c\n\x04name\x18\x04 \x01(\tB\x06\n\x04type\x1a \n\x08MetaData\x12\x14\n\x0c\x64isplay_name\x18\x01 \x01(\t\x1a\xe8\x03\n\x05Value\x12\n\n\x02id\x18\x01 \x01(\x03\x12\x0b\n\x03tag\x18\x02 \x01(\t\x12\x11\n\ttimestamp\x18\x03 \x01(\x03\x12\x0f\n\x05value\x18\x04 \x01(\x02H\x00\x12\'\n\x05image\x18\x05 \x01(\x0b\x32\x16.visualdl.Record.ImageH\x00\x12\'\n\x05\x61udio\x18\x06 \x01(\x0b\x32\x16.visualdl.Record.AudioH\x00\x12\x31\n\nembeddings\x18\x07 \x01(\x0b\x32\x1b.visualdl.Record.EmbeddingsH\x00\x12/\n\thistogram\x18\x08 \x01(\x0b\x32\x1a.visualdl.Record.HistogramH\x00\x12,\n\x08pr_curve\x18\t \x01(\x0b\x32\x18.visualdl.Record.PRCurveH\x00\x12.\n\tmeta_data\x18\n \x01(\x0b\x32\x19.visualdl.Record.MetaDataH\x00\x12/\n\troc_curve\x18\x0b \x01(\x0b\x32\x1a.visualdl.Record.ROC_CurveH\x00\x12%\n\x04text\x18\x0c \x01(\x0b\x32\x15.visualdl.Record.TextH\x00\x12)\n\x06hparam\x18\r \x01(\x0b\x32\x17.visualdl.Record.HParamH\x00\x42\x0b\n\tone_valueb\x06proto3' -) - - - - -_RECORD_IMAGE = _descriptor.Descriptor( - name='Image', - full_name='visualdl.Record.Image', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='encoded_image_string', full_name='visualdl.Record.Image.encoded_image_string', index=0, - number=4, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=b"", - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=77, - serialized_end=114, -) - -_RECORD_TEXT = _descriptor.Descriptor( - name='Text', - full_name='visualdl.Record.Text', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='encoded_text_string', full_name='visualdl.Record.Text.encoded_text_string', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=116, - serialized_end=151, -) - -_RECORD_AUDIO = _descriptor.Descriptor( - name='Audio', - full_name='visualdl.Record.Audio', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='sample_rate', full_name='visualdl.Record.Audio.sample_rate', index=0, - number=1, type=2, cpp_type=6, label=1, - has_default_value=False, default_value=float(0), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='num_channels', full_name='visualdl.Record.Audio.num_channels', index=1, - number=2, type=3, cpp_type=2, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='length_frames', full_name='visualdl.Record.Audio.length_frames', index=2, - number=3, type=3, cpp_type=2, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='encoded_audio_string', full_name='visualdl.Record.Audio.encoded_audio_string', index=3, - number=4, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=b"", - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='content_type', full_name='visualdl.Record.Audio.content_type', index=4, - number=5, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=153, - serialized_end=278, -) - -_RECORD_EMBEDDING = _descriptor.Descriptor( - name='Embedding', - full_name='visualdl.Record.Embedding', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='label', full_name='visualdl.Record.Embedding.label', index=0, - number=1, type=9, cpp_type=9, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='vectors', full_name='visualdl.Record.Embedding.vectors', index=1, - number=2, type=2, cpp_type=6, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=280, - serialized_end=323, -) - -_RECORD_EMBEDDINGS = _descriptor.Descriptor( - name='Embeddings', - full_name='visualdl.Record.Embeddings', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='embeddings', full_name='visualdl.Record.Embeddings.embeddings', index=0, - number=1, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='label_meta', full_name='visualdl.Record.Embeddings.label_meta', index=1, - number=2, type=9, cpp_type=9, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=325, - serialized_end=405, -) - -_RECORD_BYTES_EMBEDDINGS = _descriptor.Descriptor( - name='bytes_embeddings', - full_name='visualdl.Record.bytes_embeddings', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='encoded_labels', full_name='visualdl.Record.bytes_embeddings.encoded_labels', index=0, - number=1, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=b"", - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='encoded_vectors', full_name='visualdl.Record.bytes_embeddings.encoded_vectors', index=1, - number=2, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=b"", - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=407, - serialized_end=474, -) - -_RECORD_HISTOGRAM = _descriptor.Descriptor( - name='Histogram', - full_name='visualdl.Record.Histogram', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='hist', full_name='visualdl.Record.Histogram.hist', index=0, - number=1, type=1, cpp_type=5, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=b'\020\001', file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='bin_edges', full_name='visualdl.Record.Histogram.bin_edges', index=1, - number=2, type=1, cpp_type=5, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=b'\020\001', file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=476, - serialized_end=528, -) +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0crecord.proto\x12\x08visualdl\"\xa2\x0c\n\x06Record\x12&\n\x06values\x18\x01 \x03(\x0b\x32\x16.visualdl.Record.Value\x1a%\n\x05Image\x12\x1c\n\x14\x65ncoded_image_string\x18\x04 \x01(\x0c\x1a#\n\x04Text\x12\x1b\n\x13\x65ncoded_text_string\x18\x01 \x01(\t\x1a}\n\x05\x41udio\x12\x13\n\x0bsample_rate\x18\x01 \x01(\x02\x12\x14\n\x0cnum_channels\x18\x02 \x01(\x03\x12\x15\n\rlength_frames\x18\x03 \x01(\x03\x12\x1c\n\x14\x65ncoded_audio_string\x18\x04 \x01(\x0c\x12\x14\n\x0c\x63ontent_type\x18\x05 \x01(\t\x1a+\n\tEmbedding\x12\r\n\x05label\x18\x01 \x03(\t\x12\x0f\n\x07vectors\x18\x02 \x03(\x02\x1aP\n\nEmbeddings\x12.\n\nembeddings\x18\x01 \x03(\x0b\x32\x1a.visualdl.Record.Embedding\x12\x12\n\nlabel_meta\x18\x02 \x03(\t\x1a\x43\n\x10\x62ytes_embeddings\x12\x16\n\x0e\x65ncoded_labels\x18\x01 \x01(\x0c\x12\x17\n\x0f\x65ncoded_vectors\x18\x02 \x01(\x0c\x1a\x34\n\tHistogram\x12\x10\n\x04hist\x18\x01 \x03(\x01\x42\x02\x10\x01\x12\x15\n\tbin_edges\x18\x02 \x03(\x01\x42\x02\x10\x01\x1al\n\x07PRCurve\x12\x0e\n\x02TP\x18\x01 \x03(\x03\x42\x02\x10\x01\x12\x0e\n\x02\x46P\x18\x02 \x03(\x03\x42\x02\x10\x01\x12\x0e\n\x02TN\x18\x03 \x03(\x03\x42\x02\x10\x01\x12\x0e\n\x02\x46N\x18\x04 \x03(\x03\x42\x02\x10\x01\x12\x11\n\tprecision\x18\x05 \x03(\x01\x12\x0e\n\x06recall\x18\x06 \x03(\x01\x1a\x65\n\tROC_Curve\x12\x0e\n\x02TP\x18\x01 \x03(\x03\x42\x02\x10\x01\x12\x0e\n\x02\x46P\x18\x02 \x03(\x03\x42\x02\x10\x01\x12\x0e\n\x02TN\x18\x03 \x03(\x03\x42\x02\x10\x01\x12\x0e\n\x02\x46N\x18\x04 \x03(\x03\x42\x02\x10\x01\x12\x0b\n\x03tpr\x18\x05 \x03(\x01\x12\x0b\n\x03\x66pr\x18\x06 \x03(\x01\x1a\xf0\x01\n\x06HParam\x12\x37\n\x0bhparamInfos\x18\x01 \x03(\x0b\x32\".visualdl.Record.HParam.HparamInfo\x12\x37\n\x0bmetricInfos\x18\x02 \x03(\x0b\x32\".visualdl.Record.HParam.HparamInfo\x12\x0c\n\x04name\x18\x03 \x01(\t\x1a\x66\n\nHparamInfo\x12\x13\n\tint_value\x18\x01 \x01(\x03H\x00\x12\x15\n\x0b\x66loat_value\x18\x02 \x01(\x01H\x00\x12\x16\n\x0cstring_value\x18\x03 \x01(\tH\x00\x12\x0c\n\x04name\x18\x04 \x01(\tB\x06\n\x04type\x1a \n\x08MetaData\x12\x14\n\x0c\x64isplay_name\x18\x01 \x01(\t\x1a&\n\x08TagValue\x12\x0b\n\x03tag\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x02\x1a\x98\x04\n\x05Value\x12\n\n\x02id\x18\x01 \x01(\x03\x12\x0b\n\x03tag\x18\x02 \x01(\t\x12\x11\n\ttimestamp\x18\x03 \x01(\x03\x12\x0f\n\x05value\x18\x04 \x01(\x02H\x00\x12\'\n\x05image\x18\x05 \x01(\x0b\x32\x16.visualdl.Record.ImageH\x00\x12\'\n\x05\x61udio\x18\x06 \x01(\x0b\x32\x16.visualdl.Record.AudioH\x00\x12\x31\n\nembeddings\x18\x07 \x01(\x0b\x32\x1b.visualdl.Record.EmbeddingsH\x00\x12/\n\thistogram\x18\x08 \x01(\x0b\x32\x1a.visualdl.Record.HistogramH\x00\x12,\n\x08pr_curve\x18\t \x01(\x0b\x32\x18.visualdl.Record.PRCurveH\x00\x12.\n\tmeta_data\x18\n \x01(\x0b\x32\x19.visualdl.Record.MetaDataH\x00\x12/\n\troc_curve\x18\x0b \x01(\x0b\x32\x1a.visualdl.Record.ROC_CurveH\x00\x12%\n\x04text\x18\x0c \x01(\x0b\x32\x15.visualdl.Record.TextH\x00\x12)\n\x06hparam\x18\r \x01(\x0b\x32\x17.visualdl.Record.HParamH\x00\x12.\n\ttag_value\x18\x0e \x01(\x0b\x32\x19.visualdl.Record.TagValueH\x00\x42\x0b\n\tone_valueb\x06proto3') -_RECORD_PRCURVE = _descriptor.Descriptor( - name='PRCurve', - full_name='visualdl.Record.PRCurve', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='TP', full_name='visualdl.Record.PRCurve.TP', index=0, - number=1, type=3, cpp_type=2, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=b'\020\001', file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='FP', full_name='visualdl.Record.PRCurve.FP', index=1, - number=2, type=3, cpp_type=2, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=b'\020\001', file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='TN', full_name='visualdl.Record.PRCurve.TN', index=2, - number=3, type=3, cpp_type=2, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=b'\020\001', file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='FN', full_name='visualdl.Record.PRCurve.FN', index=3, - number=4, type=3, cpp_type=2, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=b'\020\001', file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='precision', full_name='visualdl.Record.PRCurve.precision', index=4, - number=5, type=1, cpp_type=5, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='recall', full_name='visualdl.Record.PRCurve.recall', index=5, - number=6, type=1, cpp_type=5, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=530, - serialized_end=638, -) -_RECORD_ROC_CURVE = _descriptor.Descriptor( - name='ROC_Curve', - full_name='visualdl.Record.ROC_Curve', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='TP', full_name='visualdl.Record.ROC_Curve.TP', index=0, - number=1, type=3, cpp_type=2, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=b'\020\001', file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='FP', full_name='visualdl.Record.ROC_Curve.FP', index=1, - number=2, type=3, cpp_type=2, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=b'\020\001', file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='TN', full_name='visualdl.Record.ROC_Curve.TN', index=2, - number=3, type=3, cpp_type=2, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=b'\020\001', file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='FN', full_name='visualdl.Record.ROC_Curve.FN', index=3, - number=4, type=3, cpp_type=2, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=b'\020\001', file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='tpr', full_name='visualdl.Record.ROC_Curve.tpr', index=4, - number=5, type=1, cpp_type=5, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='fpr', full_name='visualdl.Record.ROC_Curve.fpr', index=5, - number=6, type=1, cpp_type=5, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=640, - serialized_end=741, -) - -_RECORD_HPARAM_HPARAMINFO = _descriptor.Descriptor( - name='HparamInfo', - full_name='visualdl.Record.HParam.HparamInfo', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='int_value', full_name='visualdl.Record.HParam.HparamInfo.int_value', index=0, - number=1, type=3, cpp_type=2, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='float_value', full_name='visualdl.Record.HParam.HparamInfo.float_value', index=1, - number=2, type=1, cpp_type=5, label=1, - has_default_value=False, default_value=float(0), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='string_value', full_name='visualdl.Record.HParam.HparamInfo.string_value', index=2, - number=3, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='name', full_name='visualdl.Record.HParam.HparamInfo.name', index=3, - number=4, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - _descriptor.OneofDescriptor( - name='type', full_name='visualdl.Record.HParam.HparamInfo.type', - index=0, containing_type=None, fields=[]), - ], - serialized_start=882, - serialized_end=984, -) - -_RECORD_HPARAM = _descriptor.Descriptor( - name='HParam', - full_name='visualdl.Record.HParam', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='hparamInfos', full_name='visualdl.Record.HParam.hparamInfos', index=0, - number=1, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='metricInfos', full_name='visualdl.Record.HParam.metricInfos', index=1, - number=2, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='name', full_name='visualdl.Record.HParam.name', index=2, - number=3, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[_RECORD_HPARAM_HPARAMINFO, ], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=744, - serialized_end=984, -) - -_RECORD_METADATA = _descriptor.Descriptor( - name='MetaData', - full_name='visualdl.Record.MetaData', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='display_name', full_name='visualdl.Record.MetaData.display_name', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=986, - serialized_end=1018, -) - -_RECORD_VALUE = _descriptor.Descriptor( - name='Value', - full_name='visualdl.Record.Value', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='id', full_name='visualdl.Record.Value.id', index=0, - number=1, type=3, cpp_type=2, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='tag', full_name='visualdl.Record.Value.tag', index=1, - number=2, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='timestamp', full_name='visualdl.Record.Value.timestamp', index=2, - number=3, type=3, cpp_type=2, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='value', full_name='visualdl.Record.Value.value', index=3, - number=4, type=2, cpp_type=6, label=1, - has_default_value=False, default_value=float(0), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='image', full_name='visualdl.Record.Value.image', index=4, - number=5, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='audio', full_name='visualdl.Record.Value.audio', index=5, - number=6, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='embeddings', full_name='visualdl.Record.Value.embeddings', index=6, - number=7, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='histogram', full_name='visualdl.Record.Value.histogram', index=7, - number=8, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='pr_curve', full_name='visualdl.Record.Value.pr_curve', index=8, - number=9, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='meta_data', full_name='visualdl.Record.Value.meta_data', index=9, - number=10, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='roc_curve', full_name='visualdl.Record.Value.roc_curve', index=10, - number=11, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='text', full_name='visualdl.Record.Value.text', index=11, - number=12, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='hparam', full_name='visualdl.Record.Value.hparam', index=12, - number=13, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - _descriptor.OneofDescriptor( - name='one_value', full_name='visualdl.Record.Value.one_value', - index=0, containing_type=None, fields=[]), - ], - serialized_start=1021, - serialized_end=1509, -) - -_RECORD = _descriptor.Descriptor( - name='Record', - full_name='visualdl.Record', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='values', full_name='visualdl.Record.values', index=0, - number=1, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[_RECORD_IMAGE, _RECORD_TEXT, _RECORD_AUDIO, _RECORD_EMBEDDING, _RECORD_EMBEDDINGS, _RECORD_BYTES_EMBEDDINGS, _RECORD_HISTOGRAM, _RECORD_PRCURVE, _RECORD_ROC_CURVE, _RECORD_HPARAM, _RECORD_METADATA, _RECORD_VALUE, ], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=27, - serialized_end=1509, -) - -_RECORD_IMAGE.containing_type = _RECORD -_RECORD_TEXT.containing_type = _RECORD -_RECORD_AUDIO.containing_type = _RECORD -_RECORD_EMBEDDING.containing_type = _RECORD -_RECORD_EMBEDDINGS.fields_by_name['embeddings'].message_type = _RECORD_EMBEDDING -_RECORD_EMBEDDINGS.containing_type = _RECORD -_RECORD_BYTES_EMBEDDINGS.containing_type = _RECORD -_RECORD_HISTOGRAM.containing_type = _RECORD -_RECORD_PRCURVE.containing_type = _RECORD -_RECORD_ROC_CURVE.containing_type = _RECORD -_RECORD_HPARAM_HPARAMINFO.containing_type = _RECORD_HPARAM -_RECORD_HPARAM_HPARAMINFO.oneofs_by_name['type'].fields.append( - _RECORD_HPARAM_HPARAMINFO.fields_by_name['int_value']) -_RECORD_HPARAM_HPARAMINFO.fields_by_name['int_value'].containing_oneof = _RECORD_HPARAM_HPARAMINFO.oneofs_by_name['type'] -_RECORD_HPARAM_HPARAMINFO.oneofs_by_name['type'].fields.append( - _RECORD_HPARAM_HPARAMINFO.fields_by_name['float_value']) -_RECORD_HPARAM_HPARAMINFO.fields_by_name['float_value'].containing_oneof = _RECORD_HPARAM_HPARAMINFO.oneofs_by_name['type'] -_RECORD_HPARAM_HPARAMINFO.oneofs_by_name['type'].fields.append( - _RECORD_HPARAM_HPARAMINFO.fields_by_name['string_value']) -_RECORD_HPARAM_HPARAMINFO.fields_by_name['string_value'].containing_oneof = _RECORD_HPARAM_HPARAMINFO.oneofs_by_name['type'] -_RECORD_HPARAM.fields_by_name['hparamInfos'].message_type = _RECORD_HPARAM_HPARAMINFO -_RECORD_HPARAM.fields_by_name['metricInfos'].message_type = _RECORD_HPARAM_HPARAMINFO -_RECORD_HPARAM.containing_type = _RECORD -_RECORD_METADATA.containing_type = _RECORD -_RECORD_VALUE.fields_by_name['image'].message_type = _RECORD_IMAGE -_RECORD_VALUE.fields_by_name['audio'].message_type = _RECORD_AUDIO -_RECORD_VALUE.fields_by_name['embeddings'].message_type = _RECORD_EMBEDDINGS -_RECORD_VALUE.fields_by_name['histogram'].message_type = _RECORD_HISTOGRAM -_RECORD_VALUE.fields_by_name['pr_curve'].message_type = _RECORD_PRCURVE -_RECORD_VALUE.fields_by_name['meta_data'].message_type = _RECORD_METADATA -_RECORD_VALUE.fields_by_name['roc_curve'].message_type = _RECORD_ROC_CURVE -_RECORD_VALUE.fields_by_name['text'].message_type = _RECORD_TEXT -_RECORD_VALUE.fields_by_name['hparam'].message_type = _RECORD_HPARAM -_RECORD_VALUE.containing_type = _RECORD -_RECORD_VALUE.oneofs_by_name['one_value'].fields.append( - _RECORD_VALUE.fields_by_name['value']) -_RECORD_VALUE.fields_by_name['value'].containing_oneof = _RECORD_VALUE.oneofs_by_name['one_value'] -_RECORD_VALUE.oneofs_by_name['one_value'].fields.append( - _RECORD_VALUE.fields_by_name['image']) -_RECORD_VALUE.fields_by_name['image'].containing_oneof = _RECORD_VALUE.oneofs_by_name['one_value'] -_RECORD_VALUE.oneofs_by_name['one_value'].fields.append( - _RECORD_VALUE.fields_by_name['audio']) -_RECORD_VALUE.fields_by_name['audio'].containing_oneof = _RECORD_VALUE.oneofs_by_name['one_value'] -_RECORD_VALUE.oneofs_by_name['one_value'].fields.append( - _RECORD_VALUE.fields_by_name['embeddings']) -_RECORD_VALUE.fields_by_name['embeddings'].containing_oneof = _RECORD_VALUE.oneofs_by_name['one_value'] -_RECORD_VALUE.oneofs_by_name['one_value'].fields.append( - _RECORD_VALUE.fields_by_name['histogram']) -_RECORD_VALUE.fields_by_name['histogram'].containing_oneof = _RECORD_VALUE.oneofs_by_name['one_value'] -_RECORD_VALUE.oneofs_by_name['one_value'].fields.append( - _RECORD_VALUE.fields_by_name['pr_curve']) -_RECORD_VALUE.fields_by_name['pr_curve'].containing_oneof = _RECORD_VALUE.oneofs_by_name['one_value'] -_RECORD_VALUE.oneofs_by_name['one_value'].fields.append( - _RECORD_VALUE.fields_by_name['meta_data']) -_RECORD_VALUE.fields_by_name['meta_data'].containing_oneof = _RECORD_VALUE.oneofs_by_name['one_value'] -_RECORD_VALUE.oneofs_by_name['one_value'].fields.append( - _RECORD_VALUE.fields_by_name['roc_curve']) -_RECORD_VALUE.fields_by_name['roc_curve'].containing_oneof = _RECORD_VALUE.oneofs_by_name['one_value'] -_RECORD_VALUE.oneofs_by_name['one_value'].fields.append( - _RECORD_VALUE.fields_by_name['text']) -_RECORD_VALUE.fields_by_name['text'].containing_oneof = _RECORD_VALUE.oneofs_by_name['one_value'] -_RECORD_VALUE.oneofs_by_name['one_value'].fields.append( - _RECORD_VALUE.fields_by_name['hparam']) -_RECORD_VALUE.fields_by_name['hparam'].containing_oneof = _RECORD_VALUE.oneofs_by_name['one_value'] -_RECORD.fields_by_name['values'].message_type = _RECORD_VALUE -DESCRIPTOR.message_types_by_name['Record'] = _RECORD -_sym_db.RegisterFileDescriptor(DESCRIPTOR) +_RECORD = DESCRIPTOR.message_types_by_name['Record'] +_RECORD_IMAGE = _RECORD.nested_types_by_name['Image'] +_RECORD_TEXT = _RECORD.nested_types_by_name['Text'] +_RECORD_AUDIO = _RECORD.nested_types_by_name['Audio'] +_RECORD_EMBEDDING = _RECORD.nested_types_by_name['Embedding'] +_RECORD_EMBEDDINGS = _RECORD.nested_types_by_name['Embeddings'] +_RECORD_BYTES_EMBEDDINGS = _RECORD.nested_types_by_name['bytes_embeddings'] +_RECORD_HISTOGRAM = _RECORD.nested_types_by_name['Histogram'] +_RECORD_PRCURVE = _RECORD.nested_types_by_name['PRCurve'] +_RECORD_ROC_CURVE = _RECORD.nested_types_by_name['ROC_Curve'] +_RECORD_HPARAM = _RECORD.nested_types_by_name['HParam'] +_RECORD_HPARAM_HPARAMINFO = _RECORD_HPARAM.nested_types_by_name['HparamInfo'] +_RECORD_METADATA = _RECORD.nested_types_by_name['MetaData'] +_RECORD_TAGVALUE = _RECORD.nested_types_by_name['TagValue'] +_RECORD_VALUE = _RECORD.nested_types_by_name['Value'] Record = _reflection.GeneratedProtocolMessageType('Record', (_message.Message,), { 'Image' : _reflection.GeneratedProtocolMessageType('Image', (_message.Message,), { @@ -849,6 +119,13 @@ Record = _reflection.GeneratedProtocolMessageType('Record', (_message.Message,), }) , + 'TagValue' : _reflection.GeneratedProtocolMessageType('TagValue', (_message.Message,), { + 'DESCRIPTOR' : _RECORD_TAGVALUE, + '__module__' : 'record_pb2' + # @@protoc_insertion_point(class_scope:visualdl.Record.TagValue) + }) + , + 'Value' : _reflection.GeneratedProtocolMessageType('Value', (_message.Message,), { 'DESCRIPTOR' : _RECORD_VALUE, '__module__' : 'record_pb2' @@ -872,17 +149,60 @@ _sym_db.RegisterMessage(Record.ROC_Curve) _sym_db.RegisterMessage(Record.HParam) _sym_db.RegisterMessage(Record.HParam.HparamInfo) _sym_db.RegisterMessage(Record.MetaData) +_sym_db.RegisterMessage(Record.TagValue) _sym_db.RegisterMessage(Record.Value) - -_RECORD_HISTOGRAM.fields_by_name['hist']._options = None -_RECORD_HISTOGRAM.fields_by_name['bin_edges']._options = None -_RECORD_PRCURVE.fields_by_name['TP']._options = None -_RECORD_PRCURVE.fields_by_name['FP']._options = None -_RECORD_PRCURVE.fields_by_name['TN']._options = None -_RECORD_PRCURVE.fields_by_name['FN']._options = None -_RECORD_ROC_CURVE.fields_by_name['TP']._options = None -_RECORD_ROC_CURVE.fields_by_name['FP']._options = None -_RECORD_ROC_CURVE.fields_by_name['TN']._options = None -_RECORD_ROC_CURVE.fields_by_name['FN']._options = None +if _descriptor._USE_C_DESCRIPTORS == False: + + DESCRIPTOR._options = None + _RECORD_HISTOGRAM.fields_by_name['hist']._options = None + _RECORD_HISTOGRAM.fields_by_name['hist']._serialized_options = b'\020\001' + _RECORD_HISTOGRAM.fields_by_name['bin_edges']._options = None + _RECORD_HISTOGRAM.fields_by_name['bin_edges']._serialized_options = b'\020\001' + _RECORD_PRCURVE.fields_by_name['TP']._options = None + _RECORD_PRCURVE.fields_by_name['TP']._serialized_options = b'\020\001' + _RECORD_PRCURVE.fields_by_name['FP']._options = None + _RECORD_PRCURVE.fields_by_name['FP']._serialized_options = b'\020\001' + _RECORD_PRCURVE.fields_by_name['TN']._options = None + _RECORD_PRCURVE.fields_by_name['TN']._serialized_options = b'\020\001' + _RECORD_PRCURVE.fields_by_name['FN']._options = None + _RECORD_PRCURVE.fields_by_name['FN']._serialized_options = b'\020\001' + _RECORD_ROC_CURVE.fields_by_name['TP']._options = None + _RECORD_ROC_CURVE.fields_by_name['TP']._serialized_options = b'\020\001' + _RECORD_ROC_CURVE.fields_by_name['FP']._options = None + _RECORD_ROC_CURVE.fields_by_name['FP']._serialized_options = b'\020\001' + _RECORD_ROC_CURVE.fields_by_name['TN']._options = None + _RECORD_ROC_CURVE.fields_by_name['TN']._serialized_options = b'\020\001' + _RECORD_ROC_CURVE.fields_by_name['FN']._options = None + _RECORD_ROC_CURVE.fields_by_name['FN']._serialized_options = b'\020\001' + _RECORD._serialized_start=27 + _RECORD._serialized_end=1597 + _RECORD_IMAGE._serialized_start=77 + _RECORD_IMAGE._serialized_end=114 + _RECORD_TEXT._serialized_start=116 + _RECORD_TEXT._serialized_end=151 + _RECORD_AUDIO._serialized_start=153 + _RECORD_AUDIO._serialized_end=278 + _RECORD_EMBEDDING._serialized_start=280 + _RECORD_EMBEDDING._serialized_end=323 + _RECORD_EMBEDDINGS._serialized_start=325 + _RECORD_EMBEDDINGS._serialized_end=405 + _RECORD_BYTES_EMBEDDINGS._serialized_start=407 + _RECORD_BYTES_EMBEDDINGS._serialized_end=474 + _RECORD_HISTOGRAM._serialized_start=476 + _RECORD_HISTOGRAM._serialized_end=528 + _RECORD_PRCURVE._serialized_start=530 + _RECORD_PRCURVE._serialized_end=638 + _RECORD_ROC_CURVE._serialized_start=640 + _RECORD_ROC_CURVE._serialized_end=741 + _RECORD_HPARAM._serialized_start=744 + _RECORD_HPARAM._serialized_end=984 + _RECORD_HPARAM_HPARAMINFO._serialized_start=882 + _RECORD_HPARAM_HPARAMINFO._serialized_end=984 + _RECORD_METADATA._serialized_start=986 + _RECORD_METADATA._serialized_end=1018 + _RECORD_TAGVALUE._serialized_start=1020 + _RECORD_TAGVALUE._serialized_end=1058 + _RECORD_VALUE._serialized_start=1061 + _RECORD_VALUE._serialized_end=1597 # @@protoc_insertion_point(module_scope) diff --git a/visualdl/reader/reader.py b/visualdl/reader/reader.py index 081df075..67dc2032 100644 --- a/visualdl/reader/reader.py +++ b/visualdl/reader/reader.py @@ -123,6 +123,13 @@ class LogReader(object): for item in proto_datas: data.append([item.id, item.tag, item.timestamp, item.value]) return data + elif 'scalars' == component: + for item in proto_datas: + data.append([ + item.id, item.tag, item.tag_value.tag, item.timestamp, + item.tag_value.value + ]) + return data return proto_datas def _get_log_tags(self): @@ -143,6 +150,8 @@ class LogReader(object): else: file_path = bfile.join(run, self.walks[run]) reader = self._get_file_reader(file_path=file_path, update=False) + reader.dir = run + self.reader = reader remain = self.get_remain(reader=reader) data = self.read_log_data( remain=remain, update=False)[component][tag] @@ -191,6 +200,8 @@ class LogReader(object): component = "text" elif "hparam" == value_type: component = "hyper_parameters" + elif "tag_value" == value_type: + component = "scalars" else: raise TypeError("Invalid value type `%s`." % value_type) self._tags[path] = component @@ -265,7 +276,6 @@ class LogReader(object): if update: self.register_reader(file_path) self.reader = self.readers[file_path] - self.reader.dir = file_path return self.reader else: reader = RecordReader(filepath=file_path) @@ -275,6 +285,7 @@ class LogReader(object): if update: if path not in list(self.readers.keys()): reader = RecordReader(filepath=path, dir=dir) + reader.dir = dir self.readers[path] = reader else: pass diff --git a/visualdl/server/api.py b/visualdl/server/api.py index 2ea87d52..502bf48f 100644 --- a/visualdl/server/api.py +++ b/visualdl/server/api.py @@ -135,6 +135,11 @@ class Api(object): return self._get_with_retry('data/plugin/scalars/tags', lib.get_scalar_tags) + @result() + def scalars_tags(self): + return self._get_with_retry('data/plugin/multiscalars/tags', + lib.get_scalars_tags) + @result() def image_tags(self): return self._get_with_retry('data/plugin/images/tags', @@ -194,11 +199,24 @@ class Api(object): key = os.path.join('data/plugin/scalars/scalars', run, tag) return self._get_with_retry(key, lib.get_scalar, run, tag) + @result() + def scalars_list(self, run, tag, sub_tag): + key = os.path.join('data/plugin/multiscalars/scalars', run, tag, + sub_tag) + return self._get_with_retry(key, lib.get_scalars, run, tag, sub_tag) + @result('text/csv') def scalar_data(self, run, tag, type='tsv'): key = os.path.join('data/plugin/scalars/data', run, tag, type) return self._get_with_retry(key, lib.get_scalar_data, run, tag, type) + @result('text/csv') + def scalars_data(self, run, tag, sub_tag, type='tsv'): + key = os.path.join('data/plugin/multiscalars/data', run, tag, sub_tag, + type) + return self._get_with_retry(key, lib.get_scalars_data, run, tag, + sub_tag, type) + @result() def image_list(self, mode, tag): key = os.path.join('data/plugin/images/images', mode, tag) @@ -412,6 +430,7 @@ def create_api_call(logdir, model, cache_timeout): 'tags': (api.tags, []), 'logs': (api.logs, []), 'scalar/tags': (api.scalar_tags, []), + 'scalars/tags': (api.scalars_tags, []), 'image/tags': (api.image_tags, []), 'text/tags': (api.text_tags, []), 'audio/tags': (api.audio_tags, []), @@ -420,7 +439,9 @@ def create_api_call(logdir, model, cache_timeout): 'pr-curve/tags': (api.pr_curve_tags, []), 'roc-curve/tags': (api.roc_curve_tags, []), 'scalar/list': (api.scalar_list, ['run', 'tag']), + 'scalars/list': (api.scalars_list, ['run', 'tag', 'sub_tag']), 'scalar/data': (api.scalar_data, ['run', 'tag', 'type']), + 'scalars/data': (api.scalars_data, ['run', 'tag', 'sub_tag', 'type']), 'image/list': (api.image_list, ['run', 'tag']), 'image/image': (api.image_image, ['run', 'tag', 'index']), 'text/list': (api.text_list, ['run', 'tag']), diff --git a/visualdl/server/data_manager.py b/visualdl/server/data_manager.py index 74bb671e..87599c78 100644 --- a/visualdl/server/data_manager.py +++ b/visualdl/server/data_manager.py @@ -12,10 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. # ======================================================================= - -import threading -import random import collections +import random +import threading DEFAULT_PLUGIN_MAXSIZE = { "scalar": 1000, @@ -31,6 +30,10 @@ DEFAULT_PLUGIN_MAXSIZE = { } +def add_sub_tag(tag, sub_tag): + return tag.replace('%', '_') + '_' + sub_tag + + class Reservoir(object): """A map-to-arrays dict, with deterministic Reservoir Sampling. @@ -53,10 +56,8 @@ class Reservoir(object): if max_size < 0 or max_size != round(max_size): raise ValueError("Max_size must be nonnegative integer.") self._max_size = max_size - self._buckets = collections.defaultdict( - lambda: _ReservoirBucket(max_size=self._max_size, - random_instance=random.Random(seed)) - ) + self._buckets = collections.defaultdict(lambda: _ReservoirBucket( + max_size=self._max_size, random_instance=random.Random(seed))) self._mutex = threading.Lock() @property @@ -174,6 +175,25 @@ class Reservoir(object): with self._mutex: self._buckets[key].add_scalar_item(item) + def _add_scalars_item(self, key, item): + """Add a new scalar item to reservoir buckets with given tag as key. + + If bucket with key has not yet reached full size, each item will be + added. + + If bucket with key is full, each item will be added with same + probability. + + Add new item to buckets will always valid because self._buckets is a + collection.defaultdict. + + Args: + key: Tag of one bucket to add new item. + item: New item to add to bucket. + """ + with self._mutex: + self._buckets[key].add_scalars_item(item) + def add_item(self, run, tag, item): """Add a new item to reservoir buckets with given tag as key. @@ -197,8 +217,14 @@ class Reservoir(object): tag: Identity of one record in tablet. item: New item to add to bucket. """ - key = run + "/" + tag - self._add_scalar_item(key, item) + if item.WhichOneof("one_value") == "value": + key = run + "/" + tag + self._add_scalar_item(key, item) + elif item.WhichOneof("one_value") == "tag_value": + key = run + "/" + add_sub_tag(tag, item.tag_value.tag) + "/" + tag + self._add_scalars_item(key, item) + else: + raise ValueError("Not scalar type:" + item.WhichOneof("one_value")) def _cut_tail(self, key): with self._mutex: @@ -247,6 +273,9 @@ class _ReservoirBucket(object): self.max_scalar = None self.min_scalar = None + # improve performance when data is monotonous + self._last_special = False + def add_item(self, item): """ Add an item to bucket, replacing an old item with probability. @@ -274,24 +303,101 @@ class _ReservoirBucket(object): Use reservoir sampling to add a new item to sampling bucket, each item in a steam has same probability stay in the bucket. + use _last_special mark to improve performance when data is monotonous + Args: item: The item to add to reservoir bucket. """ with self._mutex: + # save max and min value if not self.max_scalar or self.max_scalar.value < item.value: self.max_scalar = item if not self.min_scalar or self.min_scalar.value > item.value: self.min_scalar = item if len(self._items) < self._max_size or self._max_size == 0: + # capacity is valid, append directly + self._items.append(item) + else: + if self._last_special: + if self._items[-1].id == self.min_scalar.id or self._items[ + -1].id == self.max_scalar.id: + # data is not monotonous, set special to False + self._last_special = False + else: + # data is monotonous, drop last item by reservoir algorithm + r = self._random.randint(1, self._num_items_index) + if r >= self._max_size: + self._items.pop(-1) + self._items.append(item) + self._num_items_index += 1 + return + if item.id == self.min_scalar.id or item.id == self.max_scalar.id: + # this item is max or min, should be reserved + r = self._random.randint(1, self._max_size - 1) + self._last_special = True + else: + # drop by reservoir algorithm + r = self._random.randint(1, self._num_items_index) + self._last_special = False + if r < self._max_size: + if self._items[r].id == self.min_scalar.id or self._items[ + r].id == self.max_scalar.id: + # reserve max and min point + if r - 1 > 0: + r = r - 1 + elif r + 1 < self._max_size: + r = r + 1 + self._items.pop(r) + self._items.append(item) + + self._num_items_index += 1 + + def add_scalars_item(self, item): + """ Add an scalar item to bucket, replacing an old item with probability. + + Use reservoir sampling to add a new item to sampling bucket, + each item in a steam has same probability stay in the bucket. + + Args: + item: The item to add to reservoir bucket. + """ + with self._mutex: + # save max and min value + if not self.max_scalar or self.max_scalar.tag_value.value < item.tag_value.value: + self.max_scalar = item + if not self.min_scalar or self.min_scalar.tag_value.value > item.tag_value.value: + self.min_scalar = item + + if len(self._items) < self._max_size or self._max_size == 0: + # capacity is valid, append directly self._items.append(item) else: + if self._last_special: + if self._items[-1].id == self.min_scalar.id or self._items[ + -1].id == self.max_scalar.id: + # data is not monotic, set special to False + self._last_special = False + else: + # data is monotic, drop last item by reservoir algorithm + r = self._random.randint(1, self._num_items_index) + if r >= self._max_size: + self._items.pop(-1) + self._items.append(item) + self._num_items_index += 1 + return if item.id == self.min_scalar.id or item.id == self.max_scalar.id: + # this item is max or min, should be reserved r = self._random.randint(1, self._max_size - 1) + self._last_special = True else: + # drop by reservoir algorithm r = self._random.randint(1, self._num_items_index) + self._last_special = False if r < self._max_size: - if self._items[r].id == self.min_scalar.id or self._items[r].id == self.max_scalar.id: + if self._items[r].id == self.min_scalar.id or self._items[ + r].id == self.max_scalar.id: + # reserve max and min point if r - 1 > 0: r = r - 1 elif r + 1 < self._max_size: @@ -405,8 +511,8 @@ class DataManager(object): item: The item to add to reservoir bucket. """ with self._mutex: - if 'scalar' == plugin: - self._reservoirs[plugin].add_scalar_item(run, tag, item) + if 'scalar' == plugin or 'scalars' == plugin: # We adapt scalars data to be saved in scalar reservoir. + self._reservoirs['scalar'].add_scalar_item(run, tag, item) else: self._reservoirs[plugin].add_item(run, tag, item) diff --git a/visualdl/server/lib.py b/visualdl/server/lib.py index b01f8a4c..a4029221 100644 --- a/visualdl/server/lib.py +++ b/visualdl/server/lib.py @@ -335,21 +335,38 @@ def get_scalar(log_reader, run, tag): log_reader.load_new_data() records = log_reader.data_manager.get_reservoir("scalar").get_items( run, decode_tag(tag)) + results = [[ s2ms(item.timestamp), item.id, transfer_abnomal_scalar_value(item.value) + ] if item.WhichOneof("one_value") == "value" else [ + s2ms(item.timestamp), item.id, + transfer_abnomal_scalar_value(item.tag_value.value) ] for item in records] + return results def get_scalar_data(log_reader, run, tag, type='tsv'): + is_scalars = False + if os.path.basename(run).startswith(decode_tag(tag).replace('%', '_')) and \ + log_reader.tags().get(bfile.join(os.path.dirname(run), decode_tag(tag)), None) == 'scalars': + run = os.path.dirname(run) + is_scalars = True run = log_reader.name2tags[run] if run in log_reader.name2tags else run log_reader.load_new_data() - result = log_reader.get_log_data('scalar', run, decode_tag(tag)) + if is_scalars: + result = log_reader.get_log_data('scalars', run, decode_tag(tag)) + else: + result = log_reader.get_log_data('scalar', run, decode_tag(tag)) + print('scalar', result, 'run', run, 'tag', decode_tag(tag)) delimeter = '\t' if 'tsv' == type else ',' with io.StringIO() as fp: csv_writer = csv.writer(fp, delimiter=delimeter) - csv_writer.writerow(['id', 'tag', 'timestamp', 'value']) + if is_scalars: + csv_writer.writerow(['id', 'tag', 'sub_tag', 'timestamp', 'value']) + else: + csv_writer.writerow(['id', 'tag', 'timestamp', 'value']) csv_writer.writerows(result) result = fp.getvalue() return result diff --git a/visualdl/writer/writer.py b/visualdl/writer/writer.py index 77265ad2..0501b8bc 100644 --- a/visualdl/writer/writer.py +++ b/visualdl/writer/writer.py @@ -26,6 +26,7 @@ from visualdl.component.base_component import meta_data from visualdl.component.base_component import pr_curve from visualdl.component.base_component import roc_curve from visualdl.component.base_component import scalar +from visualdl.component.base_component import scalars from visualdl.component.base_component import text from visualdl.component.graph import translate_graph from visualdl.io import bfile @@ -187,6 +188,30 @@ class LogWriter(object): self._get_file_writer().add_record( scalar(tag=tag, value=value, step=step, walltime=walltime)) + def add_scalars(self, main_tag, tag_scalar_dict, step, walltime=None): + """Add a group of scalars to vdl record file. + + Args: + main_tag (string): Data identifier + tag_scalar_dict (float): A dict to provide multi-values with tags + step (int): Step of scalar + walltime (int): Wall time of scalar + + Example: + import math + for index in range(1, 101): + alpha = index*2*math.pi/100 + tval = {'sin':math.sin(alpha), 'cos':math.cos(alpha)} + writer.add_scalars(tag="sin_and_cos", tag_value=tval, step=index) + """ + if '%' in main_tag: + raise RuntimeError("% can't appear in tag!") + if not isinstance(tag_scalar_dict, dict): + raise RuntimeError("tag_value must be a dict!") + walltime = round(time.time() * 1000) if walltime is None else walltime + for record in scalars(main_tag, tag_scalar_dict, step, walltime): + self._get_file_writer().add_record(record) + def add_image(self, tag, img, step, walltime=None, dataformats="HWC"): """Add an image to vdl record file. -- GitLab