diff --git a/mindinsight/datavisual/proto_files/mindinsight_lineage.proto b/mindinsight/datavisual/proto_files/mindinsight_lineage.proto new file mode 100644 index 0000000000000000000000000000000000000000..791f99e12e334990fe90fe60d6701cc868391e13 --- /dev/null +++ b/mindinsight/datavisual/proto_files/mindinsight_lineage.proto @@ -0,0 +1,129 @@ +// Copyright 2020 Huawei Technologies Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto2"; + +package mindinsight; +option cc_enable_arenas = true; + + +// Event Protocol buffer, Top define +message LineageEvent { + // Timestamp + required double wall_time = 1; + + // The step of train. + optional int64 step = 2; + + oneof what { + // An event file was started, with the specified version. + // Now version is "Mindspore.Event:1" + string version = 3; + + // Train lineage + TrainLineage train_lineage = 4; + + // Evaluation lineage + EvaluationLineage evaluation_lineage = 5; + + // Dataset graph + DatasetGraph dataset_graph = 6; + + // User defined info + UserDefinedInfo user_defined_info = 7; + } +} + +// User defined info +message UserDefinedInfo{ + // repeated user defined info + repeated UserDefinedInfo user_info = 1; + + // key/value which contains both scalar and dict + map map_dict = 2; + map map_int32 = 3; + map map_str = 4; + map map_double = 5; +} + +// TrainLineage records infos of a train. +message TrainLineage{ + message HyperParameters{ + optional string optimizer = 1; + optional float learning_rate = 2; + optional string loss_function = 3; + optional int32 epoch = 4; + optional string parallel_mode = 5; + optional int32 device_num = 6; + optional int32 batch_size = 8; + } + + message TrainDataset{ + optional string train_dataset_path = 1; + optional int32 train_dataset_size = 2; + } + + message Algorithm{ + optional string network = 1; + optional float loss = 2; + } + + message Model{ + optional string path = 3; + optional int64 size = 4; + } + + optional HyperParameters hyper_parameters = 1; + optional TrainDataset train_dataset = 2; + optional Algorithm algorithm = 3; + optional Model model = 4; +} + +//EvalLineage records infos of evaluation. +message EvaluationLineage{ + message ValidDataset{ + optional string valid_dataset_path = 1; + optional int32 valid_dataset_size = 2; + } + + optional string metric = 2; + optional ValidDataset valid_dataset = 3; +} + + +// DatasetGraph +message DatasetGraph { + repeated DatasetGraph children = 1; + optional OperationParameter parameter = 2; + repeated Operation operations = 3; + optional Operation sampler = 4; +} + +message Operation { + optional OperationParameter operationParam = 1; + repeated int32 size = 2; + repeated float weights = 3; +} + +message OperationParameter{ + map mapStr = 1; + map mapStrList = 2; + map mapBool = 3; + map mapInt = 4; + map mapDouble = 5; +} + +message StrList { + repeated string strValue = 1; +} diff --git a/mindinsight/datavisual/proto_files/mindinsight_lineage_pb2.py b/mindinsight/datavisual/proto_files/mindinsight_lineage_pb2.py new file mode 100644 index 0000000000000000000000000000000000000000..f3075ff8e87ddac2abe507d97799ee78e5331827 --- /dev/null +++ b/mindinsight/datavisual/proto_files/mindinsight_lineage_pb2.py @@ -0,0 +1,1246 @@ +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: mindinsight_lineage.proto + +import sys +_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) +from google.protobuf import descriptor as _descriptor +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection +from google.protobuf import symbol_database as _symbol_database +from google.protobuf import descriptor_pb2 +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + + + +DESCRIPTOR = _descriptor.FileDescriptor( + name='mindinsight_lineage.proto', + package='mindinsight', + syntax='proto2', + serialized_pb=_b('\n\x19mindinsight_lineage.proto\x12\x0bmindinsight\"\xab\x02\n\x0cLineageEvent\x12\x11\n\twall_time\x18\x01 \x02(\x01\x12\x0c\n\x04step\x18\x02 \x01(\x03\x12\x11\n\x07version\x18\x03 \x01(\tH\x00\x12\x32\n\rtrain_lineage\x18\x04 \x01(\x0b\x32\x19.mindinsight.TrainLineageH\x00\x12<\n\x12\x65valuation_lineage\x18\x05 \x01(\x0b\x32\x1e.mindinsight.EvaluationLineageH\x00\x12\x32\n\rdataset_graph\x18\x06 \x01(\x0b\x32\x19.mindinsight.DatasetGraphH\x00\x12\x39\n\x11user_defined_info\x18\x07 \x01(\x0b\x32\x1c.mindinsight.UserDefinedInfoH\x00\x42\x06\n\x04what\"\x9a\x04\n\x0fUserDefinedInfo\x12/\n\tuser_info\x18\x01 \x03(\x0b\x32\x1c.mindinsight.UserDefinedInfo\x12;\n\x08map_dict\x18\x02 \x03(\x0b\x32).mindinsight.UserDefinedInfo.MapDictEntry\x12=\n\tmap_int32\x18\x03 \x03(\x0b\x32*.mindinsight.UserDefinedInfo.MapInt32Entry\x12\x39\n\x07map_str\x18\x04 \x03(\x0b\x32(.mindinsight.UserDefinedInfo.MapStrEntry\x12?\n\nmap_double\x18\x05 \x03(\x0b\x32+.mindinsight.UserDefinedInfo.MapDoubleEntry\x1aL\n\x0cMapDictEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12+\n\x05value\x18\x02 \x01(\x0b\x32\x1c.mindinsight.UserDefinedInfo:\x02\x38\x01\x1a/\n\rMapInt32Entry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x05:\x02\x38\x01\x1a-\n\x0bMapStrEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x1a\x30\n\x0eMapDoubleEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x01:\x02\x38\x01\"\xb6\x04\n\x0cTrainLineage\x12\x43\n\x10hyper_parameters\x18\x01 \x01(\x0b\x32).mindinsight.TrainLineage.HyperParameters\x12=\n\rtrain_dataset\x18\x02 \x01(\x0b\x32&.mindinsight.TrainLineage.TrainDataset\x12\x36\n\talgorithm\x18\x03 \x01(\x0b\x32#.mindinsight.TrainLineage.Algorithm\x12.\n\x05model\x18\x04 \x01(\x0b\x32\x1f.mindinsight.TrainLineage.Model\x1a\xa0\x01\n\x0fHyperParameters\x12\x11\n\toptimizer\x18\x01 \x01(\t\x12\x15\n\rlearning_rate\x18\x02 \x01(\x02\x12\x15\n\rloss_function\x18\x03 \x01(\t\x12\r\n\x05\x65poch\x18\x04 \x01(\x05\x12\x15\n\rparallel_mode\x18\x05 \x01(\t\x12\x12\n\ndevice_num\x18\x06 \x01(\x05\x12\x12\n\nbatch_size\x18\x08 \x01(\x05\x1a\x46\n\x0cTrainDataset\x12\x1a\n\x12train_dataset_path\x18\x01 \x01(\t\x12\x1a\n\x12train_dataset_size\x18\x02 \x01(\x05\x1a*\n\tAlgorithm\x12\x0f\n\x07network\x18\x01 \x01(\t\x12\x0c\n\x04loss\x18\x02 \x01(\x02\x1a#\n\x05Model\x12\x0c\n\x04path\x18\x03 \x01(\t\x12\x0c\n\x04size\x18\x04 \x01(\x03\"\xaf\x01\n\x11\x45valuationLineage\x12\x0e\n\x06metric\x18\x02 \x01(\t\x12\x42\n\rvalid_dataset\x18\x03 \x01(\x0b\x32+.mindinsight.EvaluationLineage.ValidDataset\x1a\x46\n\x0cValidDataset\x12\x1a\n\x12valid_dataset_path\x18\x01 \x01(\t\x12\x1a\n\x12valid_dataset_size\x18\x02 \x01(\x05\"\xc4\x01\n\x0c\x44\x61tasetGraph\x12+\n\x08\x63hildren\x18\x01 \x03(\x0b\x32\x19.mindinsight.DatasetGraph\x12\x32\n\tparameter\x18\x02 \x01(\x0b\x32\x1f.mindinsight.OperationParameter\x12*\n\noperations\x18\x03 \x03(\x0b\x32\x16.mindinsight.Operation\x12\'\n\x07sampler\x18\x04 \x01(\x0b\x32\x16.mindinsight.Operation\"c\n\tOperation\x12\x37\n\x0eoperationParam\x18\x01 \x01(\x0b\x32\x1f.mindinsight.OperationParameter\x12\x0c\n\x04size\x18\x02 \x03(\x05\x12\x0f\n\x07weights\x18\x03 \x03(\x02\"\xde\x04\n\x12OperationParameter\x12;\n\x06mapStr\x18\x01 \x03(\x0b\x32+.mindinsight.OperationParameter.MapStrEntry\x12\x43\n\nmapStrList\x18\x02 \x03(\x0b\x32/.mindinsight.OperationParameter.MapStrListEntry\x12=\n\x07mapBool\x18\x03 \x03(\x0b\x32,.mindinsight.OperationParameter.MapBoolEntry\x12;\n\x06mapInt\x18\x04 \x03(\x0b\x32+.mindinsight.OperationParameter.MapIntEntry\x12\x41\n\tmapDouble\x18\x05 \x03(\x0b\x32..mindinsight.OperationParameter.MapDoubleEntry\x1a-\n\x0bMapStrEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x1aG\n\x0fMapStrListEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12#\n\x05value\x18\x02 \x01(\x0b\x32\x14.mindinsight.StrList:\x02\x38\x01\x1a.\n\x0cMapBoolEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x08:\x02\x38\x01\x1a-\n\x0bMapIntEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x05:\x02\x38\x01\x1a\x30\n\x0eMapDoubleEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x01:\x02\x38\x01\"\x1b\n\x07StrList\x12\x10\n\x08strValue\x18\x01 \x03(\tB\x03\xf8\x01\x01') +) +_sym_db.RegisterFileDescriptor(DESCRIPTOR) + + + + +_LINEAGEEVENT = _descriptor.Descriptor( + name='LineageEvent', + full_name='mindinsight.LineageEvent', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='wall_time', full_name='mindinsight.LineageEvent.wall_time', index=0, + number=1, type=1, cpp_type=5, label=2, + has_default_value=False, default_value=float(0), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='step', full_name='mindinsight.LineageEvent.step', index=1, + number=2, type=3, cpp_type=2, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='version', full_name='mindinsight.LineageEvent.version', index=2, + number=3, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='train_lineage', full_name='mindinsight.LineageEvent.train_lineage', index=3, + number=4, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='evaluation_lineage', full_name='mindinsight.LineageEvent.evaluation_lineage', index=4, + number=5, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='dataset_graph', full_name='mindinsight.LineageEvent.dataset_graph', index=5, + number=6, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='user_defined_info', full_name='mindinsight.LineageEvent.user_defined_info', index=6, + number=7, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + syntax='proto2', + extension_ranges=[], + oneofs=[ + _descriptor.OneofDescriptor( + name='what', full_name='mindinsight.LineageEvent.what', + index=0, containing_type=None, fields=[]), + ], + serialized_start=43, + serialized_end=342, +) + + +_USERDEFINEDINFO_MAPDICTENTRY = _descriptor.Descriptor( + name='MapDictEntry', + full_name='mindinsight.UserDefinedInfo.MapDictEntry', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='key', full_name='mindinsight.UserDefinedInfo.MapDictEntry.key', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='value', full_name='mindinsight.UserDefinedInfo.MapDictEntry.value', index=1, + number=2, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=_descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001')), + is_extendable=False, + syntax='proto2', + extension_ranges=[], + oneofs=[ + ], + serialized_start=661, + serialized_end=737, +) + +_USERDEFINEDINFO_MAPINT32ENTRY = _descriptor.Descriptor( + name='MapInt32Entry', + full_name='mindinsight.UserDefinedInfo.MapInt32Entry', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='key', full_name='mindinsight.UserDefinedInfo.MapInt32Entry.key', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='value', full_name='mindinsight.UserDefinedInfo.MapInt32Entry.value', index=1, + number=2, type=5, cpp_type=1, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=_descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001')), + is_extendable=False, + syntax='proto2', + extension_ranges=[], + oneofs=[ + ], + serialized_start=739, + serialized_end=786, +) + +_USERDEFINEDINFO_MAPSTRENTRY = _descriptor.Descriptor( + name='MapStrEntry', + full_name='mindinsight.UserDefinedInfo.MapStrEntry', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='key', full_name='mindinsight.UserDefinedInfo.MapStrEntry.key', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='value', full_name='mindinsight.UserDefinedInfo.MapStrEntry.value', index=1, + number=2, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=_descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001')), + is_extendable=False, + syntax='proto2', + extension_ranges=[], + oneofs=[ + ], + serialized_start=788, + serialized_end=833, +) + +_USERDEFINEDINFO_MAPDOUBLEENTRY = _descriptor.Descriptor( + name='MapDoubleEntry', + full_name='mindinsight.UserDefinedInfo.MapDoubleEntry', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='key', full_name='mindinsight.UserDefinedInfo.MapDoubleEntry.key', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='value', full_name='mindinsight.UserDefinedInfo.MapDoubleEntry.value', index=1, + number=2, type=1, cpp_type=5, label=1, + has_default_value=False, default_value=float(0), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=_descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001')), + is_extendable=False, + syntax='proto2', + extension_ranges=[], + oneofs=[ + ], + serialized_start=835, + serialized_end=883, +) + +_USERDEFINEDINFO = _descriptor.Descriptor( + name='UserDefinedInfo', + full_name='mindinsight.UserDefinedInfo', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='user_info', full_name='mindinsight.UserDefinedInfo.user_info', index=0, + number=1, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='map_dict', full_name='mindinsight.UserDefinedInfo.map_dict', index=1, + number=2, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='map_int32', full_name='mindinsight.UserDefinedInfo.map_int32', index=2, + number=3, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='map_str', full_name='mindinsight.UserDefinedInfo.map_str', index=3, + number=4, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='map_double', full_name='mindinsight.UserDefinedInfo.map_double', index=4, + number=5, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[_USERDEFINEDINFO_MAPDICTENTRY, _USERDEFINEDINFO_MAPINT32ENTRY, _USERDEFINEDINFO_MAPSTRENTRY, _USERDEFINEDINFO_MAPDOUBLEENTRY, ], + enum_types=[ + ], + options=None, + is_extendable=False, + syntax='proto2', + extension_ranges=[], + oneofs=[ + ], + serialized_start=345, + serialized_end=883, +) + + +_TRAINLINEAGE_HYPERPARAMETERS = _descriptor.Descriptor( + name='HyperParameters', + full_name='mindinsight.TrainLineage.HyperParameters', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='optimizer', full_name='mindinsight.TrainLineage.HyperParameters.optimizer', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='learning_rate', full_name='mindinsight.TrainLineage.HyperParameters.learning_rate', index=1, + number=2, type=2, cpp_type=6, label=1, + has_default_value=False, default_value=float(0), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='loss_function', full_name='mindinsight.TrainLineage.HyperParameters.loss_function', index=2, + number=3, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='epoch', full_name='mindinsight.TrainLineage.HyperParameters.epoch', index=3, + number=4, type=5, cpp_type=1, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='parallel_mode', full_name='mindinsight.TrainLineage.HyperParameters.parallel_mode', index=4, + number=5, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='device_num', full_name='mindinsight.TrainLineage.HyperParameters.device_num', index=5, + number=6, type=5, cpp_type=1, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='batch_size', full_name='mindinsight.TrainLineage.HyperParameters.batch_size', index=6, + number=8, type=5, cpp_type=1, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + syntax='proto2', + extension_ranges=[], + oneofs=[ + ], + serialized_start=1139, + serialized_end=1299, +) + +_TRAINLINEAGE_TRAINDATASET = _descriptor.Descriptor( + name='TrainDataset', + full_name='mindinsight.TrainLineage.TrainDataset', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='train_dataset_path', full_name='mindinsight.TrainLineage.TrainDataset.train_dataset_path', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='train_dataset_size', full_name='mindinsight.TrainLineage.TrainDataset.train_dataset_size', index=1, + number=2, type=5, cpp_type=1, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + syntax='proto2', + extension_ranges=[], + oneofs=[ + ], + serialized_start=1301, + serialized_end=1371, +) + +_TRAINLINEAGE_ALGORITHM = _descriptor.Descriptor( + name='Algorithm', + full_name='mindinsight.TrainLineage.Algorithm', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='network', full_name='mindinsight.TrainLineage.Algorithm.network', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='loss', full_name='mindinsight.TrainLineage.Algorithm.loss', index=1, + number=2, type=2, cpp_type=6, label=1, + has_default_value=False, default_value=float(0), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + syntax='proto2', + extension_ranges=[], + oneofs=[ + ], + serialized_start=1373, + serialized_end=1415, +) + +_TRAINLINEAGE_MODEL = _descriptor.Descriptor( + name='Model', + full_name='mindinsight.TrainLineage.Model', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='path', full_name='mindinsight.TrainLineage.Model.path', index=0, + number=3, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='size', full_name='mindinsight.TrainLineage.Model.size', index=1, + number=4, type=3, cpp_type=2, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + syntax='proto2', + extension_ranges=[], + oneofs=[ + ], + serialized_start=1417, + serialized_end=1452, +) + +_TRAINLINEAGE = _descriptor.Descriptor( + name='TrainLineage', + full_name='mindinsight.TrainLineage', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='hyper_parameters', full_name='mindinsight.TrainLineage.hyper_parameters', index=0, + number=1, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='train_dataset', full_name='mindinsight.TrainLineage.train_dataset', index=1, + number=2, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='algorithm', full_name='mindinsight.TrainLineage.algorithm', index=2, + number=3, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='model', full_name='mindinsight.TrainLineage.model', index=3, + number=4, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[_TRAINLINEAGE_HYPERPARAMETERS, _TRAINLINEAGE_TRAINDATASET, _TRAINLINEAGE_ALGORITHM, _TRAINLINEAGE_MODEL, ], + enum_types=[ + ], + options=None, + is_extendable=False, + syntax='proto2', + extension_ranges=[], + oneofs=[ + ], + serialized_start=886, + serialized_end=1452, +) + + +_EVALUATIONLINEAGE_VALIDDATASET = _descriptor.Descriptor( + name='ValidDataset', + full_name='mindinsight.EvaluationLineage.ValidDataset', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='valid_dataset_path', full_name='mindinsight.EvaluationLineage.ValidDataset.valid_dataset_path', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='valid_dataset_size', full_name='mindinsight.EvaluationLineage.ValidDataset.valid_dataset_size', index=1, + number=2, type=5, cpp_type=1, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + syntax='proto2', + extension_ranges=[], + oneofs=[ + ], + serialized_start=1560, + serialized_end=1630, +) + +_EVALUATIONLINEAGE = _descriptor.Descriptor( + name='EvaluationLineage', + full_name='mindinsight.EvaluationLineage', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='metric', full_name='mindinsight.EvaluationLineage.metric', index=0, + number=2, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='valid_dataset', full_name='mindinsight.EvaluationLineage.valid_dataset', index=1, + number=3, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[_EVALUATIONLINEAGE_VALIDDATASET, ], + enum_types=[ + ], + options=None, + is_extendable=False, + syntax='proto2', + extension_ranges=[], + oneofs=[ + ], + serialized_start=1455, + serialized_end=1630, +) + + +_DATASETGRAPH = _descriptor.Descriptor( + name='DatasetGraph', + full_name='mindinsight.DatasetGraph', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='children', full_name='mindinsight.DatasetGraph.children', index=0, + number=1, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='parameter', full_name='mindinsight.DatasetGraph.parameter', index=1, + number=2, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='operations', full_name='mindinsight.DatasetGraph.operations', index=2, + number=3, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='sampler', full_name='mindinsight.DatasetGraph.sampler', index=3, + number=4, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + syntax='proto2', + extension_ranges=[], + oneofs=[ + ], + serialized_start=1633, + serialized_end=1829, +) + + +_OPERATION = _descriptor.Descriptor( + name='Operation', + full_name='mindinsight.Operation', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='operationParam', full_name='mindinsight.Operation.operationParam', index=0, + number=1, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='size', full_name='mindinsight.Operation.size', index=1, + number=2, type=5, cpp_type=1, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='weights', full_name='mindinsight.Operation.weights', index=2, + number=3, type=2, cpp_type=6, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + syntax='proto2', + extension_ranges=[], + oneofs=[ + ], + serialized_start=1831, + serialized_end=1930, +) + + +_OPERATIONPARAMETER_MAPSTRENTRY = _descriptor.Descriptor( + name='MapStrEntry', + full_name='mindinsight.OperationParameter.MapStrEntry', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='key', full_name='mindinsight.OperationParameter.MapStrEntry.key', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='value', full_name='mindinsight.OperationParameter.MapStrEntry.value', index=1, + number=2, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=_descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001')), + is_extendable=False, + syntax='proto2', + extension_ranges=[], + oneofs=[ + ], + serialized_start=788, + serialized_end=833, +) + +_OPERATIONPARAMETER_MAPSTRLISTENTRY = _descriptor.Descriptor( + name='MapStrListEntry', + full_name='mindinsight.OperationParameter.MapStrListEntry', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='key', full_name='mindinsight.OperationParameter.MapStrListEntry.key', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='value', full_name='mindinsight.OperationParameter.MapStrListEntry.value', index=1, + number=2, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=_descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001')), + is_extendable=False, + syntax='proto2', + extension_ranges=[], + oneofs=[ + ], + serialized_start=2323, + serialized_end=2394, +) + +_OPERATIONPARAMETER_MAPBOOLENTRY = _descriptor.Descriptor( + name='MapBoolEntry', + full_name='mindinsight.OperationParameter.MapBoolEntry', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='key', full_name='mindinsight.OperationParameter.MapBoolEntry.key', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='value', full_name='mindinsight.OperationParameter.MapBoolEntry.value', index=1, + number=2, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=_descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001')), + is_extendable=False, + syntax='proto2', + extension_ranges=[], + oneofs=[ + ], + serialized_start=2396, + serialized_end=2442, +) + +_OPERATIONPARAMETER_MAPINTENTRY = _descriptor.Descriptor( + name='MapIntEntry', + full_name='mindinsight.OperationParameter.MapIntEntry', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='key', full_name='mindinsight.OperationParameter.MapIntEntry.key', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='value', full_name='mindinsight.OperationParameter.MapIntEntry.value', index=1, + number=2, type=5, cpp_type=1, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=_descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001')), + is_extendable=False, + syntax='proto2', + extension_ranges=[], + oneofs=[ + ], + serialized_start=2444, + serialized_end=2489, +) + +_OPERATIONPARAMETER_MAPDOUBLEENTRY = _descriptor.Descriptor( + name='MapDoubleEntry', + full_name='mindinsight.OperationParameter.MapDoubleEntry', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='key', full_name='mindinsight.OperationParameter.MapDoubleEntry.key', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=_b("").decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='value', full_name='mindinsight.OperationParameter.MapDoubleEntry.value', index=1, + number=2, type=1, cpp_type=5, label=1, + has_default_value=False, default_value=float(0), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=_descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001')), + is_extendable=False, + syntax='proto2', + extension_ranges=[], + oneofs=[ + ], + serialized_start=835, + serialized_end=883, +) + +_OPERATIONPARAMETER = _descriptor.Descriptor( + name='OperationParameter', + full_name='mindinsight.OperationParameter', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='mapStr', full_name='mindinsight.OperationParameter.mapStr', index=0, + number=1, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='mapStrList', full_name='mindinsight.OperationParameter.mapStrList', index=1, + number=2, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='mapBool', full_name='mindinsight.OperationParameter.mapBool', index=2, + number=3, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='mapInt', full_name='mindinsight.OperationParameter.mapInt', index=3, + number=4, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='mapDouble', full_name='mindinsight.OperationParameter.mapDouble', index=4, + number=5, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[_OPERATIONPARAMETER_MAPSTRENTRY, _OPERATIONPARAMETER_MAPSTRLISTENTRY, _OPERATIONPARAMETER_MAPBOOLENTRY, _OPERATIONPARAMETER_MAPINTENTRY, _OPERATIONPARAMETER_MAPDOUBLEENTRY, ], + enum_types=[ + ], + options=None, + is_extendable=False, + syntax='proto2', + extension_ranges=[], + oneofs=[ + ], + serialized_start=1933, + serialized_end=2539, +) + + +_STRLIST = _descriptor.Descriptor( + name='StrList', + full_name='mindinsight.StrList', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='strValue', full_name='mindinsight.StrList.strValue', index=0, + number=1, type=9, cpp_type=9, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + syntax='proto2', + extension_ranges=[], + oneofs=[ + ], + serialized_start=2541, + serialized_end=2568, +) + +_LINEAGEEVENT.fields_by_name['train_lineage'].message_type = _TRAINLINEAGE +_LINEAGEEVENT.fields_by_name['evaluation_lineage'].message_type = _EVALUATIONLINEAGE +_LINEAGEEVENT.fields_by_name['dataset_graph'].message_type = _DATASETGRAPH +_LINEAGEEVENT.fields_by_name['user_defined_info'].message_type = _USERDEFINEDINFO +_LINEAGEEVENT.oneofs_by_name['what'].fields.append( + _LINEAGEEVENT.fields_by_name['version']) +_LINEAGEEVENT.fields_by_name['version'].containing_oneof = _LINEAGEEVENT.oneofs_by_name['what'] +_LINEAGEEVENT.oneofs_by_name['what'].fields.append( + _LINEAGEEVENT.fields_by_name['train_lineage']) +_LINEAGEEVENT.fields_by_name['train_lineage'].containing_oneof = _LINEAGEEVENT.oneofs_by_name['what'] +_LINEAGEEVENT.oneofs_by_name['what'].fields.append( + _LINEAGEEVENT.fields_by_name['evaluation_lineage']) +_LINEAGEEVENT.fields_by_name['evaluation_lineage'].containing_oneof = _LINEAGEEVENT.oneofs_by_name['what'] +_LINEAGEEVENT.oneofs_by_name['what'].fields.append( + _LINEAGEEVENT.fields_by_name['dataset_graph']) +_LINEAGEEVENT.fields_by_name['dataset_graph'].containing_oneof = _LINEAGEEVENT.oneofs_by_name['what'] +_LINEAGEEVENT.oneofs_by_name['what'].fields.append( + _LINEAGEEVENT.fields_by_name['user_defined_info']) +_LINEAGEEVENT.fields_by_name['user_defined_info'].containing_oneof = _LINEAGEEVENT.oneofs_by_name['what'] +_USERDEFINEDINFO_MAPDICTENTRY.fields_by_name['value'].message_type = _USERDEFINEDINFO +_USERDEFINEDINFO_MAPDICTENTRY.containing_type = _USERDEFINEDINFO +_USERDEFINEDINFO_MAPINT32ENTRY.containing_type = _USERDEFINEDINFO +_USERDEFINEDINFO_MAPSTRENTRY.containing_type = _USERDEFINEDINFO +_USERDEFINEDINFO_MAPDOUBLEENTRY.containing_type = _USERDEFINEDINFO +_USERDEFINEDINFO.fields_by_name['user_info'].message_type = _USERDEFINEDINFO +_USERDEFINEDINFO.fields_by_name['map_dict'].message_type = _USERDEFINEDINFO_MAPDICTENTRY +_USERDEFINEDINFO.fields_by_name['map_int32'].message_type = _USERDEFINEDINFO_MAPINT32ENTRY +_USERDEFINEDINFO.fields_by_name['map_str'].message_type = _USERDEFINEDINFO_MAPSTRENTRY +_USERDEFINEDINFO.fields_by_name['map_double'].message_type = _USERDEFINEDINFO_MAPDOUBLEENTRY +_TRAINLINEAGE_HYPERPARAMETERS.containing_type = _TRAINLINEAGE +_TRAINLINEAGE_TRAINDATASET.containing_type = _TRAINLINEAGE +_TRAINLINEAGE_ALGORITHM.containing_type = _TRAINLINEAGE +_TRAINLINEAGE_MODEL.containing_type = _TRAINLINEAGE +_TRAINLINEAGE.fields_by_name['hyper_parameters'].message_type = _TRAINLINEAGE_HYPERPARAMETERS +_TRAINLINEAGE.fields_by_name['train_dataset'].message_type = _TRAINLINEAGE_TRAINDATASET +_TRAINLINEAGE.fields_by_name['algorithm'].message_type = _TRAINLINEAGE_ALGORITHM +_TRAINLINEAGE.fields_by_name['model'].message_type = _TRAINLINEAGE_MODEL +_EVALUATIONLINEAGE_VALIDDATASET.containing_type = _EVALUATIONLINEAGE +_EVALUATIONLINEAGE.fields_by_name['valid_dataset'].message_type = _EVALUATIONLINEAGE_VALIDDATASET +_DATASETGRAPH.fields_by_name['children'].message_type = _DATASETGRAPH +_DATASETGRAPH.fields_by_name['parameter'].message_type = _OPERATIONPARAMETER +_DATASETGRAPH.fields_by_name['operations'].message_type = _OPERATION +_DATASETGRAPH.fields_by_name['sampler'].message_type = _OPERATION +_OPERATION.fields_by_name['operationParam'].message_type = _OPERATIONPARAMETER +_OPERATIONPARAMETER_MAPSTRENTRY.containing_type = _OPERATIONPARAMETER +_OPERATIONPARAMETER_MAPSTRLISTENTRY.fields_by_name['value'].message_type = _STRLIST +_OPERATIONPARAMETER_MAPSTRLISTENTRY.containing_type = _OPERATIONPARAMETER +_OPERATIONPARAMETER_MAPBOOLENTRY.containing_type = _OPERATIONPARAMETER +_OPERATIONPARAMETER_MAPINTENTRY.containing_type = _OPERATIONPARAMETER +_OPERATIONPARAMETER_MAPDOUBLEENTRY.containing_type = _OPERATIONPARAMETER +_OPERATIONPARAMETER.fields_by_name['mapStr'].message_type = _OPERATIONPARAMETER_MAPSTRENTRY +_OPERATIONPARAMETER.fields_by_name['mapStrList'].message_type = _OPERATIONPARAMETER_MAPSTRLISTENTRY +_OPERATIONPARAMETER.fields_by_name['mapBool'].message_type = _OPERATIONPARAMETER_MAPBOOLENTRY +_OPERATIONPARAMETER.fields_by_name['mapInt'].message_type = _OPERATIONPARAMETER_MAPINTENTRY +_OPERATIONPARAMETER.fields_by_name['mapDouble'].message_type = _OPERATIONPARAMETER_MAPDOUBLEENTRY +DESCRIPTOR.message_types_by_name['LineageEvent'] = _LINEAGEEVENT +DESCRIPTOR.message_types_by_name['UserDefinedInfo'] = _USERDEFINEDINFO +DESCRIPTOR.message_types_by_name['TrainLineage'] = _TRAINLINEAGE +DESCRIPTOR.message_types_by_name['EvaluationLineage'] = _EVALUATIONLINEAGE +DESCRIPTOR.message_types_by_name['DatasetGraph'] = _DATASETGRAPH +DESCRIPTOR.message_types_by_name['Operation'] = _OPERATION +DESCRIPTOR.message_types_by_name['OperationParameter'] = _OPERATIONPARAMETER +DESCRIPTOR.message_types_by_name['StrList'] = _STRLIST + +LineageEvent = _reflection.GeneratedProtocolMessageType('LineageEvent', (_message.Message,), dict( + DESCRIPTOR = _LINEAGEEVENT, + __module__ = 'mindinsight_lineage_pb2' + # @@protoc_insertion_point(class_scope:mindinsight.LineageEvent) + )) +_sym_db.RegisterMessage(LineageEvent) + +UserDefinedInfo = _reflection.GeneratedProtocolMessageType('UserDefinedInfo', (_message.Message,), dict( + + MapDictEntry = _reflection.GeneratedProtocolMessageType('MapDictEntry', (_message.Message,), dict( + DESCRIPTOR = _USERDEFINEDINFO_MAPDICTENTRY, + __module__ = 'mindinsight_lineage_pb2' + # @@protoc_insertion_point(class_scope:mindinsight.UserDefinedInfo.MapDictEntry) + )) + , + + MapInt32Entry = _reflection.GeneratedProtocolMessageType('MapInt32Entry', (_message.Message,), dict( + DESCRIPTOR = _USERDEFINEDINFO_MAPINT32ENTRY, + __module__ = 'mindinsight_lineage_pb2' + # @@protoc_insertion_point(class_scope:mindinsight.UserDefinedInfo.MapInt32Entry) + )) + , + + MapStrEntry = _reflection.GeneratedProtocolMessageType('MapStrEntry', (_message.Message,), dict( + DESCRIPTOR = _USERDEFINEDINFO_MAPSTRENTRY, + __module__ = 'mindinsight_lineage_pb2' + # @@protoc_insertion_point(class_scope:mindinsight.UserDefinedInfo.MapStrEntry) + )) + , + + MapDoubleEntry = _reflection.GeneratedProtocolMessageType('MapDoubleEntry', (_message.Message,), dict( + DESCRIPTOR = _USERDEFINEDINFO_MAPDOUBLEENTRY, + __module__ = 'mindinsight_lineage_pb2' + # @@protoc_insertion_point(class_scope:mindinsight.UserDefinedInfo.MapDoubleEntry) + )) + , + DESCRIPTOR = _USERDEFINEDINFO, + __module__ = 'mindinsight_lineage_pb2' + # @@protoc_insertion_point(class_scope:mindinsight.UserDefinedInfo) + )) +_sym_db.RegisterMessage(UserDefinedInfo) +_sym_db.RegisterMessage(UserDefinedInfo.MapDictEntry) +_sym_db.RegisterMessage(UserDefinedInfo.MapInt32Entry) +_sym_db.RegisterMessage(UserDefinedInfo.MapStrEntry) +_sym_db.RegisterMessage(UserDefinedInfo.MapDoubleEntry) + +TrainLineage = _reflection.GeneratedProtocolMessageType('TrainLineage', (_message.Message,), dict( + + HyperParameters = _reflection.GeneratedProtocolMessageType('HyperParameters', (_message.Message,), dict( + DESCRIPTOR = _TRAINLINEAGE_HYPERPARAMETERS, + __module__ = 'mindinsight_lineage_pb2' + # @@protoc_insertion_point(class_scope:mindinsight.TrainLineage.HyperParameters) + )) + , + + TrainDataset = _reflection.GeneratedProtocolMessageType('TrainDataset', (_message.Message,), dict( + DESCRIPTOR = _TRAINLINEAGE_TRAINDATASET, + __module__ = 'mindinsight_lineage_pb2' + # @@protoc_insertion_point(class_scope:mindinsight.TrainLineage.TrainDataset) + )) + , + + Algorithm = _reflection.GeneratedProtocolMessageType('Algorithm', (_message.Message,), dict( + DESCRIPTOR = _TRAINLINEAGE_ALGORITHM, + __module__ = 'mindinsight_lineage_pb2' + # @@protoc_insertion_point(class_scope:mindinsight.TrainLineage.Algorithm) + )) + , + + Model = _reflection.GeneratedProtocolMessageType('Model', (_message.Message,), dict( + DESCRIPTOR = _TRAINLINEAGE_MODEL, + __module__ = 'mindinsight_lineage_pb2' + # @@protoc_insertion_point(class_scope:mindinsight.TrainLineage.Model) + )) + , + DESCRIPTOR = _TRAINLINEAGE, + __module__ = 'mindinsight_lineage_pb2' + # @@protoc_insertion_point(class_scope:mindinsight.TrainLineage) + )) +_sym_db.RegisterMessage(TrainLineage) +_sym_db.RegisterMessage(TrainLineage.HyperParameters) +_sym_db.RegisterMessage(TrainLineage.TrainDataset) +_sym_db.RegisterMessage(TrainLineage.Algorithm) +_sym_db.RegisterMessage(TrainLineage.Model) + +EvaluationLineage = _reflection.GeneratedProtocolMessageType('EvaluationLineage', (_message.Message,), dict( + + ValidDataset = _reflection.GeneratedProtocolMessageType('ValidDataset', (_message.Message,), dict( + DESCRIPTOR = _EVALUATIONLINEAGE_VALIDDATASET, + __module__ = 'mindinsight_lineage_pb2' + # @@protoc_insertion_point(class_scope:mindinsight.EvaluationLineage.ValidDataset) + )) + , + DESCRIPTOR = _EVALUATIONLINEAGE, + __module__ = 'mindinsight_lineage_pb2' + # @@protoc_insertion_point(class_scope:mindinsight.EvaluationLineage) + )) +_sym_db.RegisterMessage(EvaluationLineage) +_sym_db.RegisterMessage(EvaluationLineage.ValidDataset) + +DatasetGraph = _reflection.GeneratedProtocolMessageType('DatasetGraph', (_message.Message,), dict( + DESCRIPTOR = _DATASETGRAPH, + __module__ = 'mindinsight_lineage_pb2' + # @@protoc_insertion_point(class_scope:mindinsight.DatasetGraph) + )) +_sym_db.RegisterMessage(DatasetGraph) + +Operation = _reflection.GeneratedProtocolMessageType('Operation', (_message.Message,), dict( + DESCRIPTOR = _OPERATION, + __module__ = 'mindinsight_lineage_pb2' + # @@protoc_insertion_point(class_scope:mindinsight.Operation) + )) +_sym_db.RegisterMessage(Operation) + +OperationParameter = _reflection.GeneratedProtocolMessageType('OperationParameter', (_message.Message,), dict( + + MapStrEntry = _reflection.GeneratedProtocolMessageType('MapStrEntry', (_message.Message,), dict( + DESCRIPTOR = _OPERATIONPARAMETER_MAPSTRENTRY, + __module__ = 'mindinsight_lineage_pb2' + # @@protoc_insertion_point(class_scope:mindinsight.OperationParameter.MapStrEntry) + )) + , + + MapStrListEntry = _reflection.GeneratedProtocolMessageType('MapStrListEntry', (_message.Message,), dict( + DESCRIPTOR = _OPERATIONPARAMETER_MAPSTRLISTENTRY, + __module__ = 'mindinsight_lineage_pb2' + # @@protoc_insertion_point(class_scope:mindinsight.OperationParameter.MapStrListEntry) + )) + , + + MapBoolEntry = _reflection.GeneratedProtocolMessageType('MapBoolEntry', (_message.Message,), dict( + DESCRIPTOR = _OPERATIONPARAMETER_MAPBOOLENTRY, + __module__ = 'mindinsight_lineage_pb2' + # @@protoc_insertion_point(class_scope:mindinsight.OperationParameter.MapBoolEntry) + )) + , + + MapIntEntry = _reflection.GeneratedProtocolMessageType('MapIntEntry', (_message.Message,), dict( + DESCRIPTOR = _OPERATIONPARAMETER_MAPINTENTRY, + __module__ = 'mindinsight_lineage_pb2' + # @@protoc_insertion_point(class_scope:mindinsight.OperationParameter.MapIntEntry) + )) + , + + MapDoubleEntry = _reflection.GeneratedProtocolMessageType('MapDoubleEntry', (_message.Message,), dict( + DESCRIPTOR = _OPERATIONPARAMETER_MAPDOUBLEENTRY, + __module__ = 'mindinsight_lineage_pb2' + # @@protoc_insertion_point(class_scope:mindinsight.OperationParameter.MapDoubleEntry) + )) + , + DESCRIPTOR = _OPERATIONPARAMETER, + __module__ = 'mindinsight_lineage_pb2' + # @@protoc_insertion_point(class_scope:mindinsight.OperationParameter) + )) +_sym_db.RegisterMessage(OperationParameter) +_sym_db.RegisterMessage(OperationParameter.MapStrEntry) +_sym_db.RegisterMessage(OperationParameter.MapStrListEntry) +_sym_db.RegisterMessage(OperationParameter.MapBoolEntry) +_sym_db.RegisterMessage(OperationParameter.MapIntEntry) +_sym_db.RegisterMessage(OperationParameter.MapDoubleEntry) + +StrList = _reflection.GeneratedProtocolMessageType('StrList', (_message.Message,), dict( + DESCRIPTOR = _STRLIST, + __module__ = 'mindinsight_lineage_pb2' + # @@protoc_insertion_point(class_scope:mindinsight.StrList) + )) +_sym_db.RegisterMessage(StrList) + + +DESCRIPTOR.has_options = True +DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), _b('\370\001\001')) +_USERDEFINEDINFO_MAPDICTENTRY.has_options = True +_USERDEFINEDINFO_MAPDICTENTRY._options = _descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001')) +_USERDEFINEDINFO_MAPINT32ENTRY.has_options = True +_USERDEFINEDINFO_MAPINT32ENTRY._options = _descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001')) +_USERDEFINEDINFO_MAPSTRENTRY.has_options = True +_USERDEFINEDINFO_MAPSTRENTRY._options = _descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001')) +_USERDEFINEDINFO_MAPDOUBLEENTRY.has_options = True +_USERDEFINEDINFO_MAPDOUBLEENTRY._options = _descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001')) +_OPERATIONPARAMETER_MAPSTRENTRY.has_options = True +_OPERATIONPARAMETER_MAPSTRENTRY._options = _descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001')) +_OPERATIONPARAMETER_MAPSTRLISTENTRY.has_options = True +_OPERATIONPARAMETER_MAPSTRLISTENTRY._options = _descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001')) +_OPERATIONPARAMETER_MAPBOOLENTRY.has_options = True +_OPERATIONPARAMETER_MAPBOOLENTRY._options = _descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001')) +_OPERATIONPARAMETER_MAPINTENTRY.has_options = True +_OPERATIONPARAMETER_MAPINTENTRY._options = _descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001')) +_OPERATIONPARAMETER_MAPDOUBLEENTRY.has_options = True +_OPERATIONPARAMETER_MAPDOUBLEENTRY._options = _descriptor._ParseOptions(descriptor_pb2.MessageOptions(), _b('8\001')) +# @@protoc_insertion_point(module_scope) diff --git a/mindinsight/datavisual/proto_files/mindinsight_summary.proto b/mindinsight/datavisual/proto_files/mindinsight_summary.proto index 2b3b3d3affb7a1e8623c85531a6db4e58948476a..60e7f8af746de4fd7c209168cf3c70d15e9d3c32 100644 --- a/mindinsight/datavisual/proto_files/mindinsight_summary.proto +++ b/mindinsight/datavisual/proto_files/mindinsight_summary.proto @@ -39,60 +39,9 @@ message Event { // Summary data Summary summary = 5; - // Train lineage - TrainLineage train_lineage = 6; - - // Evaluation lineage - EvaluationLineage evaluation_lineage = 7; - - // dataset graph - DatasetGraph dataset_graph = 9; } } -// TrainLineage records infos of a train. -message TrainLineage{ - message HyperParameters{ - optional string optimizer = 1; - optional float learning_rate = 2; - optional string loss_function = 3; - optional int32 epoch = 4; - optional string parallel_mode = 5; - optional int32 device_num = 6; - optional int32 batch_size = 8; - } - - message TrainDataset{ - optional string train_dataset_path = 1; - optional int32 train_dataset_size = 2; - } - - message Algorithm{ - optional string network = 1; - optional float loss = 2; - } - - message Model{ - optional string path = 3; - optional int64 size = 4; - } - - optional HyperParameters hyper_parameters = 1; - optional TrainDataset train_dataset = 2; - optional Algorithm algorithm = 3; - optional Model model = 4; -} - -//EvalLineage records infos of evaluation. -message EvaluationLineage{ - message ValidDataset{ - optional string valid_dataset_path = 1; - optional int32 valid_dataset_size = 2; - } - - optional string metric = 2; - optional ValidDataset valid_dataset = 3; -} // A Summary is a set of named values that be produced regularly during training message Summary { @@ -127,29 +76,3 @@ message Summary { // Set of values for the summary. repeated Value value = 1; } - -// DatasetGraph -message DatasetGraph { - repeated DatasetGraph children = 1; - optional OperationParameter parameter = 2; - repeated Operation operations = 3; - optional Operation sampler = 4; -} - -message Operation { - optional OperationParameter operationParam = 1; - repeated int32 size = 2; - repeated float weights = 3; -} - -message OperationParameter{ - map mapStr = 1; - map mapStrList = 2; - map mapBool = 3; - map mapInt = 4; - map mapDouble = 5; -} - -message StrList { - repeated string strValue = 1; -} diff --git a/mindinsight/datavisual/proto_files/mindinsight_summary_pb2.py b/mindinsight/datavisual/proto_files/mindinsight_summary_pb2.py index f3b2590655239e610b379fdd96829accdc93255e..1e4528f84f8ebe97c27469aa40ced6dc86b30172 100644 --- a/mindinsight/datavisual/proto_files/mindinsight_summary_pb2.py +++ b/mindinsight/datavisual/proto_files/mindinsight_summary_pb2.py @@ -1,11 +1,13 @@ -# -*- coding: utf-8 -*- # Generated by the protocol buffer compiler. DO NOT EDIT! # source: mindinsight_summary.proto +import sys +_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) from google.protobuf import descriptor as _descriptor from google.protobuf import message as _message from google.protobuf import reflection as _reflection from google.protobuf import symbol_database as _symbol_database +from google.protobuf import descriptor_pb2 # @@protoc_insertion_point(imports) _sym_db = _symbol_database.Default() @@ -18,10 +20,10 @@ DESCRIPTOR = _descriptor.FileDescriptor( name='mindinsight_summary.proto', package='mindinsight', syntax='proto2', - serialized_options=b'\370\001\001', - serialized_pb=b'\n\x19mindinsight_summary.proto\x12\x0bmindinsight\x1a\x18mindinsight_anf_ir.proto\"\xc0\x02\n\x05\x45vent\x12\x11\n\twall_time\x18\x01 \x02(\x01\x12\x0c\n\x04step\x18\x02 \x01(\x03\x12\x11\n\x07version\x18\x03 \x01(\tH\x00\x12,\n\tgraph_def\x18\x04 \x01(\x0b\x32\x17.mindinsight.GraphProtoH\x00\x12\'\n\x07summary\x18\x05 \x01(\x0b\x32\x14.mindinsight.SummaryH\x00\x12\x32\n\rtrain_lineage\x18\x06 \x01(\x0b\x32\x19.mindinsight.TrainLineageH\x00\x12<\n\x12\x65valuation_lineage\x18\x07 \x01(\x0b\x32\x1e.mindinsight.EvaluationLineageH\x00\x12\x32\n\rdataset_graph\x18\t \x01(\x0b\x32\x19.mindinsight.DatasetGraphH\x00\x42\x06\n\x04what\"\xb6\x04\n\x0cTrainLineage\x12\x43\n\x10hyper_parameters\x18\x01 \x01(\x0b\x32).mindinsight.TrainLineage.HyperParameters\x12=\n\rtrain_dataset\x18\x02 \x01(\x0b\x32&.mindinsight.TrainLineage.TrainDataset\x12\x36\n\talgorithm\x18\x03 \x01(\x0b\x32#.mindinsight.TrainLineage.Algorithm\x12.\n\x05model\x18\x04 \x01(\x0b\x32\x1f.mindinsight.TrainLineage.Model\x1a\xa0\x01\n\x0fHyperParameters\x12\x11\n\toptimizer\x18\x01 \x01(\t\x12\x15\n\rlearning_rate\x18\x02 \x01(\x02\x12\x15\n\rloss_function\x18\x03 \x01(\t\x12\r\n\x05\x65poch\x18\x04 \x01(\x05\x12\x15\n\rparallel_mode\x18\x05 \x01(\t\x12\x12\n\ndevice_num\x18\x06 \x01(\x05\x12\x12\n\nbatch_size\x18\x08 \x01(\x05\x1a\x46\n\x0cTrainDataset\x12\x1a\n\x12train_dataset_path\x18\x01 \x01(\t\x12\x1a\n\x12train_dataset_size\x18\x02 \x01(\x05\x1a*\n\tAlgorithm\x12\x0f\n\x07network\x18\x01 \x01(\t\x12\x0c\n\x04loss\x18\x02 \x01(\x02\x1a#\n\x05Model\x12\x0c\n\x04path\x18\x03 \x01(\t\x12\x0c\n\x04size\x18\x04 \x01(\x03\"\xaf\x01\n\x11\x45valuationLineage\x12\x0e\n\x06metric\x18\x02 \x01(\t\x12\x42\n\rvalid_dataset\x18\x03 \x01(\x0b\x32+.mindinsight.EvaluationLineage.ValidDataset\x1a\x46\n\x0cValidDataset\x12\x1a\n\x12valid_dataset_path\x18\x01 \x01(\t\x12\x1a\n\x12valid_dataset_size\x18\x02 \x01(\x05\"\x98\x02\n\x07Summary\x12)\n\x05value\x18\x01 \x03(\x0b\x32\x1a.mindinsight.Summary.Value\x1aQ\n\x05Image\x12\x0e\n\x06height\x18\x01 \x02(\x05\x12\r\n\x05width\x18\x02 \x02(\x05\x12\x12\n\ncolorspace\x18\x03 \x02(\x05\x12\x15\n\rencoded_image\x18\x04 \x02(\x0c\x1a\x8e\x01\n\x05Value\x12\x0b\n\x03tag\x18\x01 \x02(\t\x12\x16\n\x0cscalar_value\x18\x03 \x01(\x02H\x00\x12+\n\x05image\x18\x04 \x01(\x0b\x32\x1a.mindinsight.Summary.ImageH\x00\x12*\n\x06tensor\x18\x08 \x01(\x0b\x32\x18.mindinsight.TensorProtoH\x00\x42\x07\n\x05value\"\xc4\x01\n\x0c\x44\x61tasetGraph\x12+\n\x08\x63hildren\x18\x01 \x03(\x0b\x32\x19.mindinsight.DatasetGraph\x12\x32\n\tparameter\x18\x02 \x01(\x0b\x32\x1f.mindinsight.OperationParameter\x12*\n\noperations\x18\x03 \x03(\x0b\x32\x16.mindinsight.Operation\x12\'\n\x07sampler\x18\x04 \x01(\x0b\x32\x16.mindinsight.Operation\"c\n\tOperation\x12\x37\n\x0eoperationParam\x18\x01 \x01(\x0b\x32\x1f.mindinsight.OperationParameter\x12\x0c\n\x04size\x18\x02 \x03(\x05\x12\x0f\n\x07weights\x18\x03 \x03(\x02\"\xde\x04\n\x12OperationParameter\x12;\n\x06mapStr\x18\x01 \x03(\x0b\x32+.mindinsight.OperationParameter.MapStrEntry\x12\x43\n\nmapStrList\x18\x02 \x03(\x0b\x32/.mindinsight.OperationParameter.MapStrListEntry\x12=\n\x07mapBool\x18\x03 \x03(\x0b\x32,.mindinsight.OperationParameter.MapBoolEntry\x12;\n\x06mapInt\x18\x04 \x03(\x0b\x32+.mindinsight.OperationParameter.MapIntEntry\x12\x41\n\tmapDouble\x18\x05 \x03(\x0b\x32..mindinsight.OperationParameter.MapDoubleEntry\x1a-\n\x0bMapStrEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x1aG\n\x0fMapStrListEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12#\n\x05value\x18\x02 \x01(\x0b\x32\x14.mindinsight.StrList:\x02\x38\x01\x1a.\n\x0cMapBoolEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x08:\x02\x38\x01\x1a-\n\x0bMapIntEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x05:\x02\x38\x01\x1a\x30\n\x0eMapDoubleEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x01:\x02\x38\x01\"\x1b\n\x07StrList\x12\x10\n\x08strValue\x18\x01 \x03(\tB\x03\xf8\x01\x01' + serialized_pb=_b('\n\x19mindinsight_summary.proto\x12\x0bmindinsight\x1a\x18mindinsight_anf_ir.proto\"\x9a\x01\n\x05\x45vent\x12\x11\n\twall_time\x18\x01 \x02(\x01\x12\x0c\n\x04step\x18\x02 \x01(\x03\x12\x11\n\x07version\x18\x03 \x01(\tH\x00\x12,\n\tgraph_def\x18\x04 \x01(\x0b\x32\x17.mindinsight.GraphProtoH\x00\x12\'\n\x07summary\x18\x05 \x01(\x0b\x32\x14.mindinsight.SummaryH\x00\x42\x06\n\x04what\"\x98\x02\n\x07Summary\x12)\n\x05value\x18\x01 \x03(\x0b\x32\x1a.mindinsight.Summary.Value\x1aQ\n\x05Image\x12\x0e\n\x06height\x18\x01 \x02(\x05\x12\r\n\x05width\x18\x02 \x02(\x05\x12\x12\n\ncolorspace\x18\x03 \x02(\x05\x12\x15\n\rencoded_image\x18\x04 \x02(\x0c\x1a\x8e\x01\n\x05Value\x12\x0b\n\x03tag\x18\x01 \x02(\t\x12\x16\n\x0cscalar_value\x18\x03 \x01(\x02H\x00\x12+\n\x05image\x18\x04 \x01(\x0b\x32\x1a.mindinsight.Summary.ImageH\x00\x12*\n\x06tensor\x18\x08 \x01(\x0b\x32\x18.mindinsight.TensorProtoH\x00\x42\x07\n\x05valueB\x03\xf8\x01\x01') , dependencies=[mindinsight__anf__ir__pb2.DESCRIPTOR,]) +_sym_db.RegisterFileDescriptor(DESCRIPTOR) @@ -39,63 +41,42 @@ _EVENT = _descriptor.Descriptor( has_default_value=False, default_value=float(0), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), + options=None), _descriptor.FieldDescriptor( name='step', full_name='mindinsight.Event.step', index=1, number=2, type=3, cpp_type=2, label=1, has_default_value=False, default_value=0, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), + options=None), _descriptor.FieldDescriptor( name='version', full_name='mindinsight.Event.version', index=2, number=3, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), + has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), + options=None), _descriptor.FieldDescriptor( name='graph_def', full_name='mindinsight.Event.graph_def', index=3, number=4, type=11, cpp_type=10, label=1, has_default_value=False, default_value=None, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), + options=None), _descriptor.FieldDescriptor( name='summary', full_name='mindinsight.Event.summary', index=4, number=5, type=11, cpp_type=10, label=1, has_default_value=False, default_value=None, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='train_lineage', full_name='mindinsight.Event.train_lineage', index=5, - number=6, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='evaluation_lineage', full_name='mindinsight.Event.evaluation_lineage', index=6, - number=7, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='dataset_graph', full_name='mindinsight.Event.dataset_graph', index=7, - number=9, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), + options=None), ], extensions=[ ], nested_types=[], enum_types=[ ], - serialized_options=None, + options=None, is_extendable=False, syntax='proto2', extension_ranges=[], @@ -105,317 +86,7 @@ _EVENT = _descriptor.Descriptor( index=0, containing_type=None, fields=[]), ], serialized_start=69, - serialized_end=389, -) - - -_TRAINLINEAGE_HYPERPARAMETERS = _descriptor.Descriptor( - name='HyperParameters', - full_name='mindinsight.TrainLineage.HyperParameters', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='optimizer', full_name='mindinsight.TrainLineage.HyperParameters.optimizer', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='learning_rate', full_name='mindinsight.TrainLineage.HyperParameters.learning_rate', index=1, - number=2, type=2, cpp_type=6, label=1, - has_default_value=False, default_value=float(0), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='loss_function', full_name='mindinsight.TrainLineage.HyperParameters.loss_function', index=2, - number=3, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='epoch', full_name='mindinsight.TrainLineage.HyperParameters.epoch', index=3, - number=4, type=5, cpp_type=1, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='parallel_mode', full_name='mindinsight.TrainLineage.HyperParameters.parallel_mode', index=4, - number=5, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='device_num', full_name='mindinsight.TrainLineage.HyperParameters.device_num', index=5, - number=6, type=5, cpp_type=1, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='batch_size', full_name='mindinsight.TrainLineage.HyperParameters.batch_size', index=6, - number=8, type=5, cpp_type=1, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto2', - extension_ranges=[], - oneofs=[ - ], - serialized_start=645, - serialized_end=805, -) - -_TRAINLINEAGE_TRAINDATASET = _descriptor.Descriptor( - name='TrainDataset', - full_name='mindinsight.TrainLineage.TrainDataset', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='train_dataset_path', full_name='mindinsight.TrainLineage.TrainDataset.train_dataset_path', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='train_dataset_size', full_name='mindinsight.TrainLineage.TrainDataset.train_dataset_size', index=1, - number=2, type=5, cpp_type=1, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto2', - extension_ranges=[], - oneofs=[ - ], - serialized_start=807, - serialized_end=877, -) - -_TRAINLINEAGE_ALGORITHM = _descriptor.Descriptor( - name='Algorithm', - full_name='mindinsight.TrainLineage.Algorithm', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='network', full_name='mindinsight.TrainLineage.Algorithm.network', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='loss', full_name='mindinsight.TrainLineage.Algorithm.loss', index=1, - number=2, type=2, cpp_type=6, label=1, - has_default_value=False, default_value=float(0), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto2', - extension_ranges=[], - oneofs=[ - ], - serialized_start=879, - serialized_end=921, -) - -_TRAINLINEAGE_MODEL = _descriptor.Descriptor( - name='Model', - full_name='mindinsight.TrainLineage.Model', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='path', full_name='mindinsight.TrainLineage.Model.path', index=0, - number=3, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='size', full_name='mindinsight.TrainLineage.Model.size', index=1, - number=4, type=3, cpp_type=2, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto2', - extension_ranges=[], - oneofs=[ - ], - serialized_start=923, - serialized_end=958, -) - -_TRAINLINEAGE = _descriptor.Descriptor( - name='TrainLineage', - full_name='mindinsight.TrainLineage', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='hyper_parameters', full_name='mindinsight.TrainLineage.hyper_parameters', index=0, - number=1, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='train_dataset', full_name='mindinsight.TrainLineage.train_dataset', index=1, - number=2, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='algorithm', full_name='mindinsight.TrainLineage.algorithm', index=2, - number=3, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='model', full_name='mindinsight.TrainLineage.model', index=3, - number=4, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[_TRAINLINEAGE_HYPERPARAMETERS, _TRAINLINEAGE_TRAINDATASET, _TRAINLINEAGE_ALGORITHM, _TRAINLINEAGE_MODEL, ], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto2', - extension_ranges=[], - oneofs=[ - ], - serialized_start=392, - serialized_end=958, -) - - -_EVALUATIONLINEAGE_VALIDDATASET = _descriptor.Descriptor( - name='ValidDataset', - full_name='mindinsight.EvaluationLineage.ValidDataset', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='valid_dataset_path', full_name='mindinsight.EvaluationLineage.ValidDataset.valid_dataset_path', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='valid_dataset_size', full_name='mindinsight.EvaluationLineage.ValidDataset.valid_dataset_size', index=1, - number=2, type=5, cpp_type=1, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto2', - extension_ranges=[], - oneofs=[ - ], - serialized_start=1066, - serialized_end=1136, -) - -_EVALUATIONLINEAGE = _descriptor.Descriptor( - name='EvaluationLineage', - full_name='mindinsight.EvaluationLineage', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='metric', full_name='mindinsight.EvaluationLineage.metric', index=0, - number=2, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='valid_dataset', full_name='mindinsight.EvaluationLineage.valid_dataset', index=1, - number=3, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[_EVALUATIONLINEAGE_VALIDDATASET, ], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto2', - extension_ranges=[], - oneofs=[ - ], - serialized_start=961, - serialized_end=1136, + serialized_end=223, ) @@ -432,42 +103,42 @@ _SUMMARY_IMAGE = _descriptor.Descriptor( has_default_value=False, default_value=0, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), + options=None), _descriptor.FieldDescriptor( name='width', full_name='mindinsight.Summary.Image.width', index=1, number=2, type=5, cpp_type=1, label=2, has_default_value=False, default_value=0, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), + options=None), _descriptor.FieldDescriptor( name='colorspace', full_name='mindinsight.Summary.Image.colorspace', index=2, number=3, type=5, cpp_type=1, label=2, has_default_value=False, default_value=0, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), + options=None), _descriptor.FieldDescriptor( name='encoded_image', full_name='mindinsight.Summary.Image.encoded_image', index=3, number=4, type=12, cpp_type=9, label=2, - has_default_value=False, default_value=b"", + has_default_value=False, default_value=_b(""), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), + options=None), ], extensions=[ ], nested_types=[], enum_types=[ ], - serialized_options=None, + options=None, is_extendable=False, syntax='proto2', extension_ranges=[], oneofs=[ ], - serialized_start=1193, - serialized_end=1274, + serialized_start=280, + serialized_end=361, ) _SUMMARY_VALUE = _descriptor.Descriptor( @@ -480,38 +151,38 @@ _SUMMARY_VALUE = _descriptor.Descriptor( _descriptor.FieldDescriptor( name='tag', full_name='mindinsight.Summary.Value.tag', index=0, number=1, type=9, cpp_type=9, label=2, - has_default_value=False, default_value=b"".decode('utf-8'), + has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), + options=None), _descriptor.FieldDescriptor( name='scalar_value', full_name='mindinsight.Summary.Value.scalar_value', index=1, number=3, type=2, cpp_type=6, label=1, has_default_value=False, default_value=float(0), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), + options=None), _descriptor.FieldDescriptor( name='image', full_name='mindinsight.Summary.Value.image', index=2, number=4, type=11, cpp_type=10, label=1, has_default_value=False, default_value=None, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), + options=None), _descriptor.FieldDescriptor( name='tensor', full_name='mindinsight.Summary.Value.tensor', index=3, number=8, type=11, cpp_type=10, label=1, has_default_value=False, default_value=None, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), + options=None), ], extensions=[ ], nested_types=[], enum_types=[ ], - serialized_options=None, + options=None, is_extendable=False, syntax='proto2', extension_ranges=[], @@ -520,8 +191,8 @@ _SUMMARY_VALUE = _descriptor.Descriptor( name='value', full_name='mindinsight.Summary.Value.value', index=0, containing_type=None, fields=[]), ], - serialized_start=1277, - serialized_end=1419, + serialized_start=364, + serialized_end=506, ) _SUMMARY = _descriptor.Descriptor( @@ -537,400 +208,25 @@ _SUMMARY = _descriptor.Descriptor( has_default_value=False, default_value=[], message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), + options=None), ], extensions=[ ], nested_types=[_SUMMARY_IMAGE, _SUMMARY_VALUE, ], enum_types=[ ], - serialized_options=None, + options=None, is_extendable=False, syntax='proto2', extension_ranges=[], oneofs=[ ], - serialized_start=1139, - serialized_end=1419, -) - - -_DATASETGRAPH = _descriptor.Descriptor( - name='DatasetGraph', - full_name='mindinsight.DatasetGraph', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='children', full_name='mindinsight.DatasetGraph.children', index=0, - number=1, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='parameter', full_name='mindinsight.DatasetGraph.parameter', index=1, - number=2, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='operations', full_name='mindinsight.DatasetGraph.operations', index=2, - number=3, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='sampler', full_name='mindinsight.DatasetGraph.sampler', index=3, - number=4, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto2', - extension_ranges=[], - oneofs=[ - ], - serialized_start=1422, - serialized_end=1618, -) - - -_OPERATION = _descriptor.Descriptor( - name='Operation', - full_name='mindinsight.Operation', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='operationParam', full_name='mindinsight.Operation.operationParam', index=0, - number=1, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='size', full_name='mindinsight.Operation.size', index=1, - number=2, type=5, cpp_type=1, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='weights', full_name='mindinsight.Operation.weights', index=2, - number=3, type=2, cpp_type=6, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto2', - extension_ranges=[], - oneofs=[ - ], - serialized_start=1620, - serialized_end=1719, -) - - -_OPERATIONPARAMETER_MAPSTRENTRY = _descriptor.Descriptor( - name='MapStrEntry', - full_name='mindinsight.OperationParameter.MapStrEntry', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='key', full_name='mindinsight.OperationParameter.MapStrEntry.key', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='value', full_name='mindinsight.OperationParameter.MapStrEntry.value', index=1, - number=2, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=b'8\001', - is_extendable=False, - syntax='proto2', - extension_ranges=[], - oneofs=[ - ], - serialized_start=2065, - serialized_end=2110, -) - -_OPERATIONPARAMETER_MAPSTRLISTENTRY = _descriptor.Descriptor( - name='MapStrListEntry', - full_name='mindinsight.OperationParameter.MapStrListEntry', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='key', full_name='mindinsight.OperationParameter.MapStrListEntry.key', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='value', full_name='mindinsight.OperationParameter.MapStrListEntry.value', index=1, - number=2, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=b'8\001', - is_extendable=False, - syntax='proto2', - extension_ranges=[], - oneofs=[ - ], - serialized_start=2112, - serialized_end=2183, -) - -_OPERATIONPARAMETER_MAPBOOLENTRY = _descriptor.Descriptor( - name='MapBoolEntry', - full_name='mindinsight.OperationParameter.MapBoolEntry', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='key', full_name='mindinsight.OperationParameter.MapBoolEntry.key', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='value', full_name='mindinsight.OperationParameter.MapBoolEntry.value', index=1, - number=2, type=8, cpp_type=7, label=1, - has_default_value=False, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=b'8\001', - is_extendable=False, - syntax='proto2', - extension_ranges=[], - oneofs=[ - ], - serialized_start=2185, - serialized_end=2231, -) - -_OPERATIONPARAMETER_MAPINTENTRY = _descriptor.Descriptor( - name='MapIntEntry', - full_name='mindinsight.OperationParameter.MapIntEntry', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='key', full_name='mindinsight.OperationParameter.MapIntEntry.key', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='value', full_name='mindinsight.OperationParameter.MapIntEntry.value', index=1, - number=2, type=5, cpp_type=1, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=b'8\001', - is_extendable=False, - syntax='proto2', - extension_ranges=[], - oneofs=[ - ], - serialized_start=2233, - serialized_end=2278, -) - -_OPERATIONPARAMETER_MAPDOUBLEENTRY = _descriptor.Descriptor( - name='MapDoubleEntry', - full_name='mindinsight.OperationParameter.MapDoubleEntry', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='key', full_name='mindinsight.OperationParameter.MapDoubleEntry.key', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=b"".decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='value', full_name='mindinsight.OperationParameter.MapDoubleEntry.value', index=1, - number=2, type=1, cpp_type=5, label=1, - has_default_value=False, default_value=float(0), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=b'8\001', - is_extendable=False, - syntax='proto2', - extension_ranges=[], - oneofs=[ - ], - serialized_start=2280, - serialized_end=2328, -) - -_OPERATIONPARAMETER = _descriptor.Descriptor( - name='OperationParameter', - full_name='mindinsight.OperationParameter', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='mapStr', full_name='mindinsight.OperationParameter.mapStr', index=0, - number=1, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='mapStrList', full_name='mindinsight.OperationParameter.mapStrList', index=1, - number=2, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='mapBool', full_name='mindinsight.OperationParameter.mapBool', index=2, - number=3, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='mapInt', full_name='mindinsight.OperationParameter.mapInt', index=3, - number=4, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='mapDouble', full_name='mindinsight.OperationParameter.mapDouble', index=4, - number=5, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[_OPERATIONPARAMETER_MAPSTRENTRY, _OPERATIONPARAMETER_MAPSTRLISTENTRY, _OPERATIONPARAMETER_MAPBOOLENTRY, _OPERATIONPARAMETER_MAPINTENTRY, _OPERATIONPARAMETER_MAPDOUBLEENTRY, ], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto2', - extension_ranges=[], - oneofs=[ - ], - serialized_start=1722, - serialized_end=2328, -) - - -_STRLIST = _descriptor.Descriptor( - name='StrList', - full_name='mindinsight.StrList', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='strValue', full_name='mindinsight.StrList.strValue', index=0, - number=1, type=9, cpp_type=9, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto2', - extension_ranges=[], - oneofs=[ - ], - serialized_start=2330, - serialized_end=2357, + serialized_start=226, + serialized_end=506, ) _EVENT.fields_by_name['graph_def'].message_type = mindinsight__anf__ir__pb2._GRAPHPROTO _EVENT.fields_by_name['summary'].message_type = _SUMMARY -_EVENT.fields_by_name['train_lineage'].message_type = _TRAINLINEAGE -_EVENT.fields_by_name['evaluation_lineage'].message_type = _EVALUATIONLINEAGE -_EVENT.fields_by_name['dataset_graph'].message_type = _DATASETGRAPH _EVENT.oneofs_by_name['what'].fields.append( _EVENT.fields_by_name['version']) _EVENT.fields_by_name['version'].containing_oneof = _EVENT.oneofs_by_name['what'] @@ -940,25 +236,6 @@ _EVENT.fields_by_name['graph_def'].containing_oneof = _EVENT.oneofs_by_name['wha _EVENT.oneofs_by_name['what'].fields.append( _EVENT.fields_by_name['summary']) _EVENT.fields_by_name['summary'].containing_oneof = _EVENT.oneofs_by_name['what'] -_EVENT.oneofs_by_name['what'].fields.append( - _EVENT.fields_by_name['train_lineage']) -_EVENT.fields_by_name['train_lineage'].containing_oneof = _EVENT.oneofs_by_name['what'] -_EVENT.oneofs_by_name['what'].fields.append( - _EVENT.fields_by_name['evaluation_lineage']) -_EVENT.fields_by_name['evaluation_lineage'].containing_oneof = _EVENT.oneofs_by_name['what'] -_EVENT.oneofs_by_name['what'].fields.append( - _EVENT.fields_by_name['dataset_graph']) -_EVENT.fields_by_name['dataset_graph'].containing_oneof = _EVENT.oneofs_by_name['what'] -_TRAINLINEAGE_HYPERPARAMETERS.containing_type = _TRAINLINEAGE -_TRAINLINEAGE_TRAINDATASET.containing_type = _TRAINLINEAGE -_TRAINLINEAGE_ALGORITHM.containing_type = _TRAINLINEAGE -_TRAINLINEAGE_MODEL.containing_type = _TRAINLINEAGE -_TRAINLINEAGE.fields_by_name['hyper_parameters'].message_type = _TRAINLINEAGE_HYPERPARAMETERS -_TRAINLINEAGE.fields_by_name['train_dataset'].message_type = _TRAINLINEAGE_TRAINDATASET -_TRAINLINEAGE.fields_by_name['algorithm'].message_type = _TRAINLINEAGE_ALGORITHM -_TRAINLINEAGE.fields_by_name['model'].message_type = _TRAINLINEAGE_MODEL -_EVALUATIONLINEAGE_VALIDDATASET.containing_type = _EVALUATIONLINEAGE -_EVALUATIONLINEAGE.fields_by_name['valid_dataset'].message_type = _EVALUATIONLINEAGE_VALIDDATASET _SUMMARY_IMAGE.containing_type = _SUMMARY _SUMMARY_VALUE.fields_by_name['image'].message_type = _SUMMARY_IMAGE _SUMMARY_VALUE.fields_by_name['tensor'].message_type = mindinsight__anf__ir__pb2._TENSORPROTO @@ -973,189 +250,40 @@ _SUMMARY_VALUE.oneofs_by_name['value'].fields.append( _SUMMARY_VALUE.fields_by_name['tensor']) _SUMMARY_VALUE.fields_by_name['tensor'].containing_oneof = _SUMMARY_VALUE.oneofs_by_name['value'] _SUMMARY.fields_by_name['value'].message_type = _SUMMARY_VALUE -_DATASETGRAPH.fields_by_name['children'].message_type = _DATASETGRAPH -_DATASETGRAPH.fields_by_name['parameter'].message_type = _OPERATIONPARAMETER -_DATASETGRAPH.fields_by_name['operations'].message_type = _OPERATION -_DATASETGRAPH.fields_by_name['sampler'].message_type = _OPERATION -_OPERATION.fields_by_name['operationParam'].message_type = _OPERATIONPARAMETER -_OPERATIONPARAMETER_MAPSTRENTRY.containing_type = _OPERATIONPARAMETER -_OPERATIONPARAMETER_MAPSTRLISTENTRY.fields_by_name['value'].message_type = _STRLIST -_OPERATIONPARAMETER_MAPSTRLISTENTRY.containing_type = _OPERATIONPARAMETER -_OPERATIONPARAMETER_MAPBOOLENTRY.containing_type = _OPERATIONPARAMETER -_OPERATIONPARAMETER_MAPINTENTRY.containing_type = _OPERATIONPARAMETER -_OPERATIONPARAMETER_MAPDOUBLEENTRY.containing_type = _OPERATIONPARAMETER -_OPERATIONPARAMETER.fields_by_name['mapStr'].message_type = _OPERATIONPARAMETER_MAPSTRENTRY -_OPERATIONPARAMETER.fields_by_name['mapStrList'].message_type = _OPERATIONPARAMETER_MAPSTRLISTENTRY -_OPERATIONPARAMETER.fields_by_name['mapBool'].message_type = _OPERATIONPARAMETER_MAPBOOLENTRY -_OPERATIONPARAMETER.fields_by_name['mapInt'].message_type = _OPERATIONPARAMETER_MAPINTENTRY -_OPERATIONPARAMETER.fields_by_name['mapDouble'].message_type = _OPERATIONPARAMETER_MAPDOUBLEENTRY DESCRIPTOR.message_types_by_name['Event'] = _EVENT -DESCRIPTOR.message_types_by_name['TrainLineage'] = _TRAINLINEAGE -DESCRIPTOR.message_types_by_name['EvaluationLineage'] = _EVALUATIONLINEAGE DESCRIPTOR.message_types_by_name['Summary'] = _SUMMARY -DESCRIPTOR.message_types_by_name['DatasetGraph'] = _DATASETGRAPH -DESCRIPTOR.message_types_by_name['Operation'] = _OPERATION -DESCRIPTOR.message_types_by_name['OperationParameter'] = _OPERATIONPARAMETER -DESCRIPTOR.message_types_by_name['StrList'] = _STRLIST -_sym_db.RegisterFileDescriptor(DESCRIPTOR) -Event = _reflection.GeneratedProtocolMessageType('Event', (_message.Message,), { - 'DESCRIPTOR' : _EVENT, - '__module__' : 'mindinsight_summary_pb2' +Event = _reflection.GeneratedProtocolMessageType('Event', (_message.Message,), dict( + DESCRIPTOR = _EVENT, + __module__ = 'mindinsight_summary_pb2' # @@protoc_insertion_point(class_scope:mindinsight.Event) - }) + )) _sym_db.RegisterMessage(Event) -TrainLineage = _reflection.GeneratedProtocolMessageType('TrainLineage', (_message.Message,), { - - 'HyperParameters' : _reflection.GeneratedProtocolMessageType('HyperParameters', (_message.Message,), { - 'DESCRIPTOR' : _TRAINLINEAGE_HYPERPARAMETERS, - '__module__' : 'mindinsight_summary_pb2' - # @@protoc_insertion_point(class_scope:mindinsight.TrainLineage.HyperParameters) - }) - , - - 'TrainDataset' : _reflection.GeneratedProtocolMessageType('TrainDataset', (_message.Message,), { - 'DESCRIPTOR' : _TRAINLINEAGE_TRAINDATASET, - '__module__' : 'mindinsight_summary_pb2' - # @@protoc_insertion_point(class_scope:mindinsight.TrainLineage.TrainDataset) - }) - , - - 'Algorithm' : _reflection.GeneratedProtocolMessageType('Algorithm', (_message.Message,), { - 'DESCRIPTOR' : _TRAINLINEAGE_ALGORITHM, - '__module__' : 'mindinsight_summary_pb2' - # @@protoc_insertion_point(class_scope:mindinsight.TrainLineage.Algorithm) - }) - , - - 'Model' : _reflection.GeneratedProtocolMessageType('Model', (_message.Message,), { - 'DESCRIPTOR' : _TRAINLINEAGE_MODEL, - '__module__' : 'mindinsight_summary_pb2' - # @@protoc_insertion_point(class_scope:mindinsight.TrainLineage.Model) - }) - , - 'DESCRIPTOR' : _TRAINLINEAGE, - '__module__' : 'mindinsight_summary_pb2' - # @@protoc_insertion_point(class_scope:mindinsight.TrainLineage) - }) -_sym_db.RegisterMessage(TrainLineage) -_sym_db.RegisterMessage(TrainLineage.HyperParameters) -_sym_db.RegisterMessage(TrainLineage.TrainDataset) -_sym_db.RegisterMessage(TrainLineage.Algorithm) -_sym_db.RegisterMessage(TrainLineage.Model) - -EvaluationLineage = _reflection.GeneratedProtocolMessageType('EvaluationLineage', (_message.Message,), { - - 'ValidDataset' : _reflection.GeneratedProtocolMessageType('ValidDataset', (_message.Message,), { - 'DESCRIPTOR' : _EVALUATIONLINEAGE_VALIDDATASET, - '__module__' : 'mindinsight_summary_pb2' - # @@protoc_insertion_point(class_scope:mindinsight.EvaluationLineage.ValidDataset) - }) - , - 'DESCRIPTOR' : _EVALUATIONLINEAGE, - '__module__' : 'mindinsight_summary_pb2' - # @@protoc_insertion_point(class_scope:mindinsight.EvaluationLineage) - }) -_sym_db.RegisterMessage(EvaluationLineage) -_sym_db.RegisterMessage(EvaluationLineage.ValidDataset) - -Summary = _reflection.GeneratedProtocolMessageType('Summary', (_message.Message,), { +Summary = _reflection.GeneratedProtocolMessageType('Summary', (_message.Message,), dict( - 'Image' : _reflection.GeneratedProtocolMessageType('Image', (_message.Message,), { - 'DESCRIPTOR' : _SUMMARY_IMAGE, - '__module__' : 'mindinsight_summary_pb2' + Image = _reflection.GeneratedProtocolMessageType('Image', (_message.Message,), dict( + DESCRIPTOR = _SUMMARY_IMAGE, + __module__ = 'mindinsight_summary_pb2' # @@protoc_insertion_point(class_scope:mindinsight.Summary.Image) - }) + )) , - 'Value' : _reflection.GeneratedProtocolMessageType('Value', (_message.Message,), { - 'DESCRIPTOR' : _SUMMARY_VALUE, - '__module__' : 'mindinsight_summary_pb2' + Value = _reflection.GeneratedProtocolMessageType('Value', (_message.Message,), dict( + DESCRIPTOR = _SUMMARY_VALUE, + __module__ = 'mindinsight_summary_pb2' # @@protoc_insertion_point(class_scope:mindinsight.Summary.Value) - }) + )) , - 'DESCRIPTOR' : _SUMMARY, - '__module__' : 'mindinsight_summary_pb2' + DESCRIPTOR = _SUMMARY, + __module__ = 'mindinsight_summary_pb2' # @@protoc_insertion_point(class_scope:mindinsight.Summary) - }) + )) _sym_db.RegisterMessage(Summary) _sym_db.RegisterMessage(Summary.Image) _sym_db.RegisterMessage(Summary.Value) -DatasetGraph = _reflection.GeneratedProtocolMessageType('DatasetGraph', (_message.Message,), { - 'DESCRIPTOR' : _DATASETGRAPH, - '__module__' : 'mindinsight_summary_pb2' - # @@protoc_insertion_point(class_scope:mindinsight.DatasetGraph) - }) -_sym_db.RegisterMessage(DatasetGraph) - -Operation = _reflection.GeneratedProtocolMessageType('Operation', (_message.Message,), { - 'DESCRIPTOR' : _OPERATION, - '__module__' : 'mindinsight_summary_pb2' - # @@protoc_insertion_point(class_scope:mindinsight.Operation) - }) -_sym_db.RegisterMessage(Operation) - -OperationParameter = _reflection.GeneratedProtocolMessageType('OperationParameter', (_message.Message,), { - - 'MapStrEntry' : _reflection.GeneratedProtocolMessageType('MapStrEntry', (_message.Message,), { - 'DESCRIPTOR' : _OPERATIONPARAMETER_MAPSTRENTRY, - '__module__' : 'mindinsight_summary_pb2' - # @@protoc_insertion_point(class_scope:mindinsight.OperationParameter.MapStrEntry) - }) - , - - 'MapStrListEntry' : _reflection.GeneratedProtocolMessageType('MapStrListEntry', (_message.Message,), { - 'DESCRIPTOR' : _OPERATIONPARAMETER_MAPSTRLISTENTRY, - '__module__' : 'mindinsight_summary_pb2' - # @@protoc_insertion_point(class_scope:mindinsight.OperationParameter.MapStrListEntry) - }) - , - - 'MapBoolEntry' : _reflection.GeneratedProtocolMessageType('MapBoolEntry', (_message.Message,), { - 'DESCRIPTOR' : _OPERATIONPARAMETER_MAPBOOLENTRY, - '__module__' : 'mindinsight_summary_pb2' - # @@protoc_insertion_point(class_scope:mindinsight.OperationParameter.MapBoolEntry) - }) - , - - 'MapIntEntry' : _reflection.GeneratedProtocolMessageType('MapIntEntry', (_message.Message,), { - 'DESCRIPTOR' : _OPERATIONPARAMETER_MAPINTENTRY, - '__module__' : 'mindinsight_summary_pb2' - # @@protoc_insertion_point(class_scope:mindinsight.OperationParameter.MapIntEntry) - }) - , - - 'MapDoubleEntry' : _reflection.GeneratedProtocolMessageType('MapDoubleEntry', (_message.Message,), { - 'DESCRIPTOR' : _OPERATIONPARAMETER_MAPDOUBLEENTRY, - '__module__' : 'mindinsight_summary_pb2' - # @@protoc_insertion_point(class_scope:mindinsight.OperationParameter.MapDoubleEntry) - }) - , - 'DESCRIPTOR' : _OPERATIONPARAMETER, - '__module__' : 'mindinsight_summary_pb2' - # @@protoc_insertion_point(class_scope:mindinsight.OperationParameter) - }) -_sym_db.RegisterMessage(OperationParameter) -_sym_db.RegisterMessage(OperationParameter.MapStrEntry) -_sym_db.RegisterMessage(OperationParameter.MapStrListEntry) -_sym_db.RegisterMessage(OperationParameter.MapBoolEntry) -_sym_db.RegisterMessage(OperationParameter.MapIntEntry) -_sym_db.RegisterMessage(OperationParameter.MapDoubleEntry) - -StrList = _reflection.GeneratedProtocolMessageType('StrList', (_message.Message,), { - 'DESCRIPTOR' : _STRLIST, - '__module__' : 'mindinsight_summary_pb2' - # @@protoc_insertion_point(class_scope:mindinsight.StrList) - }) -_sym_db.RegisterMessage(StrList) - -DESCRIPTOR._options = None -_OPERATIONPARAMETER_MAPSTRENTRY._options = None -_OPERATIONPARAMETER_MAPSTRLISTENTRY._options = None -_OPERATIONPARAMETER_MAPBOOLENTRY._options = None -_OPERATIONPARAMETER_MAPINTENTRY._options = None -_OPERATIONPARAMETER_MAPDOUBLEENTRY._options = None +DESCRIPTOR.has_options = True +DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), _b('\370\001\001')) # @@protoc_insertion_point(module_scope) diff --git a/mindinsight/lineagemgr/api/model.py b/mindinsight/lineagemgr/api/model.py index b6847201046ea5fab76d93332f54dd94fa90c5bb..d22f7a0650b07eca3838b42e447ffaa11a88408a 100644 --- a/mindinsight/lineagemgr/api/model.py +++ b/mindinsight/lineagemgr/api/model.py @@ -112,10 +112,10 @@ def filter_summary_lineage(summary_base_dir, search_condition=None): directories generated by training. search_condition (dict): The search condition. When filtering and sorting, in addition to the following supported fields, fields - prefixed with `metric_` are also supported. The fields prefixed with - `metric_` are related to the `metrics` parameter in the training + prefixed with `metric/` are also supported. The fields prefixed with + `metric/` are related to the `metrics` parameter in the training script. For example, if the key of `metrics` parameter is - `accuracy`, the field should be `metric_accuracy`. Default: None. + `accuracy`, the field should be `metric/accuracy`. Default: None. - summary_dir (dict): The filter condition of summary directory. @@ -187,7 +187,7 @@ def filter_summary_lineage(summary_base_dir, search_condition=None): >>> 'ge': 128, >>> 'le': 256 >>> }, - >>> 'metric_accuracy': { + >>> 'metric/accuracy': { >>> 'lt': 0.1 >>> }, >>> 'sorted_name': 'summary_dir', diff --git a/mindinsight/lineagemgr/collection/model/model_lineage.py b/mindinsight/lineagemgr/collection/model/model_lineage.py index ead6770cb066960ac6d47ae5267cb92a346640f1..88a963814d1097dca3cf8245610cf844d6a13eb6 100644 --- a/mindinsight/lineagemgr/collection/model/model_lineage.py +++ b/mindinsight/lineagemgr/collection/model/model_lineage.py @@ -23,7 +23,8 @@ from mindinsight.utils.exceptions import \ MindInsightException from mindinsight.lineagemgr.common.validator.validate import validate_train_run_context, \ validate_eval_run_context, validate_file_path, validate_network, \ - validate_int_params, validate_summary_record, validate_raise_exception + validate_int_params, validate_summary_record, validate_raise_exception,\ + validate_user_defined_info from mindinsight.lineagemgr.common.exceptions.error_code import LineageErrors, LineageErrorMsg from mindinsight.lineagemgr.common.exceptions.exceptions import LineageParamRunContextError, \ LineageGetModelFileError, LineageLogError @@ -71,7 +72,7 @@ class TrainLineage(Callback): >>> lineagemgr = TrainLineage(summary_record=summary_writer) >>> model.train(epoch_num, dataset, callbacks=[model_ckpt, summary_callback, lineagemgr]) """ - def __init__(self, summary_record, raise_exception=False): + def __init__(self, summary_record, raise_exception=False, user_defined_info=None): super(TrainLineage, self).__init__() try: validate_raise_exception(raise_exception) @@ -85,6 +86,11 @@ class TrainLineage(Callback): self.lineage_log_path = summary_log_path + '_lineage' self.initial_learning_rate = None + + self.user_defined_info = user_defined_info + if user_defined_info: + validate_user_defined_info(user_defined_info) + except MindInsightException as err: log.error(err) if raise_exception: @@ -104,6 +110,10 @@ class TrainLineage(Callback): """ log.info('Initialize training lineage collection...') + if self.user_defined_info: + lineage_summary = LineageSummary(summary_log_path=self.lineage_log_path) + lineage_summary.record_user_defined_info(self.user_defined_info) + if not isinstance(run_context, RunContext): error_msg = f'Invalid TrainLineage run_context.' log.error(error_msg) @@ -239,7 +249,7 @@ class EvalLineage(Callback): >>> lineagemgr = EvalLineage(summary_record=summary_writer) >>> model.eval(epoch_num, dataset, callbacks=[model_ckpt, summary_callback, lineagemgr]) """ - def __init__(self, summary_record, raise_exception=False): + def __init__(self, summary_record, raise_exception=False, user_defined_info=None): super(EvalLineage, self).__init__() try: validate_raise_exception(raise_exception) @@ -251,6 +261,11 @@ class EvalLineage(Callback): summary_log_path = summary_record.full_file_name validate_file_path(summary_log_path) self.lineage_log_path = summary_log_path + '_lineage' + + self.user_defined_info = user_defined_info + if user_defined_info: + validate_user_defined_info(user_defined_info) + except MindInsightException as err: log.error(err) if raise_exception: @@ -269,6 +284,10 @@ class EvalLineage(Callback): MindInsightException: If validating parameter fails. LineageLogError: If recording lineage information fails. """ + if self.user_defined_info: + lineage_summary = LineageSummary(summary_log_path=self.lineage_log_path) + lineage_summary.record_user_defined_info(self.user_defined_info) + if not isinstance(run_context, RunContext): error_msg = f'Invalid EvalLineage run_context.' log.error(error_msg) diff --git a/mindinsight/lineagemgr/common/validator/model_parameter.py b/mindinsight/lineagemgr/common/validator/model_parameter.py index 87de009f6215ca9472c6f348dc571f0fa232baa3..d4ee016b26046caea0b2fe81929ee7bcbc2b0eaf 100644 --- a/mindinsight/lineagemgr/common/validator/model_parameter.py +++ b/mindinsight/lineagemgr/common/validator/model_parameter.py @@ -226,7 +226,7 @@ class SearchModelConditionParameter(Schema): if not isinstance(attr, str): raise LineageParamValueError('The search attribute not supported.') - if attr not in FIELD_MAPPING and not attr.startswith('metric_'): + if attr not in FIELD_MAPPING and not attr.startswith(('metric/','user_defined/')): raise LineageParamValueError('The search attribute not supported.') if not isinstance(condition, dict): @@ -238,7 +238,7 @@ class SearchModelConditionParameter(Schema): raise LineageParamValueError("The compare condition should be in " "('eq', 'lt', 'gt', 'le', 'ge', 'in').") - if attr.startswith('metric_'): + if attr.startswith('metric/'): if len(attr) == 7: raise LineageParamValueError( 'The search attribute not supported.' diff --git a/mindinsight/lineagemgr/common/validator/validate.py b/mindinsight/lineagemgr/common/validator/validate.py index 39435986b9c8f02c15111b00df4bd108a6d53d92..44209b22b81200890d8155105040247533594fc5 100644 --- a/mindinsight/lineagemgr/common/validator/validate.py +++ b/mindinsight/lineagemgr/common/validator/validate.py @@ -349,12 +349,14 @@ def validate_condition(search_condition): if "sorted_name" in search_condition: sorted_name = search_condition.get("sorted_name") err_msg = "The sorted_name must be in {} or start with " \ - "`metric_`.".format(list(FIELD_MAPPING.keys())) + "`metric/` or `user_defined/`.".format(list(FIELD_MAPPING.keys())) if not isinstance(sorted_name, str): log.error(err_msg) raise LineageParamValueError(err_msg) - if sorted_name not in FIELD_MAPPING and not ( - sorted_name.startswith('metric_') and len(sorted_name) > 7): + if not (sorted_name in FIELD_MAPPING + or (sorted_name.startswith('metric/') and len(sorted_name) > 7) + or (sorted_name.startswith('user_defined/') and len(sorted_name) > 13) + ): log.error(err_msg) raise LineageParamValueError(err_msg) @@ -393,3 +395,38 @@ def validate_path(summary_path): raise LineageDirNotExistError("The summary path does not exist or is not a dir.") return summary_path + + +def validate_user_defined_info(user_defined_info): + """ + Validate user defined info. + + Args: + user_defined_info (dict): The user defined info. + + Raises: + LineageParamTypeError: If the type of parameters is invalid. + LineageParamValueError: If user defined keys have been defined in lineage. + + """ + if not isinstance(user_defined_info, dict): + log.error("Invalid user defined info. It should be a dict.") + raise LineageParamTypeError("Invalid user defined info. It should be dict.") + for key, value in user_defined_info: + if not isinstance(key, str): + error_msg = "Dict key type {} is not supported in user defined info." \ + "Only str is permitted now.".format(type(key)) + log.error(error_msg) + raise LineageParamTypeError(error_msg) + if not isinstance(key, (int, str, float)): + error_msg = "Dict value type {} is not supported in user defined info." \ + "Only str, int and float are permitted now.".format(type(value)) + log.error(error_msg) + raise LineageParamTypeError(error_msg) + + field_map = set(FIELD_MAPPING.keys()) + user_defined_keys = set(user_defined_info.keys()) + all_keys = field_map | user_defined_keys + + if len(field_map) + len(user_defined_keys) != len(all_keys): + raise LineageParamValueError("There are some keys have defined in lineage.") \ No newline at end of file diff --git a/mindinsight/lineagemgr/querier/querier.py b/mindinsight/lineagemgr/querier/querier.py index 407ab41aa529e0b483b237034b8c113b533f3225..2f8798126e13e56a93a3152b1193d0e5d8773fc8 100644 --- a/mindinsight/lineagemgr/querier/querier.py +++ b/mindinsight/lineagemgr/querier/querier.py @@ -236,7 +236,7 @@ class Querier: See `ConditionType` and `ExpressionType` class for the rule of filtering and sorting. The filtering and sorting fields are defined in - `FIELD_MAPPING` or prefixed with `metric_`. + `FIELD_MAPPING` or prefixed with `metric/` or 'user_defined/'. If the condition is `None`, all model lineage information will be returned. @@ -288,7 +288,7 @@ class Querier: if condition is None: condition = {} - result = list(filter(_filter, self._lineage_objects)) + results = list(filter(_filter, self._lineage_objects)) if ConditionParam.SORTED_NAME.value in condition: sorted_name = condition.get(ConditionParam.SORTED_NAME.value) @@ -299,19 +299,33 @@ class Querier: ) sorted_type = condition.get(ConditionParam.SORTED_TYPE.value) reverse = sorted_type == 'descending' - result = sorted( - result, key=functools.cmp_to_key(_cmp), reverse=reverse + results = sorted( + results, key=functools.cmp_to_key(_cmp), reverse=reverse ) - offset_result = self._handle_limit_and_offset(condition, result) + offset_results = self._handle_limit_and_offset(condition, results) + + customized = dict() + for offset_result in offset_results: + for obj_name in ["metric", "user_defined"]: + obj = getattr(offset_result, obj_name) + if obj and isinstance(obj, dict): + for key, value in obj.items(): + label = obj_name + "/" + key + customized[label] = dict() + customized[label]["label"] = label + # user defined info is default displayed + customized[label]["required"] = True + customized[label]["type"] = type(value).__name__ search_type = condition.get(ConditionParam.LINEAGE_TYPE.value) lineage_info = { + 'customized': customized, 'object': [ item.to_dataset_lineage_dict() if search_type == LineageType.DATASET.value - else item.to_filtration_dict() for item in offset_result + else item.to_filtration_dict() for item in offset_results ], - 'count': len(result) + 'count': len(results) } return lineage_info @@ -326,7 +340,8 @@ class Querier: Returns: bool, `True` if the field name is valid, else `False`. """ - return field_name not in FIELD_MAPPING and not field_name.startswith('metric_') + return field_name not in FIELD_MAPPING and \ + not field_name.startswith(('metric/', 'user_defined/')) def _handle_limit_and_offset(self, condition, result): """ @@ -397,11 +412,13 @@ class Querier: log_dir = os.path.dirname(log_path) try: lineage_info = LineageSummaryAnalyzer.get_summary_infos(log_path) + user_defined_info = LineageSummaryAnalyzer.get_user_defined_info(log_path) lineage_obj = LineageObj( log_dir, train_lineage=lineage_info.train_lineage, evaluation_lineage=lineage_info.eval_lineage, - dataset_graph=lineage_info.dataset_graph + dataset_graph=lineage_info.dataset_graph, + user_defined_info=user_defined_info ) self._lineage_objects.append(lineage_obj) self._add_dataset_mark() diff --git a/mindinsight/lineagemgr/querier/query_model.py b/mindinsight/lineagemgr/querier/query_model.py index 0ca4be642c360f7d247d1eab9505d52b164fb3e0..a70a215ba16b844fe52183ace2a79604bd46f19a 100644 --- a/mindinsight/lineagemgr/querier/query_model.py +++ b/mindinsight/lineagemgr/querier/query_model.py @@ -57,6 +57,8 @@ class LineageObj: - dataset_graph (Event): Dataset graph object. + - user_defined_info (Event): User defined info object. + Raises: LineageEventNotExistException: If train and evaluation event not exist. LineageEventFieldNotExistException: If the special event field not exist. @@ -72,16 +74,19 @@ class LineageObj: _name_valid_dataset = 'valid_dataset' _name_dataset_graph = 'dataset_graph' _name_dataset_mark = 'dataset_mark' + _name_user_defined = 'user_defined' def __init__(self, summary_dir, **kwargs): self._lineage_info = { self._name_summary_dir: summary_dir } + user_defined_info_list = kwargs.get('user_defined_info', []) train_lineage = kwargs.get('train_lineage') evaluation_lineage = kwargs.get('evaluation_lineage') dataset_graph = kwargs.get('dataset_graph') if not any([train_lineage, evaluation_lineage, dataset_graph]): raise LineageEventNotExistException() + self._parse_user_defined_info(user_defined_info_list) self._parse_train_lineage(train_lineage) self._parse_evaluation_lineage(evaluation_lineage) self._parse_dataset_graph(dataset_graph) @@ -107,6 +112,16 @@ class LineageObj: """ return self._lineage_info.get(self._name_metric) + @property + def user_defined(self): + """ + Get user defined information. + + Returns: + dict, the user defined information. + """ + return self._lineage_info.get(self._name_user_defined) + @property def hyper_parameters(self): """ @@ -237,19 +252,22 @@ class LineageObj: def get_value_by_key(self, key): """ - Get the value based on the key in `FIELD_MAPPING` or the key prefixed with `metric_`. + Get the value based on the key in `FIELD_MAPPING` or + the key prefixed with `metric/` or `user_defined/`. Args: - key (str): The key in `FIELD_MAPPING` or prefixed with `metric_`. + key (str): The key in `FIELD_MAPPING` + or prefixed with `metric/` or `user_defined/`. Returns: object, the value. """ - if key.startswith('metric_'): - metric_key = key.split('_', 1)[1] - metric = self._filtration_result.get(self._name_metric) - if metric: - return metric.get(metric_key) + if key.startswith(('metric/', 'user_defined/')): + key_name, sub_key = key.split('/', 1) + sub_value_name = self._name_metric if key_name == 'metric' else self._name_user_defined + sub_value = self._filtration_result.get(sub_value_name) + if sub_value: + return sub_value.get(sub_key) return self._filtration_result.get(key) def _organize_filtration_result(self): @@ -267,6 +285,8 @@ class LineageObj: if field.sub_name else base_attr # add metric into filtration result result[self._name_metric] = self.metric + + result[self._name_user_defined] = self.user_defined # add dataset_graph into filtration result result[self._name_dataset_graph] = getattr(self, self._name_dataset_graph) return result @@ -342,3 +362,15 @@ class LineageObj: if event_dict is None: raise LineageEventFieldNotExistException(self._name_evaluation_lineage) self._lineage_info[self._name_dataset_graph] = event_dict if event_dict else {} + + def _parse_user_defined_info(self, user_defined_info_list): + """ + Parse user defined info. + + Args: + user_defined_info_list (list): user defined info list. + """ + user_defined_infos = dict() + for user_defined_info in user_defined_info_list: + user_defined_infos.update(user_defined_info) + self._lineage_info[self._name_user_defined] = user_defined_infos diff --git a/mindinsight/lineagemgr/summary/_summary_adapter.py b/mindinsight/lineagemgr/summary/_summary_adapter.py index b2b781317787e33b09d0b29c9ffe245f2c7c7aa1..1e99a6e9883397376c4826dfb35b9da556e90ea2 100644 --- a/mindinsight/lineagemgr/summary/_summary_adapter.py +++ b/mindinsight/lineagemgr/summary/_summary_adapter.py @@ -15,8 +15,9 @@ """The converter between proto format event of lineage and dict.""" import time -from mindinsight.datavisual.proto_files.mindinsight_summary_pb2 import Event -from mindinsight.lineagemgr.common.exceptions.exceptions import LineageParamTypeError +from mindinsight.datavisual.proto_files.mindinsight_lineage_pb2 import LineageEvent, UserDefinedInfo +from mindinsight.lineagemgr.common.exceptions.exceptions import LineageParamTypeError,\ + LineageParamValueError from mindinsight.lineagemgr.common.log import logger as log @@ -28,9 +29,9 @@ def package_dataset_graph(graph): graph (dict): Dataset graph. Returns: - Event, the proto message event contains dataset graph. + LineageEvent, the proto message event contains dataset graph. """ - dataset_graph_event = Event() + dataset_graph_event = LineageEvent() dataset_graph_event.wall_time = time.time() dataset_graph = dataset_graph_event.dataset_graph @@ -291,3 +292,57 @@ def _organize_parameter(parameter): parameter_result.update(result_str_list_para) return parameter_result + + +def package_user_defined_info(user_dict): + """ + Package user defined info. + + Args: + user_dict(dict): User defined info dict. + + Returns: + LineageEvent, the proto message event contains user defined info. + + """ + user_event = LineageEvent() + user_event.wall_time = time.time() + user_defined_info = user_event.user_defined_info + _package_user_defined_info(user_dict, user_defined_info) + + return user_event + + +def _package_user_defined_info(user_defined_dict, user_defined_message): + """ + Setting attribute in user defined proto message. + + Args: + user_defined_dict (dict): User define info dict. + user_defined_message (LineageEvent): Proto message of user defined info. + + Raises: + LineageParamValueError: When the value is out of range. + LineageParamTypeError: When given a type not support yet. + """ + for key, value in user_defined_dict.items(): + if not isinstance(key, str): + raise LineageParamTypeError("The key must be str.") + + if isinstance(value, int): + attr_name = "map_int32" + elif isinstance(value, float): + attr_name = "map_double" + elif isinstance(value, str): + attr_name = "map_str" + else: + error_msg = "Value type {} is not supported in user defined event package." \ + "Only str, int and float are permitted now.".format(type(value)) + log.error(error_msg) + raise LineageParamTypeError(error_msg) + + add_user_defined_info = user_defined_message.user_info.add() + try: + getattr(add_user_defined_info, attr_name)[key] = value + except ValueError: + raise LineageParamValueError("Value is out of range or not be supported yet.") \ No newline at end of file diff --git a/mindinsight/lineagemgr/summary/lineage_summary_analyzer.py b/mindinsight/lineagemgr/summary/lineage_summary_analyzer.py index c2948acf8647f3c0aeda85e672482fde6f855e27..9f9ad25cd2161561ee5a1671e655e59de9c6d727 100644 --- a/mindinsight/lineagemgr/summary/lineage_summary_analyzer.py +++ b/mindinsight/lineagemgr/summary/lineage_summary_analyzer.py @@ -17,7 +17,9 @@ import struct from collections import namedtuple from enum import Enum -from mindinsight.datavisual.proto_files.mindinsight_summary_pb2 import Event +from google.protobuf.json_format import MessageToDict + +from mindinsight.datavisual.proto_files.mindinsight_lineage_pb2 import LineageEvent from mindinsight.datavisual.utils import crc32 from mindinsight.lineagemgr.common.exceptions.exceptions import MindInsightException, \ LineageVerificationException, LineageSummaryAnalyzeException @@ -87,11 +89,11 @@ class SummaryAnalyzer: Read event. Returns: - Event, the event body. + LineageEvent, the event body. """ body_size = self._read_header() body_str = self._read_body(body_size) - event = Event().FromString(body_str) + event = LineageEvent().FromString(body_str) return event def _read_header(self): @@ -206,3 +208,51 @@ class LineageSummaryAnalyzer(SummaryAnalyzer): raise LineageSummaryAnalyzeException() return lineage_info + + @staticmethod + def get_user_defined_info(file_path): + """ + Get user defined info. + Args: + file_path (str): The file path of summary log. + + Returns: + list, the list of dict format user defined information + which converted from proto message. + """ + all_user_message = [] + summary_analyzer = SummaryAnalyzer(file_path) + + for event in summary_analyzer.load_events(): + if event.HasField("user_defined_info"): + user_defined_info = MessageToDict( + event, + preserving_proto_field_name=True + ).get("user_defined_info") + user_dict = LineageSummaryAnalyzer._get_dict_from_proto(user_defined_info) + all_user_message.append(user_dict) + + return all_user_message + + @staticmethod + def _get_dict_from_proto(user_defined_info): + """ + Convert the proto message UserDefinedInfo to its dict format. + + Args: + user_defined_info (UserDefinedInfo): The proto message of user defined info. + + Returns: + dict, the converted dict. + """ + user_dict = dict() + proto_dict = user_defined_info.get("user_info") + for proto_item in proto_dict: + if proto_item and isinstance(proto_item, dict): + key, value = list(list(proto_item.values())[0].items())[0] + if isinstance(value, dict): + user_dict[key] = LineageSummaryAnalyzer._get_dict_from_proto(value) + else: + user_dict[key] = value + + return user_dict diff --git a/mindinsight/lineagemgr/summary/summary_record.py b/mindinsight/lineagemgr/summary/summary_record.py index 5b48cdcc813b9b3310f6c14261086248af682746..da56323fb016c76353db5008d04b9408d7f939c9 100644 --- a/mindinsight/lineagemgr/summary/summary_record.py +++ b/mindinsight/lineagemgr/summary/summary_record.py @@ -15,9 +15,9 @@ """Record message to summary log.""" import time -from mindinsight.datavisual.proto_files.mindinsight_summary_pb2 import Event +from mindinsight.datavisual.proto_files.mindinsight_lineage_pb2 import LineageEvent from mindinsight.lineagemgr.summary.event_writer import EventWriter -from ._summary_adapter import package_dataset_graph +from ._summary_adapter import package_dataset_graph, package_user_defined_info class LineageSummary: @@ -50,9 +50,9 @@ class LineageSummary: run_context_args (dict): The train lineage info to log. Returns: - Event, the proto message event contains train lineage. + LineageEvent, the proto message event contains train lineage. """ - train_lineage_event = Event() + train_lineage_event = LineageEvent() train_lineage_event.wall_time = time.time() # Init train_lineage message. @@ -124,9 +124,9 @@ class LineageSummary: run_context_args (dict): The evaluation lineage info to log. Returns: - Event, the proto message event contains evaluation lineage. + LineageEvent, the proto message event contains evaluation lineage. """ - train_lineage_event = Event() + train_lineage_event = LineageEvent() train_lineage_event.wall_time = time.time() # Init evaluation_lineage message. @@ -165,3 +165,18 @@ class LineageSummary: self.event_writer.write_event_to_file( package_dataset_graph(dataset_graph).SerializeToString() ) + + def record_user_defined_info(self, user_dict): + """ + Write user defined info to summary log. + + Note: + The type of references must be dict, the value should be + int32, float, string. Nested dict is not supported now. + + Args: + user_dict (dict): The value user defined to be recorded. + """ + self.event_writer.write_event_to_file( + package_user_defined_info(user_dict).SerializeToString() + ) \ No newline at end of file diff --git a/tests/st/func/lineagemgr/api/test_model_api.py b/tests/st/func/lineagemgr/api/test_model_api.py index 46d5027bd3a525d69af4bcf4e2698dc2f73d9776..8b2df937c9cfd3ce2becc21e168c520acdac56b6 100644 --- a/tests/st/func/lineagemgr/api/test_model_api.py +++ b/tests/st/func/lineagemgr/api/test_model_api.py @@ -32,7 +32,8 @@ from mindinsight.lineagemgr.common.exceptions.exceptions import (LineageFileNotF LineageSearchConditionParamError) from ..conftest import BASE_SUMMARY_DIR, DATASET_GRAPH, SUMMARY_DIR, SUMMARY_DIR_2 - +from .....ut.lineagemgr.querier import event_data +from os import environ LINEAGE_INFO_RUN1 = { 'summary_dir': os.path.join(BASE_SUMMARY_DIR, 'run1'), 'metric': { @@ -68,6 +69,7 @@ LINEAGE_FILTRATION_EXCEPT_RUN = { 'loss_function': 'SoftmaxCrossEntropyWithLogits', 'train_dataset_path': None, 'train_dataset_count': 1024, + 'user_defined': {}, 'test_dataset_path': None, 'test_dataset_count': None, 'network': 'ResNet', @@ -87,6 +89,7 @@ LINEAGE_FILTRATION_RUN1 = { 'train_dataset_path': None, 'train_dataset_count': 731, 'test_dataset_path': None, + 'user_defined': {}, 'test_dataset_count': 10240, 'network': 'ResNet', 'optimizer': 'Momentum', @@ -106,6 +109,7 @@ LINEAGE_FILTRATION_RUN2 = { 'loss_function': None, 'train_dataset_path': None, 'train_dataset_count': None, + 'user_defined': {}, 'test_dataset_path': None, 'test_dataset_count': 10240, 'network': None, @@ -318,6 +322,7 @@ class TestModelApi(TestCase): def test_filter_summary_lineage(self): """Test the interface of filter_summary_lineage.""" expect_result = { + 'customized': event_data.CUSTOMIZED__0, 'object': [ LINEAGE_FILTRATION_EXCEPT_RUN, LINEAGE_FILTRATION_RUN1, @@ -360,16 +365,17 @@ class TestModelApi(TestCase): SUMMARY_DIR_2 ] }, - 'metric_accuracy': { + 'metric/accuracy': { 'lt': 3.0, 'gt': 0.5 }, - 'sorted_name': 'metric_accuracy', + 'sorted_name': 'metric/accuracy', 'sorted_type': 'descending', 'limit': 3, 'offset': 0 } expect_result = { + 'customized': event_data.CUSTOMIZED__0, 'object': [ LINEAGE_FILTRATION_RUN2, LINEAGE_FILTRATION_RUN1 @@ -397,16 +403,17 @@ class TestModelApi(TestCase): './run2' ] }, - 'metric_accuracy': { + 'metric/accuracy': { 'lt': 3.0, 'gt': 0.5 }, - 'sorted_name': 'metric_accuracy', + 'sorted_name': 'metric/accuracy', 'sorted_type': 'descending', 'limit': 3, 'offset': 0 } expect_result = { + 'customized': event_data.CUSTOMIZED__0, 'object': [ LINEAGE_FILTRATION_RUN2, LINEAGE_FILTRATION_RUN1 @@ -431,10 +438,11 @@ class TestModelApi(TestCase): 'batch_size': { 'ge': 30 }, - 'sorted_name': 'metric_accuracy', + 'sorted_name': 'metric/accuracy', 'lineage_type': None } expect_result = { + 'customized': event_data.CUSTOMIZED__0, 'object': [ LINEAGE_FILTRATION_EXCEPT_RUN, LINEAGE_FILTRATION_RUN1 @@ -454,6 +462,7 @@ class TestModelApi(TestCase): 'lineage_type': 'model' } expect_result = { + 'customized': {}, 'object': [], 'count': 0 } @@ -479,6 +488,7 @@ class TestModelApi(TestCase): 'lineage_type': 'dataset' } expect_result = { + 'customized': {}, 'object': [ { 'summary_dir': summary_dir, @@ -659,13 +669,13 @@ class TestModelApi(TestCase): # the search condition type error search_condition = { - 'metric_accuracy': { + 'metric/accuracy': { 'lt': 'xxx' } } self.assertRaisesRegex( LineageSearchConditionParamError, - 'The parameter metric_accuracy is invalid.', + 'The parameter metric/accuracy is invalid.', filter_summary_lineage, BASE_SUMMARY_DIR, search_condition @@ -741,12 +751,13 @@ class TestModelApi(TestCase): """Test the abnormal execution of the filter_summary_lineage interface.""" # gt > lt search_condition1 = { - 'metric_accuracy': { + 'metric/accuracy': { 'gt': 1, 'lt': 0.5 } } expect_result = { + 'customized': {}, 'object': [], 'count': 0 } @@ -762,6 +773,7 @@ class TestModelApi(TestCase): 'offset': 4 } expect_result = { + 'customized': {}, 'object': [], 'count': 1 } diff --git a/tests/ut/backend/lineagemgr/test_lineage_api.py b/tests/ut/backend/lineagemgr/test_lineage_api.py index 9d68b397a1fa4519d949cd8a61756d08c16505d6..5e80ff439487a90a7b8f7996b5e853247942f49f 100644 --- a/tests/ut/backend/lineagemgr/test_lineage_api.py +++ b/tests/ut/backend/lineagemgr/test_lineage_api.py @@ -29,6 +29,7 @@ LINEAGE_FILTRATION_BASE = { 'loss_function': 'SoftmaxCrossEntropyWithLogits', 'train_dataset_path': None, 'train_dataset_count': 64, + 'user_defined': {}, 'test_dataset_path': None, 'test_dataset_count': None, 'network': 'str', @@ -46,6 +47,7 @@ LINEAGE_FILTRATION_RUN1 = { 'loss_function': 'SoftmaxCrossEntropyWithLogits', 'train_dataset_path': None, 'train_dataset_count': 64, + 'user_defined': {}, 'test_dataset_path': None, 'test_dataset_count': 64, 'network': 'str', diff --git a/tests/ut/lineagemgr/common/validator/test_validate.py b/tests/ut/lineagemgr/common/validator/test_validate.py index ed5c76ccc2e86b9936262df4fe8d7a17d75d8554..64f4aae6947e75e2056e5702c55660f5598941da 100644 --- a/tests/ut/lineagemgr/common/validator/test_validate.py +++ b/tests/ut/lineagemgr/common/validator/test_validate.py @@ -228,12 +228,12 @@ class TestValidateSearchModelCondition(TestCase): ) condition = { - 'metric_attribute': { + 'metric/attribute': { 'ge': 'xxx' } } self._assert_raise_of_mindinsight_exception( - "The parameter metric_attribute is invalid. " + "The parameter metric/attribute is invalid. " "It should be a dict and the value should be a float or a integer", condition ) diff --git a/tests/ut/lineagemgr/querier/event_data.py b/tests/ut/lineagemgr/querier/event_data.py index 38a971b5ee5ad25c632f204bf2091a77aa136ce5..262bc9072cd38c6648ab386778c3f7ece8270b48 100644 --- a/tests/ut/lineagemgr/querier/event_data.py +++ b/tests/ut/lineagemgr/querier/event_data.py @@ -188,6 +188,22 @@ METRIC_0 = { 'mse': 3.00000001 } +CUSTOMIZED__0 = { + 'metric/accuracy': {'label': 'metric/accuracy', 'required': True, 'type': 'float'}, +} + +CUSTOMIZED_0 = { + **CUSTOMIZED__0, + 'metric/mae': {'label': 'metric/mae', 'required': True, 'type': 'float'}, + 'metric/mse': {'label': 'metric/mse', 'required': True, 'type': 'float'} +} + +CUSTOMIZED_1 = { + 'metric/accuracy': {'label': 'metric/accuracy', 'required': True, 'type': 'NoneType'}, + 'metric/mae': {'label': 'metric/mae', 'required': True, 'type': 'float'}, + 'metric/mse': {'label': 'metric/mse', 'required': True, 'type': 'float'} +} + METRIC_1 = { 'accuracy': 1.0000002, 'mae': 2.00000002, diff --git a/tests/ut/lineagemgr/querier/test_querier.py b/tests/ut/lineagemgr/querier/test_querier.py index 5e5d86c460c6b06b03cefc53ac9336be16ea0b06..8ededa0ed02b2bd2950ceb1c658cce8e413285a8 100644 --- a/tests/ut/lineagemgr/querier/test_querier.py +++ b/tests/ut/lineagemgr/querier/test_querier.py @@ -17,7 +17,7 @@ from unittest import TestCase, mock from google.protobuf.json_format import ParseDict -import mindinsight.datavisual.proto_files.mindinsight_summary_pb2 as summary_pb2 +import mindinsight.datavisual.proto_files.mindinsight_lineage_pb2 as summary_pb2 from mindinsight.lineagemgr.common.exceptions.exceptions import (LineageParamTypeError, LineageQuerierParamException, LineageSummaryAnalyzeException, LineageSummaryParseException) @@ -40,19 +40,19 @@ def create_lineage_info(train_event_dict, eval_event_dict, dataset_event_dict): namedtuple, parsed lineage info. """ if train_event_dict is not None: - train_event = summary_pb2.Event() + train_event = summary_pb2.LineageEvent() ParseDict(train_event_dict, train_event) else: train_event = None if eval_event_dict is not None: - eval_event = summary_pb2.Event() + eval_event = summary_pb2.LineageEvent() ParseDict(eval_event_dict, eval_event) else: eval_event = None if dataset_event_dict is not None: - dataset_event = summary_pb2.Event() + dataset_event = summary_pb2.LineageEvent() ParseDict(dataset_event_dict, dataset_event) else: dataset_event = None @@ -97,6 +97,7 @@ def create_filtration_result(summary_dir, train_event_dict, "metric": metric_dict, "dataset_graph": dataset_dict, "dataset_mark": '2', + "user_defined": {} } return filtration_result @@ -208,7 +209,9 @@ LINEAGE_FILTRATION_5 = { "model_size": event_data.EVENT_TRAIN_DICT_5['train_lineage']['model']['size'], "metric": {}, "dataset_graph": event_data.DATASET_DICT_0, - "dataset_mark": '2' + "dataset_mark": '2', + "user_defined": {} + } LINEAGE_FILTRATION_6 = { "summary_dir": '/path/to/summary6', @@ -228,12 +231,14 @@ LINEAGE_FILTRATION_6 = { "model_size": None, "metric": event_data.METRIC_5, "dataset_graph": event_data.DATASET_DICT_0, - "dataset_mark": '2' + "dataset_mark": '2', + "user_defined": {} } class TestQuerier(TestCase): """Test the class of `Querier`.""" + @mock.patch('mindinsight.lineagemgr.querier.querier.LineageSummaryAnalyzer.get_user_defined_info') @mock.patch('mindinsight.lineagemgr.querier.querier.LineageSummaryAnalyzer.get_summary_infos') def setUp(self, *args): """Initialization before test case execution.""" @@ -242,6 +247,7 @@ class TestQuerier(TestCase): event_data.EVENT_EVAL_DICT_0, event_data.EVENT_DATASET_DICT_0 ) + args[1].return_value = [] single_summary_path = '/path/to/summary0/log0' self.single_querier = Querier(single_summary_path) @@ -394,6 +400,7 @@ class TestQuerier(TestCase): 'sorted_name': 'summary_dir' } expected_result = { + 'customized': event_data.CUSTOMIZED_0, 'object': [ LINEAGE_FILTRATION_1, LINEAGE_FILTRATION_2 @@ -418,6 +425,7 @@ class TestQuerier(TestCase): 'sorted_type': 'descending' } expected_result = { + 'customized': event_data.CUSTOMIZED_0, 'object': [ LINEAGE_FILTRATION_2, LINEAGE_FILTRATION_3 @@ -434,6 +442,7 @@ class TestQuerier(TestCase): 'offset': 1 } expected_result = { + 'customized': event_data.CUSTOMIZED_0, 'object': [ LINEAGE_FILTRATION_2, LINEAGE_FILTRATION_3 @@ -446,6 +455,7 @@ class TestQuerier(TestCase): def test_filter_summary_lineage_success_4(self): """Test the success of filter_summary_lineage.""" expected_result = { + 'customized': event_data.CUSTOMIZED_0, 'object': [ LINEAGE_FILTRATION_0, LINEAGE_FILTRATION_1, @@ -468,6 +478,7 @@ class TestQuerier(TestCase): } } expected_result = { + 'customized': event_data.CUSTOMIZED_0, 'object': [LINEAGE_FILTRATION_4], 'count': 1, } @@ -477,10 +488,11 @@ class TestQuerier(TestCase): def test_filter_summary_lineage_success_6(self): """Test the success of filter_summary_lineage.""" condition = { - 'sorted_name': 'metric_accuracy', + 'sorted_name': 'metric/accuracy', 'sorted_type': 'ascending' } expected_result = { + 'customized': event_data.CUSTOMIZED_0, 'object': [ LINEAGE_FILTRATION_0, LINEAGE_FILTRATION_5, @@ -498,10 +510,11 @@ class TestQuerier(TestCase): def test_filter_summary_lineage_success_7(self): """Test the success of filter_summary_lineage.""" condition = { - 'sorted_name': 'metric_accuracy', + 'sorted_name': 'metric/accuracy', 'sorted_type': 'descending' } expected_result = { + 'customized': event_data.CUSTOMIZED_1, 'object': [ LINEAGE_FILTRATION_6, LINEAGE_FILTRATION_4, @@ -519,12 +532,13 @@ class TestQuerier(TestCase): def test_filter_summary_lineage_success_8(self): """Test the success of filter_summary_lineage.""" condition = { - 'metric_accuracy': { + 'metric/accuracy': { 'lt': 1.0000006, 'gt': 1.0000004 } } expected_result = { + 'customized': event_data.CUSTOMIZED_0, 'object': [LINEAGE_FILTRATION_4], 'count': 1, } @@ -538,6 +552,7 @@ class TestQuerier(TestCase): 'offset': 3 } expected_result = { + 'customized': {}, 'object': [], 'count': 7, } @@ -594,11 +609,13 @@ class TestQuerier(TestCase): with self.assertRaises(LineageSummaryParseException): Querier(summary_path) + @mock.patch('mindinsight.lineagemgr.querier.querier.LineageSummaryAnalyzer.get_user_defined_info') @mock.patch('mindinsight.lineagemgr.querier.querier.LineageSummaryAnalyzer.get_summary_infos') def test_parse_fail_summary_logs_1(self, *args): """Test the function of parsing fail summary logs.""" lineage_infos = get_lineage_infos() args[0].side_effect = lineage_infos + args[1].return_value = [] summary_path = ['/path/to/summary0/log0'] querier = Querier(summary_path) @@ -611,6 +628,7 @@ class TestQuerier(TestCase): self.assertListEqual(expected_result, result) self.assertListEqual([], querier._parse_failed_paths) + @mock.patch('mindinsight.lineagemgr.querier.querier.LineageSummaryAnalyzer.get_user_defined_info') @mock.patch('mindinsight.lineagemgr.querier.querier.LineageSummaryAnalyzer.get_summary_infos') def test_parse_fail_summary_logs_2(self, *args): """Test the function of parsing fail summary logs.""" @@ -619,6 +637,7 @@ class TestQuerier(TestCase): event_data.EVENT_EVAL_DICT_0, event_data.EVENT_DATASET_DICT_0, ) + args[1].return_value = [] summary_path = ['/path/to/summary0/log0'] querier = Querier(summary_path) diff --git a/tests/ut/lineagemgr/summary/test_lineage_summary_analyze.py b/tests/ut/lineagemgr/summary/test_lineage_summary_analyze.py index cda64cf2eb2025d9acbf609296e9719859cce0db..f635c6697baf678879c93da24f6692daaf10129e 100644 --- a/tests/ut/lineagemgr/summary/test_lineage_summary_analyze.py +++ b/tests/ut/lineagemgr/summary/test_lineage_summary_analyze.py @@ -16,7 +16,7 @@ from unittest import mock, TestCase from unittest.mock import MagicMock -from mindinsight.datavisual.proto_files.mindinsight_summary_pb2 import Event +from mindinsight.datavisual.proto_files.mindinsight_lineage_pb2 import LineageEvent from mindinsight.lineagemgr.common.exceptions.exceptions import LineageVerificationException, \ LineageSummaryAnalyzeException from mindinsight.lineagemgr.common.log import logger as log @@ -57,7 +57,7 @@ class TestSummaryAnalyzer(TestCase): @mock.patch.object(SummaryAnalyzer, '_read_header') @mock.patch.object(SummaryAnalyzer, '_read_body') - @mock.patch.object(Event, 'FromString') + @mock.patch.object(LineageEvent, 'FromString') def test_read_event(self, *args): """Test read_event method.""" args[2].return_value = 10