diff --git a/mindinsight/backend/datavisual/train_visual_api.py b/mindinsight/backend/datavisual/train_visual_api.py index fa1462bc3ee881da3d242df418af9dc8c13a44b4..72bb45eabc0d2db9506a2761af077a9609de8fe3 100644 --- a/mindinsight/backend/datavisual/train_visual_api.py +++ b/mindinsight/backend/datavisual/train_visual_api.py @@ -24,6 +24,7 @@ from flask import jsonify from mindinsight.conf import settings from mindinsight.datavisual.utils.tools import get_train_id from mindinsight.datavisual.utils.tools import if_nan_inf_to_none +from mindinsight.datavisual.processors.histogram_processor import HistogramProcessor from mindinsight.datavisual.processors.images_processor import ImageProcessor from mindinsight.datavisual.processors.scalars_processor import ScalarsProcessor from mindinsight.datavisual.processors.graph_processor import GraphProcessor @@ -146,6 +147,22 @@ def graph_search_single_node(): return jsonify(resp) +@BLUEPRINT.route("/datavisual/histograms", methods=["GET"]) +def histogram(): + """ + Interface to obtain histogram data. + + Returns: + Response, which contains a JSON object. + """ + tag = request.args.get("tag", default=None) + train_id = get_train_id(request) + + processor = HistogramProcessor(DATA_MANAGER) + response = processor.get_histograms(train_id, tag) + return jsonify(response) + + def init_module(app): """ Init module entry. diff --git a/mindinsight/conf/constants.py b/mindinsight/conf/constants.py index bd8dd72a13ffaf5338376e8a134860ea4d79f21d..66e09a39c62643b0b9a760e099fdc2b3aefe3202 100644 --- a/mindinsight/conf/constants.py +++ b/mindinsight/conf/constants.py @@ -56,3 +56,4 @@ MAX_GRAPH_TAG_SIZE = 10 MAX_IMAGE_STEP_SIZE_PER_TAG = 10 MAX_SCALAR_STEP_SIZE_PER_TAG = 1000 MAX_GRAPH_STEP_SIZE_PER_TAG = 1 +MAX_HISTOGRAM_STEP_SIZE_PER_TAG = 50 diff --git a/mindinsight/datavisual/common/enums.py b/mindinsight/datavisual/common/enums.py index f5347f5626a679f0854ba064afda62cff3739c9c..6dbea990a21c47e359112bdf1b321fb53ba95739 100644 --- a/mindinsight/datavisual/common/enums.py +++ b/mindinsight/datavisual/common/enums.py @@ -37,3 +37,4 @@ class PluginNameEnum(BaseEnum): IMAGE = 'image' SCALAR = 'scalar' GRAPH = 'graph' + HISTOGRAM = 'histogram' diff --git a/mindinsight/datavisual/common/exceptions.py b/mindinsight/datavisual/common/exceptions.py index 046e233d7a45e9894266dfc88e539e6d980e4953..310bc48b396541b36ca2f6bc093b9d27fbffb550 100644 --- a/mindinsight/datavisual/common/exceptions.py +++ b/mindinsight/datavisual/common/exceptions.py @@ -141,3 +141,12 @@ class ScalarNotExistError(MindInsightException): super(ScalarNotExistError, self).__init__(DataVisualErrors.SCALAR_NOT_EXIST, error_msg, http_code=400) + + +class HistogramNotExistError(MindInsightException): + """Unable to get histogram values based on a given condition.""" + def __init__(self, error_detail): + error_msg = f'Histogram value is not exist. Detail: {error_detail}' + super(HistogramNotExistError, self).__init__(DataVisualErrors.HISTOGRAM_NOT_EXIST, + error_msg, + http_code=400) diff --git a/mindinsight/datavisual/data_transform/events_data.py b/mindinsight/datavisual/data_transform/events_data.py index 86150b8cf0a1cf035c217bff731ff46953c774fb..76fca275a1b0290cba7a07aee8074a6732173a81 100644 --- a/mindinsight/datavisual/data_transform/events_data.py +++ b/mindinsight/datavisual/data_transform/events_data.py @@ -41,6 +41,7 @@ CONFIG = { PluginNameEnum.SCALAR.value: settings.MAX_SCALAR_STEP_SIZE_PER_TAG, PluginNameEnum.IMAGE.value: settings.MAX_IMAGE_STEP_SIZE_PER_TAG, PluginNameEnum.GRAPH.value: settings.MAX_GRAPH_STEP_SIZE_PER_TAG, + PluginNameEnum.HISTOGRAM.value: settings.MAX_HISTOGRAM_STEP_SIZE_PER_TAG } } diff --git a/mindinsight/datavisual/data_transform/ms_data_loader.py b/mindinsight/datavisual/data_transform/ms_data_loader.py index adb8e925b70ff6821944c801395c57136397b8e0..0aea3bc3ee2cd31dbe54845677dc489ec6377add 100644 --- a/mindinsight/datavisual/data_transform/ms_data_loader.py +++ b/mindinsight/datavisual/data_transform/ms_data_loader.py @@ -234,6 +234,16 @@ class MSDataLoader: value=value.image) self._events_data.add_tensor_event(tensor_event) + if value.HasField('histogram'): + histogram_msg = value.histogram + tag = '{}/{}'.format(value.tag, PluginNameEnum.HISTOGRAM.value) + tensor_event = TensorEvent(wall_time=event.wall_time, + step=event.step, + tag=tag, + plugin_name=PluginNameEnum.HISTOGRAM.value, + value=histogram_msg) + self._events_data.add_tensor_event(tensor_event) + if event.HasField('graph_def'): graph_proto = event.graph_def graph = MSGraph() diff --git a/mindinsight/datavisual/processors/histogram_processor.py b/mindinsight/datavisual/processors/histogram_processor.py new file mode 100644 index 0000000000000000000000000000000000000000..26d99e690eed880427f589f2af4dde3aad9f7c9f --- /dev/null +++ b/mindinsight/datavisual/processors/histogram_processor.py @@ -0,0 +1,71 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Histogram Processor APIs.""" +from mindinsight.utils.exceptions import ParamValueError +from mindinsight.datavisual.common.log import logger +from mindinsight.datavisual.common.validation import Validation +from mindinsight.datavisual.common.exceptions import HistogramNotExistError +from mindinsight.datavisual.processors.base_processor import BaseProcessor + + +class HistogramProcessor(BaseProcessor): + """Histogram Processor.""" + def get_histograms(self, train_id, tag): + """ + Builds a JSON-serializable object with information about histogram data. + + Args: + train_id (str): The ID of the events data. + tag (str): The name of the tag the histogram data all belong to. + + Returns: + dict, a dict including the `train_id`, `tag`, and `histograms'. + { + "train_id": ****, + "tag": ****, + "histograms": [{ + "wall_time": ****, + "step": ****, + "bucket": [[**, **, **]], + }, + {...} + ] + } + """ + Validation.check_param_empty(train_id=train_id, tag=tag) + logger.info("Start to process histogram data...") + try: + tensors = self._data_manager.list_tensors(train_id, tag) + except ParamValueError as err: + raise HistogramNotExistError(err.message) + + histograms = [] + for tensor in tensors: + buckets = [] + for bucket in tensor.value.buckets: + buckets.append([bucket.left, bucket.width, bucket.count]) + histograms.append({ + "wall_time": tensor.wall_time, + "step": tensor.step, + "buckets": buckets + }) + + logger.info("Histogram data processing is finished!") + response = { + "train_id": train_id, + "tag": tag, + "histograms": histograms + } + return response diff --git a/mindinsight/datavisual/proto_files/mindinsight_summary.proto b/mindinsight/datavisual/proto_files/mindinsight_summary.proto index 60e7f8af746de4fd7c209168cf3c70d15e9d3c32..2b79907b789d6ed2be5ff252eb47995643c781c2 100644 --- a/mindinsight/datavisual/proto_files/mindinsight_summary.proto +++ b/mindinsight/datavisual/proto_files/mindinsight_summary.proto @@ -61,6 +61,30 @@ message Summary { required bytes encoded_image = 4; } + message Histogram { + message bucket{ + // Counting number of values fallen in [left, left + width). + // For the rightmost bucket, the range is [left, left + width]. + required double left = 1; + required double width = 2; + required int64 count = 3; + } + + repeated bucket buckets = 1; + optional int64 nan_count = 2; + optional int64 pos_inf_count = 3; + optional int64 neg_inf_count = 4; + + // max, min, sum will not take nan and inf into account. + // If there is no valid value in tensor, max and min will be nan, sum will be 0. + optional double max = 5; + optional double min = 6; + optional double sum = 7; + + // total number of values. including nan and inf. + optional int64 count = 8; + } + message Value { // Tag name for the data. required string tag = 1; @@ -70,6 +94,7 @@ message Summary { float scalar_value = 3; Image image = 4; TensorProto tensor = 8; + Histogram histogram = 9; } } diff --git a/mindinsight/datavisual/proto_files/mindinsight_summary_pb2.py b/mindinsight/datavisual/proto_files/mindinsight_summary_pb2.py index 1e4528f84f8ebe97c27469aa40ced6dc86b30172..ba45bbb9d2f492784b752c49519a004ddb120a5a 100644 --- a/mindinsight/datavisual/proto_files/mindinsight_summary_pb2.py +++ b/mindinsight/datavisual/proto_files/mindinsight_summary_pb2.py @@ -1,13 +1,11 @@ +# -*- coding: utf-8 -*- # Generated by the protocol buffer compiler. DO NOT EDIT! # source: mindinsight_summary.proto -import sys -_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) from google.protobuf import descriptor as _descriptor from google.protobuf import message as _message from google.protobuf import reflection as _reflection from google.protobuf import symbol_database as _symbol_database -from google.protobuf import descriptor_pb2 # @@protoc_insertion_point(imports) _sym_db = _symbol_database.Default() @@ -20,10 +18,10 @@ DESCRIPTOR = _descriptor.FileDescriptor( name='mindinsight_summary.proto', package='mindinsight', syntax='proto2', - serialized_pb=_b('\n\x19mindinsight_summary.proto\x12\x0bmindinsight\x1a\x18mindinsight_anf_ir.proto\"\x9a\x01\n\x05\x45vent\x12\x11\n\twall_time\x18\x01 \x02(\x01\x12\x0c\n\x04step\x18\x02 \x01(\x03\x12\x11\n\x07version\x18\x03 \x01(\tH\x00\x12,\n\tgraph_def\x18\x04 \x01(\x0b\x32\x17.mindinsight.GraphProtoH\x00\x12\'\n\x07summary\x18\x05 \x01(\x0b\x32\x14.mindinsight.SummaryH\x00\x42\x06\n\x04what\"\x98\x02\n\x07Summary\x12)\n\x05value\x18\x01 \x03(\x0b\x32\x1a.mindinsight.Summary.Value\x1aQ\n\x05Image\x12\x0e\n\x06height\x18\x01 \x02(\x05\x12\r\n\x05width\x18\x02 \x02(\x05\x12\x12\n\ncolorspace\x18\x03 \x02(\x05\x12\x15\n\rencoded_image\x18\x04 \x02(\x0c\x1a\x8e\x01\n\x05Value\x12\x0b\n\x03tag\x18\x01 \x02(\t\x12\x16\n\x0cscalar_value\x18\x03 \x01(\x02H\x00\x12+\n\x05image\x18\x04 \x01(\x0b\x32\x1a.mindinsight.Summary.ImageH\x00\x12*\n\x06tensor\x18\x08 \x01(\x0b\x32\x18.mindinsight.TensorProtoH\x00\x42\x07\n\x05valueB\x03\xf8\x01\x01') + serialized_options=b'\370\001\001', + serialized_pb=b'\n\x19mindinsight_summary.proto\x12\x0bmindinsight\x1a\x18mindinsight_anf_ir.proto\"\x9a\x01\n\x05\x45vent\x12\x11\n\twall_time\x18\x01 \x02(\x01\x12\x0c\n\x04step\x18\x02 \x01(\x03\x12\x11\n\x07version\x18\x03 \x01(\tH\x00\x12,\n\tgraph_def\x18\x04 \x01(\x0b\x32\x17.mindinsight.GraphProtoH\x00\x12\'\n\x07summary\x18\x05 \x01(\x0b\x32\x14.mindinsight.SummaryH\x00\x42\x06\n\x04what\"\xc0\x04\n\x07Summary\x12)\n\x05value\x18\x01 \x03(\x0b\x32\x1a.mindinsight.Summary.Value\x1aQ\n\x05Image\x12\x0e\n\x06height\x18\x01 \x02(\x05\x12\r\n\x05width\x18\x02 \x02(\x05\x12\x12\n\ncolorspace\x18\x03 \x02(\x05\x12\x15\n\rencoded_image\x18\x04 \x02(\x0c\x1a\xf0\x01\n\tHistogram\x12\x36\n\x07\x62uckets\x18\x01 \x03(\x0b\x32%.mindinsight.Summary.Histogram.bucket\x12\x11\n\tnan_count\x18\x02 \x01(\x03\x12\x15\n\rpos_inf_count\x18\x03 \x01(\x03\x12\x15\n\rneg_inf_count\x18\x04 \x01(\x03\x12\x0b\n\x03max\x18\x05 \x01(\x01\x12\x0b\n\x03min\x18\x06 \x01(\x01\x12\x0b\n\x03sum\x18\x07 \x01(\x01\x12\r\n\x05\x63ount\x18\x08 \x01(\x03\x1a\x34\n\x06\x62ucket\x12\x0c\n\x04left\x18\x01 \x02(\x01\x12\r\n\x05width\x18\x02 \x02(\x01\x12\r\n\x05\x63ount\x18\x03 \x02(\x03\x1a\xc3\x01\n\x05Value\x12\x0b\n\x03tag\x18\x01 \x02(\t\x12\x16\n\x0cscalar_value\x18\x03 \x01(\x02H\x00\x12+\n\x05image\x18\x04 \x01(\x0b\x32\x1a.mindinsight.Summary.ImageH\x00\x12*\n\x06tensor\x18\x08 \x01(\x0b\x32\x18.mindinsight.TensorProtoH\x00\x12\x33\n\thistogram\x18\t \x01(\x0b\x32\x1e.mindinsight.Summary.HistogramH\x00\x42\x07\n\x05valueB\x03\xf8\x01\x01' , dependencies=[mindinsight__anf__ir__pb2.DESCRIPTOR,]) -_sym_db.RegisterFileDescriptor(DESCRIPTOR) @@ -41,42 +39,42 @@ _EVENT = _descriptor.Descriptor( has_default_value=False, default_value=float(0), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - options=None), + serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='step', full_name='mindinsight.Event.step', index=1, number=2, type=3, cpp_type=2, label=1, has_default_value=False, default_value=0, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - options=None), + serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='version', full_name='mindinsight.Event.version', index=2, number=3, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), + has_default_value=False, default_value=b"".decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - options=None), + serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='graph_def', full_name='mindinsight.Event.graph_def', index=3, number=4, type=11, cpp_type=10, label=1, has_default_value=False, default_value=None, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - options=None), + serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='summary', full_name='mindinsight.Event.summary', index=4, number=5, type=11, cpp_type=10, label=1, has_default_value=False, default_value=None, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - options=None), + serialized_options=None, file=DESCRIPTOR), ], extensions=[ ], nested_types=[], enum_types=[ ], - options=None, + serialized_options=None, is_extendable=False, syntax='proto2', extension_ranges=[], @@ -103,35 +101,35 @@ _SUMMARY_IMAGE = _descriptor.Descriptor( has_default_value=False, default_value=0, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - options=None), + serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='width', full_name='mindinsight.Summary.Image.width', index=1, number=2, type=5, cpp_type=1, label=2, has_default_value=False, default_value=0, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - options=None), + serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='colorspace', full_name='mindinsight.Summary.Image.colorspace', index=2, number=3, type=5, cpp_type=1, label=2, has_default_value=False, default_value=0, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - options=None), + serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='encoded_image', full_name='mindinsight.Summary.Image.encoded_image', index=3, number=4, type=12, cpp_type=9, label=2, - has_default_value=False, default_value=_b(""), + has_default_value=False, default_value=b"", message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - options=None), + serialized_options=None, file=DESCRIPTOR), ], extensions=[ ], nested_types=[], enum_types=[ ], - options=None, + serialized_options=None, is_extendable=False, syntax='proto2', extension_ranges=[], @@ -141,6 +139,129 @@ _SUMMARY_IMAGE = _descriptor.Descriptor( serialized_end=361, ) +_SUMMARY_HISTOGRAM_BUCKET = _descriptor.Descriptor( + name='bucket', + full_name='mindinsight.Summary.Histogram.bucket', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='left', full_name='mindinsight.Summary.Histogram.bucket.left', index=0, + number=1, type=1, cpp_type=5, label=2, + has_default_value=False, default_value=float(0), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='width', full_name='mindinsight.Summary.Histogram.bucket.width', index=1, + number=2, type=1, cpp_type=5, label=2, + has_default_value=False, default_value=float(0), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='count', full_name='mindinsight.Summary.Histogram.bucket.count', index=2, + number=3, type=3, cpp_type=2, label=2, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto2', + extension_ranges=[], + oneofs=[ + ], + serialized_start=552, + serialized_end=604, +) + +_SUMMARY_HISTOGRAM = _descriptor.Descriptor( + name='Histogram', + full_name='mindinsight.Summary.Histogram', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='buckets', full_name='mindinsight.Summary.Histogram.buckets', index=0, + number=1, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='nan_count', full_name='mindinsight.Summary.Histogram.nan_count', index=1, + number=2, type=3, cpp_type=2, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='pos_inf_count', full_name='mindinsight.Summary.Histogram.pos_inf_count', index=2, + number=3, type=3, cpp_type=2, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='neg_inf_count', full_name='mindinsight.Summary.Histogram.neg_inf_count', index=3, + number=4, type=3, cpp_type=2, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='max', full_name='mindinsight.Summary.Histogram.max', index=4, + number=5, type=1, cpp_type=5, label=1, + has_default_value=False, default_value=float(0), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='min', full_name='mindinsight.Summary.Histogram.min', index=5, + number=6, type=1, cpp_type=5, label=1, + has_default_value=False, default_value=float(0), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='sum', full_name='mindinsight.Summary.Histogram.sum', index=6, + number=7, type=1, cpp_type=5, label=1, + has_default_value=False, default_value=float(0), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='count', full_name='mindinsight.Summary.Histogram.count', index=7, + number=8, type=3, cpp_type=2, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), + ], + extensions=[ + ], + nested_types=[_SUMMARY_HISTOGRAM_BUCKET, ], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto2', + extension_ranges=[], + oneofs=[ + ], + serialized_start=364, + serialized_end=604, +) + _SUMMARY_VALUE = _descriptor.Descriptor( name='Value', full_name='mindinsight.Summary.Value', @@ -151,38 +272,45 @@ _SUMMARY_VALUE = _descriptor.Descriptor( _descriptor.FieldDescriptor( name='tag', full_name='mindinsight.Summary.Value.tag', index=0, number=1, type=9, cpp_type=9, label=2, - has_default_value=False, default_value=_b("").decode('utf-8'), + has_default_value=False, default_value=b"".decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - options=None), + serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='scalar_value', full_name='mindinsight.Summary.Value.scalar_value', index=1, number=3, type=2, cpp_type=6, label=1, has_default_value=False, default_value=float(0), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - options=None), + serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='image', full_name='mindinsight.Summary.Value.image', index=2, number=4, type=11, cpp_type=10, label=1, has_default_value=False, default_value=None, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - options=None), + serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( name='tensor', full_name='mindinsight.Summary.Value.tensor', index=3, number=8, type=11, cpp_type=10, label=1, has_default_value=False, default_value=None, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - options=None), + serialized_options=None, file=DESCRIPTOR), + _descriptor.FieldDescriptor( + name='histogram', full_name='mindinsight.Summary.Value.histogram', index=4, + number=9, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR), ], extensions=[ ], nested_types=[], enum_types=[ ], - options=None, + serialized_options=None, is_extendable=False, syntax='proto2', extension_ranges=[], @@ -191,8 +319,8 @@ _SUMMARY_VALUE = _descriptor.Descriptor( name='value', full_name='mindinsight.Summary.Value.value', index=0, containing_type=None, fields=[]), ], - serialized_start=364, - serialized_end=506, + serialized_start=607, + serialized_end=802, ) _SUMMARY = _descriptor.Descriptor( @@ -208,21 +336,21 @@ _SUMMARY = _descriptor.Descriptor( has_default_value=False, default_value=[], message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, - options=None), + serialized_options=None, file=DESCRIPTOR), ], extensions=[ ], - nested_types=[_SUMMARY_IMAGE, _SUMMARY_VALUE, ], + nested_types=[_SUMMARY_IMAGE, _SUMMARY_HISTOGRAM, _SUMMARY_VALUE, ], enum_types=[ ], - options=None, + serialized_options=None, is_extendable=False, syntax='proto2', extension_ranges=[], oneofs=[ ], serialized_start=226, - serialized_end=506, + serialized_end=802, ) _EVENT.fields_by_name['graph_def'].message_type = mindinsight__anf__ir__pb2._GRAPHPROTO @@ -237,8 +365,12 @@ _EVENT.oneofs_by_name['what'].fields.append( _EVENT.fields_by_name['summary']) _EVENT.fields_by_name['summary'].containing_oneof = _EVENT.oneofs_by_name['what'] _SUMMARY_IMAGE.containing_type = _SUMMARY +_SUMMARY_HISTOGRAM_BUCKET.containing_type = _SUMMARY_HISTOGRAM +_SUMMARY_HISTOGRAM.fields_by_name['buckets'].message_type = _SUMMARY_HISTOGRAM_BUCKET +_SUMMARY_HISTOGRAM.containing_type = _SUMMARY _SUMMARY_VALUE.fields_by_name['image'].message_type = _SUMMARY_IMAGE _SUMMARY_VALUE.fields_by_name['tensor'].message_type = mindinsight__anf__ir__pb2._TENSORPROTO +_SUMMARY_VALUE.fields_by_name['histogram'].message_type = _SUMMARY_HISTOGRAM _SUMMARY_VALUE.containing_type = _SUMMARY _SUMMARY_VALUE.oneofs_by_name['value'].fields.append( _SUMMARY_VALUE.fields_by_name['scalar_value']) @@ -249,41 +381,60 @@ _SUMMARY_VALUE.fields_by_name['image'].containing_oneof = _SUMMARY_VALUE.oneofs_ _SUMMARY_VALUE.oneofs_by_name['value'].fields.append( _SUMMARY_VALUE.fields_by_name['tensor']) _SUMMARY_VALUE.fields_by_name['tensor'].containing_oneof = _SUMMARY_VALUE.oneofs_by_name['value'] +_SUMMARY_VALUE.oneofs_by_name['value'].fields.append( + _SUMMARY_VALUE.fields_by_name['histogram']) +_SUMMARY_VALUE.fields_by_name['histogram'].containing_oneof = _SUMMARY_VALUE.oneofs_by_name['value'] _SUMMARY.fields_by_name['value'].message_type = _SUMMARY_VALUE DESCRIPTOR.message_types_by_name['Event'] = _EVENT DESCRIPTOR.message_types_by_name['Summary'] = _SUMMARY +_sym_db.RegisterFileDescriptor(DESCRIPTOR) -Event = _reflection.GeneratedProtocolMessageType('Event', (_message.Message,), dict( - DESCRIPTOR = _EVENT, - __module__ = 'mindinsight_summary_pb2' +Event = _reflection.GeneratedProtocolMessageType('Event', (_message.Message,), { + 'DESCRIPTOR' : _EVENT, + '__module__' : 'mindinsight_summary_pb2' # @@protoc_insertion_point(class_scope:mindinsight.Event) - )) + }) _sym_db.RegisterMessage(Event) -Summary = _reflection.GeneratedProtocolMessageType('Summary', (_message.Message,), dict( +Summary = _reflection.GeneratedProtocolMessageType('Summary', (_message.Message,), { - Image = _reflection.GeneratedProtocolMessageType('Image', (_message.Message,), dict( - DESCRIPTOR = _SUMMARY_IMAGE, - __module__ = 'mindinsight_summary_pb2' + 'Image' : _reflection.GeneratedProtocolMessageType('Image', (_message.Message,), { + 'DESCRIPTOR' : _SUMMARY_IMAGE, + '__module__' : 'mindinsight_summary_pb2' # @@protoc_insertion_point(class_scope:mindinsight.Summary.Image) - )) + }) + , + + 'Histogram' : _reflection.GeneratedProtocolMessageType('Histogram', (_message.Message,), { + + 'bucket' : _reflection.GeneratedProtocolMessageType('bucket', (_message.Message,), { + 'DESCRIPTOR' : _SUMMARY_HISTOGRAM_BUCKET, + '__module__' : 'mindinsight_summary_pb2' + # @@protoc_insertion_point(class_scope:mindinsight.Summary.Histogram.bucket) + }) + , + 'DESCRIPTOR' : _SUMMARY_HISTOGRAM, + '__module__' : 'mindinsight_summary_pb2' + # @@protoc_insertion_point(class_scope:mindinsight.Summary.Histogram) + }) , - Value = _reflection.GeneratedProtocolMessageType('Value', (_message.Message,), dict( - DESCRIPTOR = _SUMMARY_VALUE, - __module__ = 'mindinsight_summary_pb2' + 'Value' : _reflection.GeneratedProtocolMessageType('Value', (_message.Message,), { + 'DESCRIPTOR' : _SUMMARY_VALUE, + '__module__' : 'mindinsight_summary_pb2' # @@protoc_insertion_point(class_scope:mindinsight.Summary.Value) - )) + }) , - DESCRIPTOR = _SUMMARY, - __module__ = 'mindinsight_summary_pb2' + 'DESCRIPTOR' : _SUMMARY, + '__module__' : 'mindinsight_summary_pb2' # @@protoc_insertion_point(class_scope:mindinsight.Summary) - )) + }) _sym_db.RegisterMessage(Summary) _sym_db.RegisterMessage(Summary.Image) +_sym_db.RegisterMessage(Summary.Histogram) +_sym_db.RegisterMessage(Summary.Histogram.bucket) _sym_db.RegisterMessage(Summary.Value) -DESCRIPTOR.has_options = True -DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), _b('\370\001\001')) +DESCRIPTOR._options = None # @@protoc_insertion_point(module_scope) diff --git a/mindinsight/utils/constant.py b/mindinsight/utils/constant.py index d6ec95f8209e09f8fc895a71e89285a8b3037f29..97c860c4437e4daa302c5d3966a050b85f4f0915 100644 --- a/mindinsight/utils/constant.py +++ b/mindinsight/utils/constant.py @@ -62,3 +62,4 @@ class DataVisualErrors(Enum): GRAPH_NOT_EXIST = 12 IMAGE_NOT_EXIST = 13 SCALAR_NOT_EXIST = 14 + HISTOGRAM_NOT_EXIST = 15 diff --git a/tests/st/func/datavisual/conftest.py b/tests/st/func/datavisual/conftest.py index bc0b2292d52b69b1c14bfc453c5b92f8545734a5..e7053d6b7fdafdb1408819bca9e94af47017e3b4 100644 --- a/tests/st/func/datavisual/conftest.py +++ b/tests/st/func/datavisual/conftest.py @@ -65,6 +65,7 @@ def init_summary_logs(): summaries_metadata.update( log_operations.create_summary_logs(summary_base_dir, constants.SUMMARY_DIR_NUM_SECOND, + constants.SUMMARY_DIR_PREFIX, constants.SUMMARY_DIR_NUM_FIRST)) summaries_metadata.update( log_operations.create_multiple_logs(summary_base_dir, constants.MULTIPLE_DIR_NAME, diff --git a/tests/ut/datavisual/processors/test_train_task_manager.py b/tests/ut/datavisual/processors/test_train_task_manager.py index 131f6724a3553203e121cf111a9d4fe5d9570921..55761f96de8646801610201dfec5336f6c71d54e 100644 --- a/tests/ut/datavisual/processors/test_train_task_manager.py +++ b/tests/ut/datavisual/processors/test_train_task_manager.py @@ -70,7 +70,7 @@ class TestTrainTaskManager: def load_data(self): """Load data.""" log_operation = LogOperations() - self._plugins_id_map = {'image': [], 'scalar': [], 'graph': []} + self._plugins_id_map = {'image': [], 'scalar': [], 'graph': [], 'histogram': []} self._events_names = [] self._train_id_list = [] diff --git a/tests/utils/log_generators/histogram_log_generator.py b/tests/utils/log_generators/histogram_log_generator.py new file mode 100644 index 0000000000000000000000000000000000000000..5d6e2abafa185f53dd6a24a06841a17c7a6f8d58 --- /dev/null +++ b/tests/utils/log_generators/histogram_log_generator.py @@ -0,0 +1,114 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Log generator for histogram data.""" +import time + +import numpy as np + +from mindinsight.datavisual.proto_files import mindinsight_summary_pb2 as summary_pb2 + +from .log_generator import LogGenerator + + +class HistogramLogGenerator(LogGenerator): + """ + Log generator for histogram data. + + This is a log generator writing histogram data. User can use it to generate fake + summary logs about histogram. + """ + + def generate_event(self, values): + """ + Method for generating histogram event. + + Args: + values (dict): A dict contains: + { + wall_time (float): Timestamp. + step (int): Train step. + value (float): Histogram value. + tag (str): Tag name. + } + + Returns: + summary_pb2.Event. + + """ + histogram_event = summary_pb2.Event() + histogram_event.wall_time = values.get('wall_time') + histogram_event.step = values.get('step') + + value = histogram_event.summary.value.add() + value.tag = values.get('tag') + + buckets = values.get('buckets') + for bucket in buckets: + left, width, count = bucket + bucket = value.histogram.buckets.add() + bucket.left = left + bucket.width = width + bucket.count = count + + return histogram_event + + def generate_log(self, file_path, steps_list, tag_name): + """ + Generate log for external calls. + + Args: + file_path (str): Path to write logs. + steps_list (list): A list consists of step. + tag_name (str): Tag name. + + Returns: + list[dict], generated histogram metadata. + None, to be consistent with return value of HistogramGenerator. + + """ + histogram_metadata = [] + for step in steps_list: + histogram = dict() + + wall_time = time.time() + histogram.update({'wall_time': wall_time}) + histogram.update({'step': step}) + histogram.update({'tag': tag_name}) + + # Construct buckets + buckets = [] + leftmost = list(np.random.randn(11)) + leftmost.sort() + for i in range(10): + left = leftmost[i] + width = leftmost[i+1] - left + count = np.random.randint(20) + bucket = [left, width, count] + buckets.append(bucket) + + histogram.update({'buckets': buckets}) + histogram_metadata.append(histogram) + + self._write_log_one_step(file_path, histogram) + + return histogram_metadata, None + + +if __name__ == "__main__": + histogram_log_generator = HistogramLogGenerator() + test_file_name = '%s.%s.%s' % ('histogram', 'summary', str(time.time())) + test_steps = [1, 3, 5] + test_tag = "test_histogram_tag_name" + histogram_log_generator.generate_log(test_file_name, test_steps, test_tag) diff --git a/tests/utils/log_operations.py b/tests/utils/log_operations.py index cd0d45a4aa91a985723ba100b48001d07d47e3a9..51a4fe150da16bd8b8d224019d80e98000827cf7 100644 --- a/tests/utils/log_operations.py +++ b/tests/utils/log_operations.py @@ -24,11 +24,13 @@ from mindinsight.datavisual.common.enums import PluginNameEnum from .log_generators.graph_log_generator import GraphLogGenerator from .log_generators.images_log_generator import ImagesLogGenerator from .log_generators.scalars_log_generator import ScalarsLogGenerator +from .log_generators.histogram_log_generator import HistogramLogGenerator log_generators = { PluginNameEnum.GRAPH.value: GraphLogGenerator(), PluginNameEnum.IMAGE.value: ImagesLogGenerator(), - PluginNameEnum.SCALAR.value: ScalarsLogGenerator() + PluginNameEnum.SCALAR.value: ScalarsLogGenerator(), + PluginNameEnum.HISTOGRAM.value: HistogramLogGenerator() }