Skip to content

M
mindinsight

MindSpore / mindinsight

通知 7
Star 3
Fork 2
M
mindinsight

体验新版 GitCode，发现更多精彩内容 >>

提交 7b9823b1 编写于 作者: Z zhangyunshu
浏览文件
操作

support for histogram API

上级 d3dd97cd
显示空白变更内容
内联 并排
Showing with 454 addition and 50 deletion
mindinsight/backend/datavisual/train_visual_api.py
浏览文件 @ 7b9823b1
......@@ -24,6 +24,7 @@ from flask import jsonify
from mindinsight.conf import settings
from mindinsight.datavisual.utils.tools import get_train_id
from mindinsight.datavisual.utils.tools import if_nan_inf_to_none
from mindinsight.datavisual.processors.histogram_processor import HistogramProcessor
from mindinsight.datavisual.processors.images_processor import ImageProcessor
from mindinsight.datavisual.processors.scalars_processor import ScalarsProcessor
from mindinsight.datavisual.processors.graph_processor import GraphProcessor
......@@ -146,6 +147,22 @@ def graph_search_single_node():
return jsonify(resp)
@BLUEPRINT.route("/datavisual/histograms", methods=["GET"])
def histogram():
"""
Interface to obtain histogram data.
Returns:
Response, which contains a JSON object.
"""
tag = request.args.get("tag", default=None)
train_id = get_train_id(request)
processor = HistogramProcessor(DATA_MANAGER)
response = processor.get_histograms(train_id, tag)
return jsonify(response)
def init_module(app):
"""
Init module entry.
......
mindinsight/conf/constants.py
浏览文件 @ 7b9823b1
......@@ -56,3 +56,4 @@ MAX_GRAPH_TAG_SIZE = 10
MAX_IMAGE_STEP_SIZE_PER_TAG = 10
MAX_SCALAR_STEP_SIZE_PER_TAG = 1000
MAX_GRAPH_STEP_SIZE_PER_TAG = 1
MAX_HISTOGRAM_STEP_SIZE_PER_TAG = 50
mindinsight/datavisual/common/enums.py
浏览文件 @ 7b9823b1
......@@ -37,3 +37,4 @@ class PluginNameEnum(BaseEnum):
IMAGE = 'image'
SCALAR = 'scalar'
GRAPH = 'graph'
HISTOGRAM = 'histogram'
mindinsight/datavisual/common/exceptions.py
浏览文件 @ 7b9823b1
......@@ -138,3 +138,12 @@ class ScalarNotExistError(MindInsightException):
super(ScalarNotExistError, self).__init__(DataVisualErrors.SCALAR_NOT_EXIST,
error_msg,
http_code=400)
class HistogramNotExistError(MindInsightException):
"""Unable to get histogram values based on a given condition."""
def __init__(self, error_detail):
error_msg = f'Histogram value is not exist. Detail: {error_detail}'
super(HistogramNotExistError, self).__init__(DataVisualErrors.HISTOGRAM_NOT_EXIST,
error_msg,
http_code=400)
mindinsight/datavisual/data_transform/events_data.py
浏览文件 @ 7b9823b1
......@@ -41,6 +41,7 @@ CONFIG = {
PluginNameEnum.SCALAR.value: settings.MAX_SCALAR_STEP_SIZE_PER_TAG,
PluginNameEnum.IMAGE.value: settings.MAX_IMAGE_STEP_SIZE_PER_TAG,
PluginNameEnum.GRAPH.value: settings.MAX_GRAPH_STEP_SIZE_PER_TAG,
PluginNameEnum.HISTOGRAM.value: settings.MAX_HISTOGRAM_STEP_SIZE_PER_TAG
}
}
......
mindinsight/datavisual/data_transform/ms_data_loader.py
浏览文件 @ 7b9823b1
......@@ -234,6 +234,16 @@ class MSDataLoader:
value=value.image)
self._events_data.add_tensor_event(tensor_event)
if value.HasField('histogram'):
histogram_msg = value.histogram
tag = '{}/{}'.format(value.tag, PluginNameEnum.HISTOGRAM.value)
tensor_event = TensorEvent(wall_time=event.wall_time,
step=event.step,
tag=tag,
plugin_name=PluginNameEnum.HISTOGRAM.value,
value=histogram_msg)
self._events_data.add_tensor_event(tensor_event)
if event.HasField('graph_def'):
graph_proto = event.graph_def
graph = MSGraph()
......
mindinsight/datavisual/processors/histogram_processor.py 0 → 100644
浏览文件 @ 7b9823b1
# Copyright 2020 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ============================================================================
"""Histogram Processor APIs."""
from mindinsight.utils.exceptions import ParamValueError
from mindinsight.datavisual.common.log import logger
from mindinsight.datavisual.common.validation import Validation
from mindinsight.datavisual.common.exceptions import HistogramNotExistError
from mindinsight.datavisual.processors.base_processor import BaseProcessor
class HistogramProcessor(BaseProcessor):
"""Histogram Processor."""
def get_histograms(self, train_id, tag):
"""
Builds a JSON-serializable object with information about histogram data.
Args:
train_id (str): The ID of the events data.
tag (str): The name of the tag the histogram data all belong to.
Returns:
dict, a dict including the `train_id`, `tag`, and `histograms'.
{
"train_id": ****,
"tag": ****,
"histograms": [{
"wall_time": ****,
"step": ****,
"bucket": [[**, **, **]],
},
{...}
]
}
"""
Validation.check_param_empty(train_id=train_id, tag=tag)
logger.info("Start to process histogram data...")
try:
tensors = self._data_manager.list_tensors(train_id, tag)
except ParamValueError as err:
raise HistogramNotExistError(err.message)
histograms = []
for tensor in tensors:
buckets = []
for bucket in tensor.value.buckets:
buckets.append([bucket.left, bucket.width, bucket.count])
histograms.append({
"wall_time": tensor.wall_time,
"step": tensor.step,
"buckets": buckets
})
logger.info("Histogram data processing is finished!")
response = {
"train_id": train_id,
"tag": tag,
"histograms": histograms
}
return response
mindinsight/datavisual/proto_files/mindinsight_summary.proto
浏览文件 @ 7b9823b1
......@@ -61,6 +61,30 @@ message Summary {
required bytes encoded_image = 4;
}
message Histogram {
message bucket{
// Counting number of values fallen in [left, left + width).
// For the rightmost bucket, the range is [left, left + width].
required double left = 1;
required double width = 2;
required int64 count = 3;
}
repeated bucket buckets = 1;
optional int64 nan_count = 2;
optional int64 pos_inf_count = 3;
optional int64 neg_inf_count = 4;
// max, min, sum will not take nan and inf into account.
// If there is no valid value in tensor, max and min will be nan, sum will be 0.
optional double max = 5;
optional double min = 6;
optional double sum = 7;
// total number of values. including nan and inf.
optional int64 count = 8;
}
message Value {
// Tag name for the data.
required string tag = 1;
......@@ -70,6 +94,7 @@ message Summary {
float scalar_value = 3;
Image image = 4;
TensorProto tensor = 8;
Histogram histogram = 9;
}
}
......
mindinsight/datavisual/proto_files/mindinsight_summary_pb2.py
浏览文件 @ 7b9823b1
# -*- coding: utf-8 -*-
# Generated by the protocol buffer compiler. DO NOT EDIT!
# source: mindinsight_summary.proto
import sys
_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))
from google.protobuf import descriptor as _descriptor
from google.protobuf import message as _message
from google.protobuf import reflection as _reflection
from google.protobuf import symbol_database as _symbol_database
from google.protobuf import descriptor_pb2
# @@protoc_insertion_point(imports)
_sym_db = _symbol_database.Default()
......@@ -20,10 +18,10 @@ DESCRIPTOR = _descriptor.FileDescriptor(
name='mindinsight_summary.proto',
package='mindinsight',
syntax='proto2',
serialized_pb=_b('\n\x19mindinsight_summary.proto\x12\x0bmindinsight\x1a\x18mindinsight_anf_ir.proto\"\x9a\x01\n\x05\x45vent\x12\x11\n\twall_time\x18\x01 \x02(\x01\x12\x0c\n\x04step\x18\x02 \x01(\x03\x12\x11\n\x07version\x18\x03 \x01(\tH\x00\x12,\n\tgraph_def\x18\x04 \x01(\x0b\x32\x17.mindinsight.GraphProtoH\x00\x12\'\n\x07summary\x18\x05 \x01(\x0b\x32\x14.mindinsight.SummaryH\x00\x42\x06\n\x04what\"\x98\x02\n\x07Summary\x12)\n\x05value\x18\x01 \x03(\x0b\x32\x1a.mindinsight.Summary.Value\x1aQ\n\x05Image\x12\x0e\n\x06height\x18\x01 \x02(\x05\x12\r\n\x05width\x18\x02 \x02(\x05\x12\x12\n\ncolorspace\x18\x03 \x02(\x05\x12\x15\n\rencoded_image\x18\x04 \x02(\x0c\x1a\x8e\x01\n\x05Value\x12\x0b\n\x03tag\x18\x01 \x02(\t\x12\x16\n\x0cscalar_value\x18\x03 \x01(\x02H\x00\x12+\n\x05image\x18\x04 \x01(\x0b\x32\x1a.mindinsight.Summary.ImageH\x00\x12*\n\x06tensor\x18\x08 \x01(\x0b\x32\x18.mindinsight.TensorProtoH\x00\x42\x07\n\x05valueB\x03\xf8\x01\x01')
serialized_options=b'\370\001\001',
serialized_pb=b'\n\x19mindinsight_summary.proto\x12\x0bmindinsight\x1a\x18mindinsight_anf_ir.proto\"\x9a\x01\n\x05\x45vent\x12\x11\n\twall_time\x18\x01 \x02(\x01\x12\x0c\n\x04step\x18\x02 \x01(\x03\x12\x11\n\x07version\x18\x03 \x01(\tH\x00\x12,\n\tgraph_def\x18\x04 \x01(\x0b\x32\x17.mindinsight.GraphProtoH\x00\x12\'\n\x07summary\x18\x05 \x01(\x0b\x32\x14.mindinsight.SummaryH\x00\x42\x06\n\x04what\"\xc0\x04\n\x07Summary\x12)\n\x05value\x18\x01 \x03(\x0b\x32\x1a.mindinsight.Summary.Value\x1aQ\n\x05Image\x12\x0e\n\x06height\x18\x01 \x02(\x05\x12\r\n\x05width\x18\x02 \x02(\x05\x12\x12\n\ncolorspace\x18\x03 \x02(\x05\x12\x15\n\rencoded_image\x18\x04 \x02(\x0c\x1a\xf0\x01\n\tHistogram\x12\x36\n\x07\x62uckets\x18\x01 \x03(\x0b\x32%.mindinsight.Summary.Histogram.bucket\x12\x11\n\tnan_count\x18\x02 \x01(\x03\x12\x15\n\rpos_inf_count\x18\x03 \x01(\x03\x12\x15\n\rneg_inf_count\x18\x04 \x01(\x03\x12\x0b\n\x03max\x18\x05 \x01(\x01\x12\x0b\n\x03min\x18\x06 \x01(\x01\x12\x0b\n\x03sum\x18\x07 \x01(\x01\x12\r\n\x05\x63ount\x18\x08 \x01(\x03\x1a\x34\n\x06\x62ucket\x12\x0c\n\x04left\x18\x01 \x02(\x01\x12\r\n\x05width\x18\x02 \x02(\x01\x12\r\n\x05\x63ount\x18\x03 \x02(\x03\x1a\xc3\x01\n\x05Value\x12\x0b\n\x03tag\x18\x01 \x02(\t\x12\x16\n\x0cscalar_value\x18\x03 \x01(\x02H\x00\x12+\n\x05image\x18\x04 \x01(\x0b\x32\x1a.mindinsight.Summary.ImageH\x00\x12*\n\x06tensor\x18\x08 \x01(\x0b\x32\x18.mindinsight.TensorProtoH\x00\x12\x33\n\thistogram\x18\t \x01(\x0b\x32\x1e.mindinsight.Summary.HistogramH\x00\x42\x07\n\x05valueB\x03\xf8\x01\x01'
,
dependencies=[mindinsight__anf__ir__pb2.DESCRIPTOR,])
_sym_db.RegisterFileDescriptor(DESCRIPTOR)
......@@ -41,42 +39,42 @@ _EVENT = _descriptor.Descriptor(
has_default_value=False, default_value=float(0),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='step', full_name='mindinsight.Event.step', index=1,
number=2, type=3, cpp_type=2, label=1,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='version', full_name='mindinsight.Event.version', index=2,
number=3, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
has_default_value=False, default_value=b"".decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='graph_def', full_name='mindinsight.Event.graph_def', index=3,
number=4, type=11, cpp_type=10, label=1,
has_default_value=False, default_value=None,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='summary', full_name='mindinsight.Event.summary', index=4,
number=5, type=11, cpp_type=10, label=1,
has_default_value=False, default_value=None,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
serialized_options=None, file=DESCRIPTOR),
],
extensions=[
],
nested_types=[],
enum_types=[
],
options=None,
serialized_options=None,
is_extendable=False,
syntax='proto2',
extension_ranges=[],
......@@ -103,35 +101,35 @@ _SUMMARY_IMAGE = _descriptor.Descriptor(
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='width', full_name='mindinsight.Summary.Image.width', index=1,
number=2, type=5, cpp_type=1, label=2,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='colorspace', full_name='mindinsight.Summary.Image.colorspace', index=2,
number=3, type=5, cpp_type=1, label=2,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='encoded_image', full_name='mindinsight.Summary.Image.encoded_image', index=3,
number=4, type=12, cpp_type=9, label=2,
has_default_value=False, default_value=_b(""),
has_default_value=False, default_value=b"",
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
serialized_options=None, file=DESCRIPTOR),
],
extensions=[
],
nested_types=[],
enum_types=[
],
options=None,
serialized_options=None,
is_extendable=False,
syntax='proto2',
extension_ranges=[],
......@@ -141,6 +139,129 @@ _SUMMARY_IMAGE = _descriptor.Descriptor(
serialized_end=361,
)
_SUMMARY_HISTOGRAM_BUCKET = _descriptor.Descriptor(
name='bucket',
full_name='mindinsight.Summary.Histogram.bucket',
filename=None,
file=DESCRIPTOR,
containing_type=None,
fields=[
_descriptor.FieldDescriptor(
name='left', full_name='mindinsight.Summary.Histogram.bucket.left', index=0,
number=1, type=1, cpp_type=5, label=2,
has_default_value=False, default_value=float(0),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='width', full_name='mindinsight.Summary.Histogram.bucket.width', index=1,
number=2, type=1, cpp_type=5, label=2,
has_default_value=False, default_value=float(0),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='count', full_name='mindinsight.Summary.Histogram.bucket.count', index=2,
number=3, type=3, cpp_type=2, label=2,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
],
extensions=[
],
nested_types=[],
enum_types=[
],
serialized_options=None,
is_extendable=False,
syntax='proto2',
extension_ranges=[],
oneofs=[
],
serialized_start=552,
serialized_end=604,
)
_SUMMARY_HISTOGRAM = _descriptor.Descriptor(
name='Histogram',
full_name='mindinsight.Summary.Histogram',
filename=None,
file=DESCRIPTOR,
containing_type=None,
fields=[
_descriptor.FieldDescriptor(
name='buckets', full_name='mindinsight.Summary.Histogram.buckets', index=0,
number=1, type=11, cpp_type=10, label=3,
has_default_value=False, default_value=[],
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='nan_count', full_name='mindinsight.Summary.Histogram.nan_count', index=1,
number=2, type=3, cpp_type=2, label=1,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='pos_inf_count', full_name='mindinsight.Summary.Histogram.pos_inf_count', index=2,
number=3, type=3, cpp_type=2, label=1,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='neg_inf_count', full_name='mindinsight.Summary.Histogram.neg_inf_count', index=3,
number=4, type=3, cpp_type=2, label=1,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='max', full_name='mindinsight.Summary.Histogram.max', index=4,
number=5, type=1, cpp_type=5, label=1,
has_default_value=False, default_value=float(0),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='min', full_name='mindinsight.Summary.Histogram.min', index=5,
number=6, type=1, cpp_type=5, label=1,
has_default_value=False, default_value=float(0),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='sum', full_name='mindinsight.Summary.Histogram.sum', index=6,
number=7, type=1, cpp_type=5, label=1,
has_default_value=False, default_value=float(0),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='count', full_name='mindinsight.Summary.Histogram.count', index=7,
number=8, type=3, cpp_type=2, label=1,
has_default_value=False, default_value=0,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
],
extensions=[
],
nested_types=[_SUMMARY_HISTOGRAM_BUCKET, ],
enum_types=[
],
serialized_options=None,
is_extendable=False,
syntax='proto2',
extension_ranges=[],
oneofs=[
],
serialized_start=364,
serialized_end=604,
)
_SUMMARY_VALUE = _descriptor.Descriptor(
name='Value',
full_name='mindinsight.Summary.Value',
......@@ -151,38 +272,45 @@ _SUMMARY_VALUE = _descriptor.Descriptor(
_descriptor.FieldDescriptor(
name='tag', full_name='mindinsight.Summary.Value.tag', index=0,
number=1, type=9, cpp_type=9, label=2,
has_default_value=False, default_value=_b("").decode('utf-8'),
has_default_value=False, default_value=b"".decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='scalar_value', full_name='mindinsight.Summary.Value.scalar_value', index=1,
number=3, type=2, cpp_type=6, label=1,
has_default_value=False, default_value=float(0),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='image', full_name='mindinsight.Summary.Value.image', index=2,
number=4, type=11, cpp_type=10, label=1,
has_default_value=False, default_value=None,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='tensor', full_name='mindinsight.Summary.Value.tensor', index=3,
number=8, type=11, cpp_type=10, label=1,
has_default_value=False, default_value=None,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='histogram', full_name='mindinsight.Summary.Value.histogram', index=4,
number=9, type=11, cpp_type=10, label=1,
has_default_value=False, default_value=None,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
],
extensions=[
],
nested_types=[],
enum_types=[
],
options=None,
serialized_options=None,
is_extendable=False,
syntax='proto2',
extension_ranges=[],
......@@ -191,8 +319,8 @@ _SUMMARY_VALUE = _descriptor.Descriptor(
name='value', full_name='mindinsight.Summary.Value.value',
index=0, containing_type=None, fields=[]),
],
serialized_start=364,
serialized_end=506,
serialized_start=607,
serialized_end=802,
)
_SUMMARY = _descriptor.Descriptor(
......@@ -208,21 +336,21 @@ _SUMMARY = _descriptor.Descriptor(
has_default_value=False, default_value=[],
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
options=None),
serialized_options=None, file=DESCRIPTOR),
],
extensions=[
],
nested_types=[_SUMMARY_IMAGE, _SUMMARY_VALUE, ],
nested_types=[_SUMMARY_IMAGE, _SUMMARY_HISTOGRAM, _SUMMARY_VALUE, ],
enum_types=[
],
options=None,
serialized_options=None,
is_extendable=False,
syntax='proto2',
extension_ranges=[],
oneofs=[
],
serialized_start=226,
serialized_end=506,
serialized_end=802,
)
_EVENT.fields_by_name['graph_def'].message_type = mindinsight__anf__ir__pb2._GRAPHPROTO
......@@ -237,8 +365,12 @@ _EVENT.oneofs_by_name['what'].fields.append(
_EVENT.fields_by_name['summary'])
_EVENT.fields_by_name['summary'].containing_oneof = _EVENT.oneofs_by_name['what']
_SUMMARY_IMAGE.containing_type = _SUMMARY
_SUMMARY_HISTOGRAM_BUCKET.containing_type = _SUMMARY_HISTOGRAM
_SUMMARY_HISTOGRAM.fields_by_name['buckets'].message_type = _SUMMARY_HISTOGRAM_BUCKET
_SUMMARY_HISTOGRAM.containing_type = _SUMMARY
_SUMMARY_VALUE.fields_by_name['image'].message_type = _SUMMARY_IMAGE
_SUMMARY_VALUE.fields_by_name['tensor'].message_type = mindinsight__anf__ir__pb2._TENSORPROTO
_SUMMARY_VALUE.fields_by_name['histogram'].message_type = _SUMMARY_HISTOGRAM
_SUMMARY_VALUE.containing_type = _SUMMARY
_SUMMARY_VALUE.oneofs_by_name['value'].fields.append(
_SUMMARY_VALUE.fields_by_name['scalar_value'])
......@@ -249,41 +381,60 @@ _SUMMARY_VALUE.fields_by_name['image'].containing_oneof = _SUMMARY_VALUE.oneofs_
_SUMMARY_VALUE.oneofs_by_name['value'].fields.append(
_SUMMARY_VALUE.fields_by_name['tensor'])
_SUMMARY_VALUE.fields_by_name['tensor'].containing_oneof = _SUMMARY_VALUE.oneofs_by_name['value']
_SUMMARY_VALUE.oneofs_by_name['value'].fields.append(
_SUMMARY_VALUE.fields_by_name['histogram'])
_SUMMARY_VALUE.fields_by_name['histogram'].containing_oneof = _SUMMARY_VALUE.oneofs_by_name['value']
_SUMMARY.fields_by_name['value'].message_type = _SUMMARY_VALUE
DESCRIPTOR.message_types_by_name['Event'] = _EVENT
DESCRIPTOR.message_types_by_name['Summary'] = _SUMMARY
_sym_db.RegisterFileDescriptor(DESCRIPTOR)
Event = _reflection.GeneratedProtocolMessageType('Event', (_message.Message,), dict(
DESCRIPTOR = _EVENT,
__module__ = 'mindinsight_summary_pb2'
Event = _reflection.GeneratedProtocolMessageType('Event', (_message.Message,), {
'DESCRIPTOR' : _EVENT,
'__module__' : 'mindinsight_summary_pb2'
# @@protoc_insertion_point(class_scope:mindinsight.Event)
))
})
_sym_db.RegisterMessage(Event)
Summary = _reflection.GeneratedProtocolMessageType('Summary', (_message.Message,), dict(
Summary = _reflection.GeneratedProtocolMessageType('Summary', (_message.Message,), {
Image = _reflection.GeneratedProtocolMessageType('Image', (_message.Message,), dict(
DESCRIPTOR = _SUMMARY_IMAGE,
__module__ = 'mindinsight_summary_pb2'
'Image' : _reflection.GeneratedProtocolMessageType('Image', (_message.Message,), {
'DESCRIPTOR' : _SUMMARY_IMAGE,
'__module__' : 'mindinsight_summary_pb2'
# @@protoc_insertion_point(class_scope:mindinsight.Summary.Image)
))
})
,
'Histogram' : _reflection.GeneratedProtocolMessageType('Histogram', (_message.Message,), {
'bucket' : _reflection.GeneratedProtocolMessageType('bucket', (_message.Message,), {
'DESCRIPTOR' : _SUMMARY_HISTOGRAM_BUCKET,
'__module__' : 'mindinsight_summary_pb2'
# @@protoc_insertion_point(class_scope:mindinsight.Summary.Histogram.bucket)
})
,
'DESCRIPTOR' : _SUMMARY_HISTOGRAM,
'__module__' : 'mindinsight_summary_pb2'
# @@protoc_insertion_point(class_scope:mindinsight.Summary.Histogram)
})
,
Value = _reflection.GeneratedProtocolMessageType('Value', (_message.Message,), dict(
DESCRIPTOR = _SUMMARY_VALUE,
__module__ = 'mindinsight_summary_pb2'
'Value' : _reflection.GeneratedProtocolMessageType('Value', (_message.Message,), {
'DESCRIPTOR' : _SUMMARY_VALUE,
'__module__' : 'mindinsight_summary_pb2'
# @@protoc_insertion_point(class_scope:mindinsight.Summary.Value)
))
})
,
DESCRIPTOR = _SUMMARY,
__module__ = 'mindinsight_summary_pb2'
'DESCRIPTOR' : _SUMMARY,
'__module__' : 'mindinsight_summary_pb2'
# @@protoc_insertion_point(class_scope:mindinsight.Summary)
))
})
_sym_db.RegisterMessage(Summary)
_sym_db.RegisterMessage(Summary.Image)
_sym_db.RegisterMessage(Summary.Histogram)
_sym_db.RegisterMessage(Summary.Histogram.bucket)
_sym_db.RegisterMessage(Summary.Value)
DESCRIPTOR.has_options = True
DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), _b('\370\001\001'))
DESCRIPTOR._options = None
# @@protoc_insertion_point(module_scope)
mindinsight/utils/constant.py
浏览文件 @ 7b9823b1
......@@ -62,3 +62,4 @@ class DataVisualErrors(Enum):
GRAPH_NOT_EXIST = 12
IMAGE_NOT_EXIST = 13
SCALAR_NOT_EXIST = 14
HISTOGRAM_NOT_EXIST = 15
tests/st/func/datavisual/conftest.py
浏览文件 @ 7b9823b1
......@@ -65,6 +65,7 @@ def init_summary_logs():
summaries_metadata.update(
log_operations.create_summary_logs(summary_base_dir, constants.SUMMARY_DIR_NUM_SECOND,
constants.SUMMARY_DIR_PREFIX,
constants.SUMMARY_DIR_NUM_FIRST))
summaries_metadata.update(
log_operations.create_multiple_logs(summary_base_dir, constants.MULTIPLE_DIR_NAME,
......
tests/ut/datavisual/processors/test_train_task_manager.py
浏览文件 @ 7b9823b1
......@@ -70,7 +70,7 @@ class TestTrainTaskManager:
def load_data(self):
"""Load data."""
log_operation = LogOperations()
self._plugins_id_map = {'image': [], 'scalar': [], 'graph': []}
self._plugins_id_map = {'image': [], 'scalar': [], 'graph': [], 'histogram': []}
self._events_names = []
self._train_id_list = []
......
tests/utils/log_generators/histogram_log_generator.py 0 → 100644
浏览文件 @ 7b9823b1
# Copyright 2020 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ============================================================================
"""Log generator for histogram data."""
import time
import numpy as np
from mindinsight.datavisual.proto_files import mindinsight_summary_pb2 as summary_pb2
from .log_generator import LogGenerator
class HistogramLogGenerator(LogGenerator):
"""
Log generator for histogram data.
This is a log generator writing histogram data. User can use it to generate fake
summary logs about histogram.
"""
def generate_event(self, values):
"""
Method for generating histogram event.
Args:
values (dict): A dict contains:
{
wall_time (float): Timestamp.
step (int): Train step.
value (float): Histogram value.
tag (str): Tag name.
}
Returns:
summary_pb2.Event.
"""
histogram_event = summary_pb2.Event()
histogram_event.wall_time = values.get('wall_time')
histogram_event.step = values.get('step')
value = histogram_event.summary.value.add()
value.tag = values.get('tag')
buckets = values.get('buckets')
for bucket in buckets:
left, width, count = bucket
bucket = value.histogram.buckets.add()
bucket.left = left
bucket.width = width
bucket.count = count
return histogram_event
def generate_log(self, file_path, steps_list, tag_name):
"""
Generate log for external calls.
Args:
file_path (str): Path to write logs.
steps_list (list): A list consists of step.
tag_name (str): Tag name.
Returns:
list[dict], generated histogram metadata.
None, to be consistent with return value of HistogramGenerator.
"""
histogram_metadata = []
for step in steps_list:
histogram = dict()
wall_time = time.time()
histogram.update({'wall_time': wall_time})
histogram.update({'step': step})
histogram.update({'tag': tag_name})
# Construct buckets
buckets = []
leftmost = list(np.random.randn(11))
leftmost.sort()
for i in range(10):
left = leftmost[i]
width = leftmost[i+1] - left
count = np.random.randint(20)
bucket = [left, width, count]
buckets.append(bucket)
histogram.update({'buckets': buckets})
histogram_metadata.append(histogram)
self._write_log_one_step(file_path, histogram)
return histogram_metadata, None
if __name__ == "__main__":
histogram_log_generator = HistogramLogGenerator()
test_file_name = '%s.%s.%s' % ('histogram', 'summary', str(time.time()))
test_steps = [1, 3, 5]
test_tag = "test_histogram_tag_name"
histogram_log_generator.generate_log(test_file_name, test_steps, test_tag)
tests/utils/log_operations.py
浏览文件 @ 7b9823b1
......@@ -24,11 +24,13 @@ from mindinsight.datavisual.common.enums import PluginNameEnum
from .log_generators.graph_log_generator import GraphLogGenerator
from .log_generators.images_log_generator import ImagesLogGenerator
from .log_generators.scalars_log_generator import ScalarsLogGenerator
from .log_generators.histogram_log_generator import HistogramLogGenerator
log_generators = {
PluginNameEnum.GRAPH.value: GraphLogGenerator(),
PluginNameEnum.IMAGE.value: ImagesLogGenerator(),
PluginNameEnum.SCALAR.value: ScalarsLogGenerator()
PluginNameEnum.SCALAR.value: ScalarsLogGenerator(),
PluginNameEnum.HISTOGRAM.value: HistogramLogGenerator()
}
......
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或