!90 Drop histogram steps if original_buckets_count is too large. [ for r0.2]

Merge pull request !90 from liangyongxiong/r0.2

mindinsight/datavisual/data_transform/histogram_container.py

@@ -72,6 +72,10 @@ class Bucket:
 
 
 class HistogramContainer:
+
+    # Max quantity of original buckets.
+    MAX_ORIGINAL_BUCKETS_COUNT = 90
+
     """
     Histogram data container.
 
@@ -114,6 +118,11 @@ class HistogramContainer:
         """Gets original proto message."""
         return self._msg
 
+    @property
+    def original_buckets_count(self):
+        """Gets original buckets quantity."""
+        return len(self._original_buckets)
+
     def set_visual_range(self, max_val: float, min_val: float, bins: int) -> None:
         """
         Sets visual range for later re-sampling.

mindinsight/datavisual/data_transform/ms_data_loader.py

@@ -239,14 +239,19 @@ class MSDataLoader:
 
                 if value.HasField('histogram'):
                     histogram_msg = HistogramContainer(value.histogram)
-                    tag = '{}/{}'.format(value.tag, PluginNameEnum.HISTOGRAM.value)
-                    tensor_event = TensorEvent(wall_time=event.wall_time,
-                                               step=event.step,
-                                               tag=tag,
-                                               plugin_name=PluginNameEnum.HISTOGRAM.value,
-                                               value=histogram_msg,
-                                               filename=self._latest_summary_filename)
-                    self._events_data.add_tensor_event(tensor_event)
+                    # Drop steps if original_buckets_count exceeds HistogramContainer.MAX_ORIGINAL_BUCKETS_COUNT
+                    # to avoid time-consuming re-sample process.
+                    if histogram_msg.original_buckets_count > HistogramContainer.MAX_ORIGINAL_BUCKETS_COUNT:
+                        logger.warning('original_buckets_count exceeds HistogramContainer.MAX_ORIGINAL_BUCKETS_COUNT')
+                    else:
+                        tag = '{}/{}'.format(value.tag, PluginNameEnum.HISTOGRAM.value)
+                        tensor_event = TensorEvent(wall_time=event.wall_time,
+                                                   step=event.step,
+                                                   tag=tag,
+                                                   plugin_name=PluginNameEnum.HISTOGRAM.value,
+                                                   value=histogram_msg,
+                                                   filename=self._latest_summary_filename)
+                        self._events_data.add_tensor_event(tensor_event)
 
         if event.HasField('graph_def'):
             graph_proto = event.graph_def