From ff12f13f4cb2c3e061117444a32d237ce5195e06 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=9D=8E=E9=B8=BF=E7=AB=A0?= Date: Tue, 28 Apr 2020 18:11:46 +0800 Subject: [PATCH] fix: samples are erased when tag size overflow --- .../datavisual/data_transform/events_data.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/mindinsight/datavisual/data_transform/events_data.py b/mindinsight/datavisual/data_transform/events_data.py index 5c4797a..282bb2a 100644 --- a/mindinsight/datavisual/data_transform/events_data.py +++ b/mindinsight/datavisual/data_transform/events_data.py @@ -17,10 +17,10 @@ import collections import threading +from mindinsight.conf import settings from mindinsight.datavisual.common.enums import PluginNameEnum +from mindinsight.datavisual.common.log import logger from mindinsight.datavisual.data_transform import reservoir -from mindinsight.conf import settings - # Type of the tensor event from external component _Tensor = collections.namedtuple('_Tensor', ['wall_time', 'step', 'value', 'filename']) @@ -29,7 +29,7 @@ TensorEvent = collections.namedtuple( # config for `EventsData` _DEFAULT_STEP_SIZES_PER_TAG = settings.DEFAULT_STEP_SIZES_PER_TAG - +_MAX_DELETED_TAGS_SIZE = settings.MAX_TAG_SIZE_PER_EVENTS_DATA * 100 CONFIG = { 'max_total_tag_sizes': settings.MAX_TAG_SIZE_PER_EVENTS_DATA, 'max_tag_sizes_per_plugin': @@ -60,6 +60,7 @@ class EventsData: self._max_step_sizes_per_tag = self._config['max_step_sizes_per_tag'] self._tags = list() + self._deleted_tags = set() self._reservoir_by_tag = {} self._reservoir_mutex_lock = threading.Lock() @@ -82,6 +83,8 @@ class EventsData: if tag not in set(self._tags): deleted_tag = self._check_tag_out_of_spec(plugin_name) if deleted_tag is not None: + if tag in self._deleted_tags: + return self.delete_tensor_event(deleted_tag) self._tags.append(tag) @@ -114,6 +117,13 @@ class EventsData: Args: tag (str): The tag name. """ + if len(self._deleted_tags) < _MAX_DELETED_TAGS_SIZE: + self._deleted_tags.add(tag) + else: + logger.warning( + 'Too many deleted tags, %d upper limit reached, tags updating may not function hereafter', + _MAX_DELETED_TAGS_SIZE) + logger.warning('%r and all related samples are going to be deleted', tag) self._tags.remove(tag) for plugin_name, lock in self._tags_by_plugin_mutex_lock.items(): with lock: -- GitLab