提交 fa302959 编写于 作者: 李鸿章

fix: samples are erased when tag size overflow

上级 e21949d7
...@@ -29,7 +29,7 @@ TensorEvent = collections.namedtuple( ...@@ -29,7 +29,7 @@ TensorEvent = collections.namedtuple(
# config for `EventsData` # config for `EventsData`
_DEFAULT_STEP_SIZES_PER_TAG = settings.DEFAULT_STEP_SIZES_PER_TAG _DEFAULT_STEP_SIZES_PER_TAG = settings.DEFAULT_STEP_SIZES_PER_TAG
_MAX_DELETED_TAGS_SIZE = settings.MAX_TAG_SIZE_PER_EVENTS_DATA * 100
CONFIG = { CONFIG = {
'max_total_tag_sizes': settings.MAX_TAG_SIZE_PER_EVENTS_DATA, 'max_total_tag_sizes': settings.MAX_TAG_SIZE_PER_EVENTS_DATA,
'max_tag_sizes_per_plugin': 'max_tag_sizes_per_plugin':
...@@ -60,6 +60,7 @@ class EventsData: ...@@ -60,6 +60,7 @@ class EventsData:
self._max_step_sizes_per_tag = self._config['max_step_sizes_per_tag'] self._max_step_sizes_per_tag = self._config['max_step_sizes_per_tag']
self._tags = list() self._tags = list()
self._deleted_tags = set()
self._reservoir_by_tag = {} self._reservoir_by_tag = {}
self._reservoir_mutex_lock = threading.Lock() self._reservoir_mutex_lock = threading.Lock()
...@@ -82,6 +83,8 @@ class EventsData: ...@@ -82,6 +83,8 @@ class EventsData:
if tag not in set(self._tags): if tag not in set(self._tags):
deleted_tag = self._check_tag_out_of_spec(plugin_name) deleted_tag = self._check_tag_out_of_spec(plugin_name)
if deleted_tag is not None: if deleted_tag is not None:
if tag in self._deleted_tags:
return
self.delete_tensor_event(deleted_tag) self.delete_tensor_event(deleted_tag)
self._tags.append(tag) self._tags.append(tag)
...@@ -114,6 +117,8 @@ class EventsData: ...@@ -114,6 +117,8 @@ class EventsData:
Args: Args:
tag (str): The tag name. tag (str): The tag name.
""" """
if len(self._deleted_tags) < _MAX_DELETED_TAGS_SIZE:
self._deleted_tags.add(tag)
self._tags.remove(tag) self._tags.remove(tag)
for plugin_name, lock in self._tags_by_plugin_mutex_lock.items(): for plugin_name, lock in self._tags_by_plugin_mutex_lock.items():
with lock: with lock:
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册