提交 665f2d68 编写于 作者: L luopengting

modify lineage parsing for multi lineage files, modify ut/st

上级 f3b621a9
...@@ -72,7 +72,7 @@ class _BasicTrainJob: ...@@ -72,7 +72,7 @@ class _BasicTrainJob:
self._update_time = update_time self._update_time = update_time
@property @property
def summary_dir(self): def abs_summary_dir(self):
"""Get summary directory path.""" """Get summary directory path."""
return self._abs_summary_dir return self._abs_summary_dir
...@@ -131,9 +131,9 @@ class CachedTrainJob: ...@@ -131,9 +131,9 @@ class CachedTrainJob:
return self._last_access_time return self._last_access_time
@property @property
def summary_dir(self): def abs_summary_dir(self):
"""Get summary directory path.""" """Get summary directory path."""
return self._basic_info.summary_dir return self._basic_info.abs_summary_dir
@property @property
def summary_base_dir(self): def summary_base_dir(self):
...@@ -144,12 +144,33 @@ class CachedTrainJob: ...@@ -144,12 +144,33 @@ class CachedTrainJob:
"""Set value to cache.""" """Set value to cache."""
self._content[key] = value self._content[key] = value
def get(self, key): def delete(self, key):
"""Get value from cache.""" """Delete key in cache."""
if key in self._content:
self._content.pop(key)
def get(self, key, raise_exception=True):
"""
Get value from cache.
Args:
key (str): Key of content.
raise_exception (bool): If the key does not exist and
raise_exception is True, it will raise an Exception.
Returns:
Union[Object, None], Return value if key in content,
return False else if raise_exception is False.
Raises:
ParamValueError, if the key does not exist and raise_exception is True.
"""
try: try:
return self._content[key] return self._content[key]
except KeyError: except KeyError:
raise ParamValueError("Invalid cache key({}).".format(key)) if raise_exception:
raise ParamValueError("Invalid cache key({}).".format(key))
return None
@property @property
def basic_info(self): def basic_info(self):
...@@ -163,9 +184,7 @@ class CachedTrainJob: ...@@ -163,9 +184,7 @@ class CachedTrainJob:
def lock_key(self, key): def lock_key(self, key):
"""Threading lock with given key.""" """Threading lock with given key."""
if key not in self._key_locks: return self._key_locks.setdefault(key, threading.Lock())
self._key_locks[key] = threading.Lock()
return self._key_locks[key]
class TrainJob: class TrainJob:
......
...@@ -96,11 +96,8 @@ def general_get_summary_lineage(data_manager=None, summary_dir=None, keys=None): ...@@ -96,11 +96,8 @@ def general_get_summary_lineage(data_manager=None, summary_dir=None, keys=None):
if data_manager is None: if data_manager is None:
normalize_summary_dir(summary_dir) normalize_summary_dir(summary_dir)
super_lineage_obj = None
if os.path.isabs(summary_dir):
super_lineage_obj = LineageParser(summary_dir).super_lineage_obj super_lineage_obj = LineageParser(summary_dir).super_lineage_obj
elif data_manager is not None: else:
validate_train_id(summary_dir) validate_train_id(summary_dir)
super_lineage_obj = LineageOrganizer(data_manager=data_manager).get_super_lineage_obj(summary_dir) super_lineage_obj = LineageOrganizer(data_manager=data_manager).get_super_lineage_obj(summary_dir)
......
...@@ -27,15 +27,19 @@ def update_lineage_object(data_manager, train_id, added_info: dict): ...@@ -27,15 +27,19 @@ def update_lineage_object(data_manager, train_id, added_info: dict):
"""Update lineage objects about tag and remark.""" """Update lineage objects about tag and remark."""
validate_train_id(train_id) validate_train_id(train_id)
cache_item = data_manager.get_brief_train_job(train_id) cache_item = data_manager.get_brief_train_job(train_id)
cached_added_info = cache_item.get(key=LINEAGE).added_info lineage_item = cache_item.get(key=LINEAGE, raise_exception=False)
if lineage_item is None:
logger.warning("Cannot update the lineage for tran job %s, because it does not exist.", train_id)
raise ParamValueError("Cannot update the lineage for tran job %s, because it does not exist." % train_id)
cached_added_info = lineage_item.super_lineage_obj.added_info
new_added_info = dict(cached_added_info) new_added_info = dict(cached_added_info)
for key, value in added_info.items(): for key, value in added_info.items():
if key in ["tag", "remark"]: new_added_info.update({key: value})
new_added_info.update({key: value})
with cache_item.lock_key(LINEAGE): with cache_item.lock_key(LINEAGE):
cache_item.get(key=LINEAGE).added_info = new_added_info cache_item.get(key=LINEAGE).super_lineage_obj.added_info = new_added_info
class LineageCacheItemUpdater(BaseCacheItemUpdater): class LineageCacheItemUpdater(BaseCacheItemUpdater):
...@@ -44,8 +48,7 @@ class LineageCacheItemUpdater(BaseCacheItemUpdater): ...@@ -44,8 +48,7 @@ class LineageCacheItemUpdater(BaseCacheItemUpdater):
def update_item(self, cache_item: CachedTrainJob): def update_item(self, cache_item: CachedTrainJob):
"""Update cache item in place.""" """Update cache item in place."""
summary_base_dir = cache_item.summary_base_dir summary_base_dir = cache_item.summary_base_dir
summary_dir = cache_item.summary_dir summary_dir = cache_item.abs_summary_dir
update_time = cache_item.basic_info.update_time
# The summary_base_dir and summary_dir have been normalized in data_manager. # The summary_base_dir and summary_dir have been normalized in data_manager.
if summary_base_dir == summary_dir: if summary_base_dir == summary_dir:
...@@ -54,17 +57,31 @@ class LineageCacheItemUpdater(BaseCacheItemUpdater): ...@@ -54,17 +57,31 @@ class LineageCacheItemUpdater(BaseCacheItemUpdater):
relative_path = f'./{os.path.basename(summary_dir)}' relative_path = f'./{os.path.basename(summary_dir)}'
try: try:
cached_added_info = cache_item.get(key=LINEAGE).added_info lineage_parser = self._lineage_parsing(cache_item)
except ParamValueError:
cached_added_info = None
try:
lineage_parser = LineageParser(summary_dir, update_time, cached_added_info)
super_lineage_obj = lineage_parser.super_lineage_obj
except LineageFileNotFoundError: except LineageFileNotFoundError:
super_lineage_obj = None with cache_item.lock_key(LINEAGE):
cache_item.delete(key=LINEAGE)
return
super_lineage_obj = lineage_parser.super_lineage_obj
if super_lineage_obj is None: if super_lineage_obj is None:
logger.warning("There is no lineage to update in tran job %s.", relative_path) logger.warning("There is no lineage to update in tran job %s.", relative_path)
return return
with cache_item.lock_key(LINEAGE):
cache_item.set(key=LINEAGE, value=super_lineage_obj) cache_item.set(key=LINEAGE, value=lineage_parser)
def _lineage_parsing(self, cache_item):
"""Parse summaries and return lineage parser."""
summary_dir = cache_item.abs_summary_dir
update_time = cache_item.basic_info.update_time
cached_lineage_item = cache_item.get(key=LINEAGE, raise_exception=False)
if cached_lineage_item is None:
lineage_parser = LineageParser(summary_dir, update_time)
else:
lineage_parser = cached_lineage_item
with cache_item.lock_key(LINEAGE):
lineage_parser.update_time = update_time
lineage_parser.load()
return lineage_parser
...@@ -13,9 +13,10 @@ ...@@ -13,9 +13,10 @@
# limitations under the License. # limitations under the License.
# ============================================================================ # ============================================================================
"""This file provides path resolution.""" """This file provides path resolution."""
import os
from mindinsight.datavisual.data_transform.summary_watcher import SummaryWatcher from mindinsight.datavisual.data_transform.summary_watcher import SummaryWatcher
from mindinsight.lineagemgr.common.log import logger
from mindinsight.lineagemgr.common.utils import get_timestamp
from mindinsight.utils.exceptions import MindInsightException
class SummaryPathParser: class SummaryPathParser:
...@@ -29,121 +30,36 @@ class SummaryPathParser: ...@@ -29,121 +30,36 @@ class SummaryPathParser:
_LINEAGE_SUMMARY_SUFFIX_LEN = len(LINEAGE_SUMMARY_SUFFIX) _LINEAGE_SUMMARY_SUFFIX_LEN = len(LINEAGE_SUMMARY_SUFFIX)
@staticmethod @staticmethod
def get_summary_dirs(summary_base_dir): def get_lineage_summaries(summary_dir, is_sorted=False, reverse=True):
"""
Get summary dirs according to summary base dir.
Args:
summary_base_dir (str): Summary base dir.
Returns:
list[str], all summary dirs in summary base dir. The summary dir is
absolute path.
"""
summary_watcher = SummaryWatcher()
relative_dirs = summary_watcher.list_summary_directories(
summary_base_dir=summary_base_dir
)
summary_dirs = list(
map(
lambda item: os.path.realpath(
os.path.join(summary_base_dir, item.get('relative_path'))
),
relative_dirs
)
)
return summary_dirs
@staticmethod
def get_latest_lineage_summary(summary_dir):
""" """
Get latest lineage summary log path according to summary dir. Get lineage summaries according to summary dir.
Args: Args:
summary_dir (str): Summary dir. summary_dir (str): Summary dir.
is_sorted (bool): If it is True, files will be sorted.
reverse (bool): If it is True, sort by timestamp increments and filename decrement.
Returns: Returns:
Union[str, None], if the lineage summary log exist, return the path, list, if the lineage summary log exist, return the file names, else return [].
else return None. The lineage summary log path is absolute path.
"""
summary_watcher = SummaryWatcher()
summaries = summary_watcher.list_summaries(summary_base_dir=summary_dir)
latest_file_name = SummaryPathParser._get_latest_lineage_file(summaries)
return os.path.join(summary_dir, latest_file_name) \
if latest_file_name is not None else None
@staticmethod
def get_latest_lineage_summaries(summary_base_dir):
"""
Get all latest lineage summary logs in summary base dir.
Args:
summary_base_dir (str): Summary base dir.
Returns:
list[str], all latest lineage summary logs in summary base dir. The
lineage summary log is absolute path.
"""
summary_watcher = SummaryWatcher()
relative_dirs = summary_watcher.list_summary_directories(
summary_base_dir=summary_base_dir
)
latest_summaries = []
for item in relative_dirs:
relative_dir = item.get('relative_path')
summaries = summary_watcher.list_summaries(
summary_base_dir=summary_base_dir,
relative_path=relative_dir
)
latest_file_name = SummaryPathParser._get_latest_lineage_file(
summaries
)
if latest_file_name is None:
continue
latest_file = os.path.realpath(
os.path.join(
summary_base_dir,
relative_dir,
latest_file_name
)
)
latest_summaries.append(latest_file)
return latest_summaries
@staticmethod
def _get_latest_lineage_file(summaries):
"""
Get latest lineage summary file.
If there is a file with the suffix `LINEAGE_SUMMARY_SUFFIX`, check
whether there is a file with the same name that does not include the
suffix `LINEAGE_SUMMARY_SUFFIX`. When both exist, the file is considered
to be a lineage summary log.
Args:
summaries (list[dict]): All summary logs info in summary dir.
Returns:
str, the latest lineage summary file name.
""" """
try: try:
latest_summary = max( summary_watcher = SummaryWatcher()
summaries, summaries = summary_watcher.list_summaries(summary_base_dir=summary_dir)
key=lambda summary: summary.get('create_time') except MindInsightException as err:
) logger.warning(str(err))
except ValueError: return []
return None summary_files = [summary.get('file_name') for summary in summaries]
max_create_time = latest_summary.get('create_time') lineage_files_name = list(filter(
summary_file_names = [] lambda filename: (filename.endswith(SummaryPathParser.LINEAGE_SUMMARY_SUFFIX)), summary_files))
for summary in summaries: if is_sorted:
if summary.get('create_time') == max_create_time: lineage_files_name = SummaryPathParser._sorted_summary_files(lineage_files_name, reverse)
summary_file_names.append(summary.get('file_name'))
return lineage_files_name
latest_lineage_name = None @staticmethod
for name in summary_file_names: def _sorted_summary_files(summary_files, reverse):
if not name.endswith(SummaryPathParser.LINEAGE_SUMMARY_SUFFIX): """Sort by timestamp increments and filename decrement."""
continue sorted_files = sorted(summary_files,
ms_name = name[:-SummaryPathParser._LINEAGE_SUMMARY_SUFFIX_LEN] key=lambda filename: (-get_timestamp(filename), filename),
if ms_name in summary_file_names: reverse=reverse)
latest_lineage_name = name return sorted_files
return latest_lineage_name
...@@ -14,7 +14,9 @@ ...@@ -14,7 +14,9 @@
# ============================================================================ # ============================================================================
"""Lineage utils.""" """Lineage utils."""
from functools import wraps from functools import wraps
import re
from mindinsight.datavisual.data_transform.summary_watcher import SummaryWatcher
from mindinsight.lineagemgr.common.log import logger as log from mindinsight.lineagemgr.common.log import logger as log
from mindinsight.lineagemgr.common.exceptions.exceptions import LineageParamRunContextError, \ from mindinsight.lineagemgr.common.exceptions.exceptions import LineageParamRunContextError, \
LineageGetModelFileError, LineageLogError, LineageParamValueError, LineageDirNotExistError, \ LineageGetModelFileError, LineageLogError, LineageParamValueError, LineageDirNotExistError, \
...@@ -68,3 +70,9 @@ def normalize_summary_dir(summary_dir): ...@@ -68,3 +70,9 @@ def normalize_summary_dir(summary_dir):
log.exception(error) log.exception(error)
raise LineageParamSummaryPathError(str(error.message)) raise LineageParamSummaryPathError(str(error.message))
return summary_dir return summary_dir
def get_timestamp(filename):
"""Get timestamp from filename."""
timestamp = int(re.search(SummaryWatcher().SUMMARY_FILENAME_REGEX, filename)[1])
return timestamp
...@@ -21,6 +21,7 @@ from mindinsight.lineagemgr.common.exceptions.exceptions import LineageSummaryAn ...@@ -21,6 +21,7 @@ from mindinsight.lineagemgr.common.exceptions.exceptions import LineageSummaryAn
MindInsightException MindInsightException
from mindinsight.lineagemgr.common.log import logger from mindinsight.lineagemgr.common.log import logger
from mindinsight.lineagemgr.common.path_parser import SummaryPathParser from mindinsight.lineagemgr.common.path_parser import SummaryPathParser
from mindinsight.lineagemgr.summary.file_handler import FileHandler
from mindinsight.lineagemgr.summary.lineage_summary_analyzer import LineageSummaryAnalyzer from mindinsight.lineagemgr.summary.lineage_summary_analyzer import LineageSummaryAnalyzer
from mindinsight.lineagemgr.querier.query_model import LineageObj from mindinsight.lineagemgr.querier.query_model import LineageObj
from mindinsight.utils.exceptions import ParamValueError from mindinsight.utils.exceptions import ParamValueError
...@@ -30,7 +31,7 @@ LINEAGE = "lineage" ...@@ -30,7 +31,7 @@ LINEAGE = "lineage"
class SuperLineageObj: class SuperLineageObj:
"""This is an object for LineageObj and its additional info.""" """This is an object for LineageObj and its additional info."""
def __init__(self, lineage_obj, update_time, added_info=None): def __init__(self, lineage_obj: LineageObj, update_time, added_info=None):
self._lineage_obj = lineage_obj self._lineage_obj = lineage_obj
self._update_time = update_time self._update_time = update_time
self._added_info = added_info if added_info is not None else dict() self._added_info = added_info if added_info is not None else dict()
...@@ -59,11 +60,60 @@ class SuperLineageObj: ...@@ -59,11 +60,60 @@ class SuperLineageObj:
class LineageParser: class LineageParser:
"""Lineage parser.""" """Lineage parser."""
def __init__(self, summary_dir, update_time=None, added_info=None): def __init__(self, summary_dir, update_time=None, added_info=None):
self._super_lineage_obj = None
self._summary_dir = summary_dir self._summary_dir = summary_dir
self._update_time = update_time self.update_time = update_time
self._added_info = added_info self._added_info = added_info
self._parse_summary_log()
self._init_variables()
self.load()
def _init_variables(self):
"""Init variables."""
self._super_lineage_obj = None
self._latest_filename = None
self._latest_file_size = None
self._cached_file_list = None
def load(self):
"""Find and load summaries."""
# get sorted lineage files
lineage_files = SummaryPathParser.get_lineage_summaries(self._summary_dir, is_sorted=True)
if not lineage_files:
logger.warning('There is no summary log file under summary_dir %s.', self._summary_dir)
raise LineageFileNotFoundError(
'There is no summary log file under summary_dir.'
)
self._init_if_files_deleted(lineage_files)
index = 0
if self._latest_filename is not None:
index = lineage_files.index(self._latest_filename)
for filename in lineage_files[index:]:
if filename != self._latest_filename:
self._latest_filename = filename
self._latest_file_size = 0
file_path = os.path.join(self._summary_dir, filename)
new_size = FileHandler(file_path).size
if new_size == self._latest_file_size:
continue
self._latest_file_size = new_size
self._parse_summary_log()
def _init_if_files_deleted(self, file_list):
"""Init variables if files deleted."""
cached_file_list = self._cached_file_list
self._cached_file_list = file_list
if cached_file_list is None:
return
deleted_files = set(cached_file_list) - set(file_list)
if deleted_files:
logger.warning("There are some files has been deleted, "
"all files will be reloaded in path %s.", self._summary_dir)
self._init_variables()
def _parse_summary_log(self): def _parse_summary_log(self):
""" """
...@@ -72,15 +122,22 @@ class LineageParser: ...@@ -72,15 +122,22 @@ class LineageParser:
Returns: Returns:
bool, `True` if parse summary log success, else `False`. bool, `True` if parse summary log success, else `False`.
""" """
file_path = SummaryPathParser.get_latest_lineage_summary(self._summary_dir) file_path = os.path.realpath(os.path.join(self._summary_dir, self._latest_filename))
if file_path is None:
logger.warning('There is no summary log file under summary_dir %s.', self._summary_dir)
raise LineageFileNotFoundError(
'There is no summary log file under summary_dir.'
)
try: try:
lineage_info = LineageSummaryAnalyzer.get_summary_infos(file_path) lineage_info = LineageSummaryAnalyzer.get_summary_infos(file_path)
user_defined_info = LineageSummaryAnalyzer.get_user_defined_info(file_path) user_defined_info = LineageSummaryAnalyzer.get_user_defined_info(file_path)
self._update_lineage_obj(lineage_info, user_defined_info)
except LineageSummaryAnalyzeException:
logger.warning("Parse file failed under summary_dir %s.", file_path)
except (LineageEventNotExistException, LineageEventFieldNotExistException) as error:
logger.warning("Parse file failed under summary_dir %s. Detail: %s.", file_path, str(error))
except MindInsightException as error:
logger.exception(error)
logger.warning("Parse file failed under summary_dir %s.", file_path)
def _update_lineage_obj(self, lineage_info, user_defined_info):
"""Update lineage object."""
if self._super_lineage_obj is None:
lineage_obj = LineageObj( lineage_obj = LineageObj(
self._summary_dir, self._summary_dir,
train_lineage=lineage_info.train_lineage, train_lineage=lineage_info.train_lineage,
...@@ -88,15 +145,14 @@ class LineageParser: ...@@ -88,15 +145,14 @@ class LineageParser:
dataset_graph=lineage_info.dataset_graph, dataset_graph=lineage_info.dataset_graph,
user_defined_info=user_defined_info user_defined_info=user_defined_info
) )
self._super_lineage_obj = SuperLineageObj(lineage_obj, self._update_time, self._added_info) self._super_lineage_obj = SuperLineageObj(lineage_obj, self.update_time, self._added_info)
except (LineageSummaryAnalyzeException, else:
LineageEventNotExistException, self._super_lineage_obj.lineage_obj.parse_and_update_lineage(
LineageEventFieldNotExistException): train_lineage=lineage_info.train_lineage,
logger.warning("Parse file failed under summary_dir %s.", self._summary_dir) evaluation_lineage=lineage_info.eval_lineage,
except MindInsightException as error: dataset_graph=lineage_info.dataset_graph,
logger.error(str(error)) user_defined_info=user_defined_info
logger.exception(error) )
logger.warning("Parse file failed under summary_dir %s.", self._summary_dir)
@property @property
def super_lineage_obj(self): def super_lineage_obj(self):
...@@ -156,7 +212,7 @@ class LineageOrganizer: ...@@ -156,7 +212,7 @@ class LineageOrganizer:
cache_items = brief_cache.cache_items cache_items = brief_cache.cache_items
for relative_dir, cache_train_job in cache_items.items(): for relative_dir, cache_train_job in cache_items.items():
try: try:
super_lineage_obj = cache_train_job.get("lineage") super_lineage_obj = cache_train_job.get("lineage").super_lineage_obj
self._super_lineage_objs.update({relative_dir: super_lineage_obj}) self._super_lineage_objs.update({relative_dir: super_lineage_obj})
except ParamValueError: except ParamValueError:
logger.info("This is no lineage info in train job %s.", relative_dir) logger.info("This is no lineage info in train job %s.", relative_dir)
......
...@@ -82,6 +82,30 @@ class LineageObj: ...@@ -82,6 +82,30 @@ class LineageObj:
self._lineage_info = { self._lineage_info = {
self._name_summary_dir: summary_dir self._name_summary_dir: summary_dir
} }
self._filtration_result = None
self._init_lineage()
self.parse_and_update_lineage(**kwargs)
def _init_lineage(self):
"""Init lineage info."""
# train
self._lineage_info[self._name_model] = {}
self._lineage_info[self._name_algorithm] = {}
self._lineage_info[self._name_hyper_parameters] = {}
self._lineage_info[self._name_train_dataset] = {}
# eval
self._lineage_info[self._name_metric] = {}
self._lineage_info[self._name_valid_dataset] = {}
# dataset graph
self._lineage_info[self._name_dataset_graph] = {}
# user defined
self._lineage_info[self._name_user_defined] = {}
def parse_and_update_lineage(self, **kwargs):
"""Parse and update lineage."""
user_defined_info_list = kwargs.get('user_defined_info', []) user_defined_info_list = kwargs.get('user_defined_info', [])
train_lineage = kwargs.get('train_lineage') train_lineage = kwargs.get('train_lineage')
evaluation_lineage = kwargs.get('evaluation_lineage') evaluation_lineage = kwargs.get('evaluation_lineage')
...@@ -92,6 +116,7 @@ class LineageObj: ...@@ -92,6 +116,7 @@ class LineageObj:
self._parse_train_lineage(train_lineage) self._parse_train_lineage(train_lineage)
self._parse_evaluation_lineage(evaluation_lineage) self._parse_evaluation_lineage(evaluation_lineage)
self._parse_dataset_graph(dataset_graph) self._parse_dataset_graph(dataset_graph)
self._filtration_result = self._organize_filtration_result() self._filtration_result = self._organize_filtration_result()
@property @property
...@@ -309,10 +334,6 @@ class LineageObj: ...@@ -309,10 +334,6 @@ class LineageObj:
train_lineage (Event): Train lineage. train_lineage (Event): Train lineage.
""" """
if train_lineage is None: if train_lineage is None:
self._lineage_info[self._name_model] = {}
self._lineage_info[self._name_algorithm] = {}
self._lineage_info[self._name_hyper_parameters] = {}
self._lineage_info[self._name_train_dataset] = {}
return return
event_dict = MessageToDict( event_dict = MessageToDict(
...@@ -341,8 +362,6 @@ class LineageObj: ...@@ -341,8 +362,6 @@ class LineageObj:
evaluation_lineage (Event): Evaluation lineage. evaluation_lineage (Event): Evaluation lineage.
""" """
if evaluation_lineage is None: if evaluation_lineage is None:
self._lineage_info[self._name_metric] = {}
self._lineage_info[self._name_valid_dataset] = {}
return return
event_dict = MessageToDict( event_dict = MessageToDict(
...@@ -364,9 +383,7 @@ class LineageObj: ...@@ -364,9 +383,7 @@ class LineageObj:
Args: Args:
dataset_graph (Event): Dataset graph. dataset_graph (Event): Dataset graph.
""" """
if dataset_graph is None: if dataset_graph is not None:
self._lineage_info[self._name_dataset_graph] = {}
else:
# convert message to dict # convert message to dict
event_dict = organize_graph(dataset_graph.dataset_graph) event_dict = organize_graph(dataset_graph.dataset_graph)
if event_dict is None: if event_dict is None:
...@@ -380,6 +397,8 @@ class LineageObj: ...@@ -380,6 +397,8 @@ class LineageObj:
Args: Args:
user_defined_info_list (list): user defined info list. user_defined_info_list (list): user defined info list.
""" """
if not user_defined_info_list:
return
user_defined_infos = dict() user_defined_infos = dict()
for user_defined_info in user_defined_info_list: for user_defined_info in user_defined_info_list:
user_defined_infos.update(user_defined_info) user_defined_infos.update(user_defined_info)
......
...@@ -204,11 +204,9 @@ class LineageSummaryAnalyzer(SummaryAnalyzer): ...@@ -204,11 +204,9 @@ class LineageSummaryAnalyzer(SummaryAnalyzer):
try: try:
lineage_info = analyzer.get_latest_info() lineage_info = analyzer.get_latest_info()
except (MindInsightException, IOError, DecodeError) as err: except (MindInsightException, IOError, DecodeError) as err:
log.error("Failed to get lineage information.")
log.exception(err) log.exception(err)
raise LineageSummaryAnalyzeException() raise LineageSummaryAnalyzeException()
except Exception as err: except Exception as err:
log.error("Failed to get lineage information.")
log.exception(err) log.exception(err)
raise LineageSummaryAnalyzeException() raise LineageSummaryAnalyzeException()
......
...@@ -111,26 +111,26 @@ LINEAGE_FILTRATION_RUN1 = { ...@@ -111,26 +111,26 @@ LINEAGE_FILTRATION_RUN1 = {
LINEAGE_FILTRATION_RUN2 = { LINEAGE_FILTRATION_RUN2 = {
'summary_dir': os.path.join(BASE_SUMMARY_DIR, 'run2'), 'summary_dir': os.path.join(BASE_SUMMARY_DIR, 'run2'),
'model_lineage': { 'model_lineage': {
'loss_function': None, 'loss_function': "SoftmaxCrossEntropyWithLogits",
'train_dataset_path': None, 'train_dataset_path': None,
'train_dataset_count': None, 'train_dataset_count': 1024,
'test_dataset_path': None, 'test_dataset_path': None,
'test_dataset_count': 10240, 'test_dataset_count': 10240,
'user_defined': {}, 'user_defined': {},
'network': None, 'network': "ResNet",
'optimizer': None, 'optimizer': "Momentum",
'learning_rate': None, 'learning_rate': 0.11999999731779099,
'epoch': None, 'epoch': 10,
'batch_size': None, 'batch_size': 32,
'device_num': None, 'device_num': 2,
'loss': None, 'loss': 0.029999999329447746,
'model_size': None, 'model_size': 10,
'metric': { 'metric': {
'accuracy': 2.7800000000000002 'accuracy': 2.7800000000000002
}, },
'dataset_mark': 3 'dataset_mark': 3
}, },
'dataset_graph': {} 'dataset_graph': DATASET_GRAPH
} }
...@@ -460,9 +460,10 @@ class TestModelApi(TestCase): ...@@ -460,9 +460,10 @@ class TestModelApi(TestCase):
'customized': event_data.CUSTOMIZED__0, 'customized': event_data.CUSTOMIZED__0,
'object': [ 'object': [
LINEAGE_FILTRATION_EXCEPT_RUN, LINEAGE_FILTRATION_EXCEPT_RUN,
LINEAGE_FILTRATION_RUN1 LINEAGE_FILTRATION_RUN1,
LINEAGE_FILTRATION_RUN2
], ],
'count': 2 'count': 3
} }
partial_res1 = filter_summary_lineage(BASE_SUMMARY_DIR, search_condition1) partial_res1 = filter_summary_lineage(BASE_SUMMARY_DIR, search_condition1)
expect_objects = expect_result.get('object') expect_objects = expect_result.get('object')
...@@ -746,7 +747,7 @@ class TestModelApi(TestCase): ...@@ -746,7 +747,7 @@ class TestModelApi(TestCase):
expect_result = { expect_result = {
'customized': {}, 'customized': {},
'object': [], 'object': [],
'count': 1 'count': 2
} }
partial_res2 = filter_summary_lineage(BASE_SUMMARY_DIR, search_condition2) partial_res2 = filter_summary_lineage(BASE_SUMMARY_DIR, search_condition2)
assert expect_result == partial_res2 assert expect_result == partial_res2
......
...@@ -32,8 +32,7 @@ from mindinsight.lineagemgr import get_summary_lineage ...@@ -32,8 +32,7 @@ from mindinsight.lineagemgr import get_summary_lineage
from mindinsight.lineagemgr.collection.model.model_lineage import TrainLineage, EvalLineage, \ from mindinsight.lineagemgr.collection.model.model_lineage import TrainLineage, EvalLineage, \
AnalyzeObject AnalyzeObject
from mindinsight.lineagemgr.common.exceptions.error_code import LineageErrors from mindinsight.lineagemgr.common.exceptions.error_code import LineageErrors
from mindinsight.lineagemgr.common.exceptions.exceptions import LineageParamRunContextError, \ from mindinsight.lineagemgr.common.exceptions.exceptions import LineageParamRunContextError
LineageFileNotFoundError
from mindinsight.utils.exceptions import MindInsightException from mindinsight.utils.exceptions import MindInsightException
from mindspore.application.model_zoo.resnet import ResNet from mindspore.application.model_zoo.resnet import ResNet
from mindspore.common.tensor import Tensor from mindspore.common.tensor import Tensor
...@@ -343,5 +342,3 @@ class TestModelLineage(TestCase): ...@@ -343,5 +342,3 @@ class TestModelLineage(TestCase):
full_file_name = summary_record.full_file_name full_file_name = summary_record.full_file_name
assert full_file_name.endswith('_lineage') assert full_file_name.endswith('_lineage')
assert os.path.isfile(full_file_name) assert os.path.isfile(full_file_name)
with self.assertRaisesRegex(LineageFileNotFoundError, 'no summary log file'):
get_summary_lineage(summary_dir)
...@@ -54,10 +54,10 @@ class TestModel(TestCase): ...@@ -54,10 +54,10 @@ class TestModel(TestCase):
) )
@mock.patch('mindinsight.lineagemgr.common.utils.validate_path') @mock.patch('mindinsight.lineagemgr.common.utils.validate_path')
@mock.patch.object(SummaryPathParser, 'get_latest_lineage_summary') @mock.patch.object(SummaryPathParser, 'get_lineage_summaries')
def test_get_summary_lineage_failed2(self, mock_summary, mock_valid): def test_get_summary_lineage_failed2(self, mock_summary, mock_valid):
"""Test get_summary_lineage failed.""" """Test get_summary_lineage failed."""
mock_summary.return_value = None mock_summary.return_value = []
mock_valid.return_value = '/path/to/summary/dir' mock_valid.return_value = '/path/to/summary/dir'
self.assertRaisesRegex( self.assertRaisesRegex(
LineageFileNotFoundError, LineageFileNotFoundError,
...@@ -66,17 +66,21 @@ class TestModel(TestCase): ...@@ -66,17 +66,21 @@ class TestModel(TestCase):
'/path/to/summary_dir' '/path/to/summary_dir'
) )
@mock.patch('mindinsight.lineagemgr.lineage_parser.FileHandler')
@mock.patch('mindinsight.lineagemgr.lineage_parser.LineageParser._parse_summary_log') @mock.patch('mindinsight.lineagemgr.lineage_parser.LineageParser._parse_summary_log')
@mock.patch('mindinsight.lineagemgr.common.utils.validate_path') @mock.patch('mindinsight.lineagemgr.common.utils.validate_path')
@mock.patch.object(SummaryPathParser, 'get_latest_lineage_summary') @mock.patch.object(SummaryPathParser, 'get_lineage_summaries')
def test_get_summary_lineage_failed3(self, def test_get_summary_lineage_failed3(self,
mock_summary, mock_summary,
mock_valid, mock_valid,
mock_paser): mock_parser,
mock_file_handler):
"""Test get_summary_lineage failed.""" """Test get_summary_lineage failed."""
mock_summary.return_value = '/path/to/summary/file' mock_summary.return_value = ['/path/to/summary/file']
mock_valid.return_value = '/path/to/summary_dir' mock_valid.return_value = '/path/to/summary_dir'
mock_paser.return_value = None mock_parser.return_value = None
mock_file_handler = MagicMock()
mock_file_handler.size = 1
result = get_summary_lineage('/path/to/summary_dir') result = get_summary_lineage('/path/to/summary_dir')
assert {} == result assert {} == result
...@@ -127,7 +131,7 @@ class TestFilterAPI(TestCase): ...@@ -127,7 +131,7 @@ class TestFilterAPI(TestCase):
"""Test the function of filter_summary_lineage.""" """Test the function of filter_summary_lineage."""
@mock.patch('mindinsight.lineagemgr.api.model.LineageOrganizer') @mock.patch('mindinsight.lineagemgr.api.model.LineageOrganizer')
@mock.patch('mindinsight.lineagemgr.api.model.Querier') @mock.patch('mindinsight.lineagemgr.api.model.Querier')
@mock.patch('mindinsight.lineagemgr.lineage_parser.SummaryPathParser.get_latest_lineage_summary') @mock.patch('mindinsight.lineagemgr.lineage_parser.SummaryPathParser.get_lineage_summaries')
@mock.patch('mindinsight.lineagemgr.api.model._convert_relative_path_to_abspath') @mock.patch('mindinsight.lineagemgr.api.model._convert_relative_path_to_abspath')
@mock.patch('mindinsight.lineagemgr.api.model.normalize_summary_dir') @mock.patch('mindinsight.lineagemgr.api.model.normalize_summary_dir')
def test_filter_summary_lineage(self, validate_path_mock, convert_path_mock, def test_filter_summary_lineage(self, validate_path_mock, convert_path_mock,
...@@ -198,11 +202,11 @@ class TestFilterAPI(TestCase): ...@@ -198,11 +202,11 @@ class TestFilterAPI(TestCase):
@mock.patch('mindinsight.lineagemgr.api.model.validate_search_model_condition') @mock.patch('mindinsight.lineagemgr.api.model.validate_search_model_condition')
@mock.patch('mindinsight.lineagemgr.api.model.validate_condition') @mock.patch('mindinsight.lineagemgr.api.model.validate_condition')
@mock.patch('mindinsight.lineagemgr.api.model.normalize_summary_dir') @mock.patch('mindinsight.lineagemgr.api.model.normalize_summary_dir')
@mock.patch.object(SummaryPathParser, 'get_latest_lineage_summary') @mock.patch.object(SummaryPathParser, 'get_lineage_summaries')
def test_failed_to_get_summary_filesh(self, mock_parse, *args): def test_failed_to_get_summary_filesh(self, mock_parse, *args):
"""Test filter_summary_lineage with invalid invalid param.""" """Test filter_summary_lineage with invalid invalid param."""
path = '/path/to/summary/dir' path = '/path/to/summary/dir'
mock_parse.return_value = None mock_parse.return_value = []
args[0].return_value = path args[0].return_value = path
self.assertRaisesRegex( self.assertRaisesRegex(
LineageFileNotFoundError, LineageFileNotFoundError,
...@@ -215,7 +219,7 @@ class TestFilterAPI(TestCase): ...@@ -215,7 +219,7 @@ class TestFilterAPI(TestCase):
@mock.patch('mindinsight.lineagemgr.api.model.validate_search_model_condition') @mock.patch('mindinsight.lineagemgr.api.model.validate_search_model_condition')
@mock.patch('mindinsight.lineagemgr.api.model.validate_condition') @mock.patch('mindinsight.lineagemgr.api.model.validate_condition')
@mock.patch('mindinsight.lineagemgr.api.model.normalize_summary_dir') @mock.patch('mindinsight.lineagemgr.api.model.normalize_summary_dir')
@mock.patch.object(SummaryPathParser, 'get_latest_lineage_summaries') @mock.patch.object(SummaryPathParser, 'get_lineage_summaries')
@mock.patch('mindinsight.lineagemgr.api.model.Querier') @mock.patch('mindinsight.lineagemgr.api.model.Querier')
def test_failed_to_querier(self, mock_query, mock_parse, *args): def test_failed_to_querier(self, mock_query, mock_parse, *args):
"""Test filter_summary_lineage with invalid invalid param.""" """Test filter_summary_lineage with invalid invalid param."""
......
...@@ -33,19 +33,19 @@ MOCK_SUMMARY_DIRS = [ ...@@ -33,19 +33,19 @@ MOCK_SUMMARY_DIRS = [
] ]
MOCK_SUMMARIES = [ MOCK_SUMMARIES = [
{ {
'file_name': 'file0', 'file_name': 'file0.summary.1',
'create_time': datetime.fromtimestamp(1582031970) 'create_time': datetime.fromtimestamp(1582031970)
}, },
{ {
'file_name': 'file0_lineage', 'file_name': 'file0.summary.1_lineage',
'create_time': datetime.fromtimestamp(1582031970) 'create_time': datetime.fromtimestamp(1582031970)
}, },
{ {
'file_name': 'file1', 'file_name': 'file1.summary.2',
'create_time': datetime.fromtimestamp(1582031971) 'create_time': datetime.fromtimestamp(1582031971)
}, },
{ {
'file_name': 'file1_lineage', 'file_name': 'file1.summary.2_lineage',
'create_time': datetime.fromtimestamp(1582031971) 'create_time': datetime.fromtimestamp(1582031971)
} }
] ]
...@@ -54,92 +54,54 @@ MOCK_SUMMARIES = [ ...@@ -54,92 +54,54 @@ MOCK_SUMMARIES = [
class TestSummaryPathParser(TestCase): class TestSummaryPathParser(TestCase):
"""Test the class of SummaryPathParser.""" """Test the class of SummaryPathParser."""
@mock.patch.object(SummaryWatcher, 'list_summary_directories')
def test_get_summary_dirs(self, *args):
"""Test the function of get_summary_dirs."""
args[0].return_value = MOCK_SUMMARY_DIRS
expected_result = [
'/path/to/base/relative_path0',
'/path/to/base',
'/path/to/base/relative_path1'
]
base_dir = '/path/to/base'
result = SummaryPathParser.get_summary_dirs(base_dir)
self.assertListEqual(expected_result, result)
args[0].return_value = []
result = SummaryPathParser.get_summary_dirs(base_dir)
self.assertListEqual([], result)
@mock.patch.object(SummaryWatcher, 'list_summaries') @mock.patch.object(SummaryWatcher, 'list_summaries')
def test_get_latest_lineage_summary(self, *args): def test_get_lineage_summaries(self, *args):
"""Test the function of get_latest_lineage_summary.""" """Test the function of get_lineage_summaries."""
args[0].return_value = MOCK_SUMMARIES args[0].return_value = MOCK_SUMMARIES
exp_result = ['file0.summary.1_lineage', 'file1.summary.2_lineage']
summary_dir = '/path/to/summary_dir' summary_dir = '/path/to/summary_dir'
result = SummaryPathParser.get_latest_lineage_summary(summary_dir) result = SummaryPathParser.get_lineage_summaries(summary_dir)
self.assertEqual('/path/to/summary_dir/file1_lineage', result) self.assertEqual(exp_result, result)
args[0].return_value = [ args[0].return_value = [
{ {
'file_name': 'file0', 'file_name': 'file0.summary.1',
'create_time': datetime.fromtimestamp(1582031970) 'create_time': datetime.fromtimestamp(1582031970)
} }
] ]
result = SummaryPathParser.get_latest_lineage_summary(summary_dir) result = SummaryPathParser.get_lineage_summaries(summary_dir)
self.assertEqual(None, result) self.assertEqual([], result)
args[0].return_value = [ args[0].return_value = [
{ {
'file_name': 'file0_lineage', 'file_name': 'file0.summary.1_lineage',
'create_time': datetime.fromtimestamp(1582031970) 'create_time': datetime.fromtimestamp(1582031970)
} }
] ]
result = SummaryPathParser.get_latest_lineage_summary(summary_dir) result = SummaryPathParser.get_lineage_summaries(summary_dir)
self.assertEqual(None, result) self.assertEqual(['file0.summary.1_lineage'], result)
args[0].return_value = [ args[0].return_value = [
{ {
'file_name': 'file0_lineage', 'file_name': 'file0.summary.3_lineage',
'create_time': datetime.fromtimestamp(1582031970) 'create_time': datetime.fromtimestamp(1582031970)
}, },
{ {
'file_name': 'file0_lineage_lineage', 'file_name': 'file0.summary.2_lineage_lineage',
'create_time': datetime.fromtimestamp(1582031970) 'create_time': datetime.fromtimestamp(1582031970)
}, },
{ {
'file_name': 'file1_lineage', 'file_name': 'file1.summary.1_lineage',
'create_time': datetime.fromtimestamp(1582031971) 'create_time': datetime.fromtimestamp(1582031971)
}, },
{ {
'file_name': 'file1_lineage_lineage', 'file_name': 'file1.summary.7_lineage_lineage',
'create_time': datetime.fromtimestamp(1582031971) 'create_time': datetime.fromtimestamp(1582031971)
} }
] ]
result = SummaryPathParser.get_latest_lineage_summary(summary_dir) exp_result = ['file1.summary.1_lineage',
self.assertEqual('/path/to/summary_dir/file1_lineage_lineage', result) 'file0.summary.2_lineage_lineage',
'file0.summary.3_lineage',
@mock.patch.object(SummaryWatcher, 'list_summaries') 'file1.summary.7_lineage_lineage']
@mock.patch.object(SummaryWatcher, 'list_summary_directories') result = SummaryPathParser.get_lineage_summaries(summary_dir, is_sorted=True)
def test_get_latest_lineage_summaries(self, *args): self.assertEqual(exp_result, result)
"""Test the function of get_latest_lineage_summaries."""
args[0].return_value = MOCK_SUMMARY_DIRS
args[1].return_value = MOCK_SUMMARIES
expected_result = [
'/path/to/base/relative_path0/file1_lineage',
'/path/to/base/file1_lineage',
'/path/to/base/relative_path1/file1_lineage'
]
base_dir = '/path/to/base'
result = SummaryPathParser.get_latest_lineage_summaries(base_dir)
self.assertListEqual(expected_result, result)
args[1].return_value = [
{
'file_name': 'file0_lineage',
'create_time': datetime.fromtimestamp(1582031970)
}
]
result = SummaryPathParser.get_latest_lineage_summaries(base_dir)
self.assertListEqual([], result)
...@@ -16,6 +16,7 @@ ...@@ -16,6 +16,7 @@
import time import time
from unittest import TestCase, mock from unittest import TestCase, mock
from unittest.mock import MagicMock
from google.protobuf.json_format import ParseDict from google.protobuf.json_format import ParseDict
...@@ -248,11 +249,12 @@ LINEAGE_FILTRATION_6 = { ...@@ -248,11 +249,12 @@ LINEAGE_FILTRATION_6 = {
class TestQuerier(TestCase): class TestQuerier(TestCase):
"""Test the class of `Querier`.""" """Test the class of `Querier`."""
@mock.patch('mindinsight.lineagemgr.lineage_parser.SummaryPathParser.get_latest_lineage_summary') @mock.patch('mindinsight.lineagemgr.lineage_parser.SummaryPathParser.get_lineage_summaries')
@mock.patch('mindinsight.lineagemgr.lineage_parser.SummaryWatcher.list_summary_directories') @mock.patch('mindinsight.lineagemgr.lineage_parser.SummaryWatcher.list_summary_directories')
@mock.patch('mindinsight.lineagemgr.lineage_parser.LineageSummaryAnalyzer.get_user_defined_info') @mock.patch('mindinsight.lineagemgr.lineage_parser.LineageSummaryAnalyzer.get_user_defined_info')
@mock.patch('mindinsight.lineagemgr.lineage_parser.LineageSummaryAnalyzer.get_summary_infos') @mock.patch('mindinsight.lineagemgr.lineage_parser.LineageSummaryAnalyzer.get_summary_infos')
def setUp(self, *args): @mock.patch('mindinsight.lineagemgr.lineage_parser.FileHandler')
def setUp(self, mock_file_handler, *args):
"""Initialization before test case execution.""" """Initialization before test case execution."""
args[0].return_value = create_lineage_info( args[0].return_value = create_lineage_info(
event_data.EVENT_TRAIN_DICT_0, event_data.EVENT_TRAIN_DICT_0,
...@@ -260,7 +262,10 @@ class TestQuerier(TestCase): ...@@ -260,7 +262,10 @@ class TestQuerier(TestCase):
event_data.EVENT_DATASET_DICT_0 event_data.EVENT_DATASET_DICT_0
) )
args[1].return_value = [] args[1].return_value = []
args[3].return_value = 'path' args[3].return_value = ['path']
mock_file_handler = MagicMock()
mock_file_handler.size = 1
args[2].return_value = [{'relative_path': './', 'update_time': 1}] args[2].return_value = [{'relative_path': './', 'update_time': 1}]
single_summary_path = '/path/to/summary0' single_summary_path = '/path/to/summary0'
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册