diff --git a/mindinsight/lineagemgr/cache_item_updater.py b/mindinsight/lineagemgr/cache_item_updater.py index 6c6749ae86eecfcabd945b0f7b2f4734707f0b34..bb7b5b59983db1d667c7aacddc53cddb193f945a 100644 --- a/mindinsight/lineagemgr/cache_item_updater.py +++ b/mindinsight/lineagemgr/cache_item_updater.py @@ -17,7 +17,7 @@ import os from mindinsight.datavisual.data_transform.data_manager import BaseCacheItemUpdater, CachedTrainJob from mindinsight.lineagemgr.common.log import logger -from mindinsight.lineagemgr.common.exceptions.exceptions import MindInsightException +from mindinsight.lineagemgr.common.exceptions.exceptions import LineageFileNotFoundError from mindinsight.lineagemgr.common.validator.validate import validate_train_id, validate_added_info from mindinsight.lineagemgr.lineage_parser import LineageParser, LINEAGE from mindinsight.utils.exceptions import ParamValueError @@ -59,19 +59,13 @@ class LineageCacheItemUpdater(BaseCacheItemUpdater): try: lineage_parser = self._lineage_parsing(cache_item) - except MindInsightException as err: - with cache_item.lock_key(LINEAGE): - try: - cache_item.delete(key=LINEAGE) - logger.info("Parse failed, delete the tran job %s. Detail: %s.", relative_path, str(err)) - except ParamValueError: - logger.debug("Parse failed, no need to delete the train job %s. " - "Detail: %s.", relative_path, str(err)) + except LineageFileNotFoundError: + self._delete_lineage_in_cache(cache_item, LINEAGE, relative_path) return super_lineage_obj = lineage_parser.super_lineage_obj if super_lineage_obj is None: - logger.debug("There is no lineage to update in tran job %s.", relative_path) + logger.debug("There is no lineage to update in train job %s.", relative_path) return cache_item.set(key=LINEAGE, value=lineage_parser) @@ -91,3 +85,12 @@ class LineageCacheItemUpdater(BaseCacheItemUpdater): lineage_parser.load() return lineage_parser + + def _delete_lineage_in_cache(self, cache_item, key, relative_path): + with cache_item.lock_key(key): + try: + cache_item.delete(key=key) + logger.info("Parse failed, delete the tran job %s.", relative_path) + except ParamValueError: + logger.debug("Parse failed, and it is not in cache, " + "no need to delete the train job %s.", relative_path) diff --git a/mindinsight/lineagemgr/lineage_parser.py b/mindinsight/lineagemgr/lineage_parser.py index d5c2a83ec3d82df042b794516d1284f9bb71bfba..e3815253241c16b2b3a5aaf32a63ba181300efb7 100644 --- a/mindinsight/lineagemgr/lineage_parser.py +++ b/mindinsight/lineagemgr/lineage_parser.py @@ -100,7 +100,15 @@ class LineageParser: continue self._latest_file_size = new_size - self._parse_summary_log() + try: + self._parse_summary_log() + except (LineageSummaryAnalyzeException, + LineageEventNotExistException, + LineageEventFieldNotExistException) as error: + logger.debug("Parse file failed, file_path is %s. Detail: %s.", file_path, str(error)) + except MindInsightException as error: + logger.exception(error) + logger.debug("Parse file failed, file_path is %s.", file_path) def _init_if_files_deleted(self, file_list): """Init variables if files deleted.""" @@ -189,13 +197,6 @@ class LineageOrganizer: self._super_lineage_objs.update({abs_summary_dir: super_lineage_obj}) except LineageFileNotFoundError: no_lineage_count += 1 - except (LineageSummaryAnalyzeException, - LineageEventNotExistException, - LineageEventFieldNotExistException) as error: - logger.warning("Parse file failed under summary_dir %s. Detail: %s.", relative_dir, str(error)) - except MindInsightException as error: - logger.exception(error) - logger.warning("Parse file failed under summary_dir %s.", relative_dir) if no_lineage_count == len(relative_dirs): logger.info('There is no summary log file under summary_base_dir.')