!312 function Profiler._init() and anlayse() is too large

Merge pull request !312 from WeibiaoYu/master

mindinsight/profiler/parser/hwts_log_parser.py

@@ -98,7 +98,7 @@ class HWTSLogParser:
                     continue
 
                 if int(task_id) < 25000:
-                    task_id = stream_id + "_" + task_id
+                    task_id = str(stream_id) + "_" + str(task_id)
                 result_data += ("%-14s %-4s %-8s %-9s %-8s %-15s %s\n" %(log_type[int(ms_type, 2)], cnt, core_id,
                                                                          blk_id, task_id, syscnt, stream_id))
 

mindinsight/profiler/profiling.py

@@ -78,31 +78,8 @@ class Profiler:
     def __init__(self, subgraph='all', is_detail=True, is_show_op_path=False, output_path='./data',
                  optypes_to_deal='', optypes_not_deal='Variable', job_id=""):
         # get device_id and device_target
-        device_target = ""
-        dev_id = ""
-        try:
-            import mindspore.context as context
-            dev_id = str(context.get_context("device_id"))
-            device_target = context.get_context("device_target")
-        except ImportError:
-            logger.error("Profiling: fail to import context from mindspore.")
-        except ValueError as err:
-            logger.error("Profiling: fail to get context, %s", err)
-
-        if not dev_id or not dev_id.isdigit():
-            dev_id = os.getenv('DEVICE_ID')
-        if not dev_id or not dev_id.isdigit():
-            dev_id = "0"
-            logger.error("Fail to get DEVICE_ID, use 0 instead.")
-
-        if device_target and device_target != "Davinci" \
-            and device_target != "Ascend":
-            msg = ("Profiling: unsupport backend: %s" \
-                   % device_target)
-            raise RuntimeError(msg)
-
-        self._dev_id = dev_id
-        self._container_path = os.path.join(self._base_profiling_container_path, dev_id)
+        self._get_devid_and_devtarget()
+        self._container_path = os.path.join(self._base_profiling_container_path, self._dev_id)
         data_path = os.path.join(self._container_path, "data")
         if not os.path.exists(data_path):
             os.makedirs(data_path, exist_ok=True)
@@ -158,8 +135,6 @@ class Profiler:
         except ImportError:
             logger.error("Profiling: fail to import release from mindspore.")
 
-        logger.info("begin profiler analyse")
-
         job_id = self._get_profiling_job_id()
         if not job_id:
             msg = ("Fail to get profiling job, please check whether job dir was generated")
@@ -197,11 +172,8 @@ class Profiler:
         # parse DATA_PREPROCESS.dev.AICPU file, write output_data_preprocess_aicpu_x.txt
         output_data_preprocess_aicpu = self._aicpu_op_output_filename_target + self._dev_id + ".txt"
         output_data_preprocess_aicpu = os.path.join(self._output_path, output_data_preprocess_aicpu)
-        try:
-            aicpu_data_parser = DataPreProcessParser(source_path, output_data_preprocess_aicpu)
-            aicpu_data_parser.execute()
-        except FileNotFoundError as err:
-            logger.exception(err)
+        aicpu_data_parser = DataPreProcessParser(source_path, output_data_preprocess_aicpu)
+        aicpu_data_parser.execute()
 
         # Parsing minddata AICPU profiling
         MinddataParser.execute(source_path, self._output_path, self._dev_id)
@@ -299,9 +271,7 @@ class Profiler:
             import mindspore.context as context
             context.set_context(enable_profiling=False)
         except ImportError:
-            logger.error("Profiling: fail to import context from mindspore.")
-        except ValueError:
-            logger.error("Profiling: fail to set context enable_profiling")
+            pass
 
     def _get_profiling_job_id(self):
         """Get profiling job id, which was generated by ada service.
@@ -448,3 +418,32 @@ class Profiler:
             filter_condition, op_type_order
         )
         return result
+
+
+    def _get_devid_and_devtarget(self):
+        """Get device id and target of this training."""
+
+        device_target = ""
+        dev_id = ""
+        try:
+            import mindspore.context as context
+            dev_id = str(context.get_context("device_id"))
+            device_target = context.get_context("device_target")
+        except ImportError:
+            logger.error("Profiling: fail to import context from mindspore.")
+        except ValueError as err:
+            logger.error("Profiling: fail to get context, %s", err)
+
+        if not dev_id or not dev_id.isdigit():
+            dev_id = os.getenv('DEVICE_ID')
+        if not dev_id or not dev_id.isdigit():
+            dev_id = "0"
+            logger.error("Fail to get DEVICE_ID, use 0 instead.")
+
+        if device_target and device_target != "Davinci" \
+            and device_target != "Ascend":
+            msg = ("Profiling: unsupport backend: %s" \
+                   % device_target)
+            raise RuntimeError(msg)
+
+        self._dev_id = dev_id