diff --git a/mindinsight/profiler/parser/aicpu_data_parser.py b/mindinsight/profiler/parser/aicpu_data_parser.py index 3b014996169163b288a02e351aa450d5656d24e8..04038f771d8a49e7a8d040803a709ccae4699272 100644 --- a/mindinsight/profiler/parser/aicpu_data_parser.py +++ b/mindinsight/profiler/parser/aicpu_data_parser.py @@ -41,9 +41,14 @@ class DataPreProcessParser: def __init__(self, input_path, output_filename): self._input_path = input_path self._output_filename = output_filename - self._source_file_name = self.get_source_file() + self._source_file_name = self._get_source_file() + self._ms_kernel_flag = 3 + self._other_kernel_flag = 6 + self._thread_flag = 7 + self._ms_kernel_run_end_index = 2 + self._other_kernel_run_end_index = 5 - def get_source_file(self): + def _get_source_file(self): """Get log file name, which was created by ada service.""" file_name = get_file_join_name(self._input_path, self._source_file_target) if not file_name: @@ -51,6 +56,30 @@ class DataPreProcessParser: file_name = get_file_join_name(data_path, self._source_file_target) return file_name + def _get_kernel_result(self, number, node_list, thread_list): + """Get the profiling data form different aicpu kernel""" + try: + if len(node_list) == self._ms_kernel_flag and len(thread_list) == self._thread_flag: + node_type_name = node_list[0].split(':')[-1] + run_end_index = self._ms_kernel_run_end_index + elif len(node_list) == self._other_kernel_flag and len(thread_list) == self._thread_flag: + node_type_name = node_list[0].split(':')[-1].split('/')[-1].split('-')[0] + run_end_index = self._other_kernel_run_end_index + else: + logger.warning("the data format can't support 'node_list':%s", str(node_list)) + return None + + run_start = node_list[1].split(':')[-1].split(' ')[0] + run_end = node_list[run_end_index].split(':')[-1].split(' ')[0] + total_time = thread_list[-1].split('=')[-1].split()[0] + dispatch_time = thread_list[-2].split('=')[-1].split()[0] + + return [number, node_type_name, total_time, dispatch_time, + run_start, run_end] + except IndexError as e: + logger.exception(e) + return None + def execute(self): """Execute the parser, get result data, and write it to the output file.""" @@ -63,23 +92,26 @@ class DataPreProcessParser: .replace(b'\x00', b' ___ '))[2:-1] ai_cpu_lines = ai_cpu_str.split(" ___ ") - node_list = list() + result_list = list() ai_cpu_total_time_summary = 0 # Node serial number. serial_number = 1 for i in range(len(ai_cpu_lines) - 1): node_line = ai_cpu_lines[i] thread_line = ai_cpu_lines[i + 1] + result = [] if "Node" in node_line and "Thread" in thread_line: # Get the node data from node_line - node_type_name = node_line.split(',')[0].split(':')[-1] - run_start = node_line.split(',')[1].split(':')[-1] - run_end = node_line.split(',')[2].split(':')[-1] - total_time = thread_line.split(',')[-1].split('=')[-1].split()[0] - dispatch_time = thread_line.split(',')[-2].split('=')[-1].split()[0] - node_list.append([serial_number, node_type_name, total_time, - dispatch_time, run_start, run_end]) + node_list = node_line.split(',') + thread_list = thread_line.split(',') + result = self._get_kernel_result(serial_number, node_list, thread_list) + + if result is None: + continue + + result_list.append(result) # Calculate the total time. + total_time = result[2] ai_cpu_total_time_summary += int(total_time) # Increase node serial number. serial_number += 1 @@ -87,11 +119,11 @@ class DataPreProcessParser: node_type_name = node_line.split(',')[0].split(':')[-1] logger.warning("The node type:%s cannot find thread data", node_type_name) - if node_list: - node_list.append(["AI CPU Total Time(us):", ai_cpu_total_time_summary]) + if result_list: + result_list.append(["AI CPU Total Time(us):", ai_cpu_total_time_summary]) fwrite_format(self._output_filename, data_source=self._dst_file_title, is_print=True, is_start=True) fwrite_format(self._output_filename, - data_source=tabulate(node_list, self._dst_file_column_title, + data_source=tabulate(result_list, self._dst_file_column_title, tablefmt='simple'), is_start=True, is_print=True) diff --git a/tests/ut/profiler/parser/test_aicpu_parser.py b/tests/ut/profiler/parser/test_aicpu_parser.py index 340f3ed6f727416b98d2f076b8bde5f9610fcd08..7f18274c301b21b179a1b9ab216dabc0842e00a6 100644 --- a/tests/ut/profiler/parser/test_aicpu_parser.py +++ b/tests/ut/profiler/parser/test_aicpu_parser.py @@ -52,10 +52,10 @@ class TestAicpuParser(TestCase): 'JOB_AICPU/data')) self.expect_dir = os.path.realpath(os.path.join(os.path.dirname(__file__), '../../../utils/resource/' - 'JOB_AICPU/data')) + 'JOB_AICPU/expect')) self.output_path = tempfile.mkdtemp(prefix='output_data_preprocess_aicpu_') self.output_file = os.path.join(self.output_path, 'output_data_preprocess_aicpu_0.txt') - self.expect_file = os.path.join(self.output_path, 'output_data_preprocess_aicpu_0.txt') + self.expect_file = os.path.join(self.expect_dir, 'output_data_preprocess_aicpu_0.txt') def test_aicpu_parser(self): """Test the class of Aicpu Parser.""" diff --git a/tests/utils/resource/JOB_AICPU/data/DATA_PREPROCESS.dev.AICPU.0.slice_0 b/tests/utils/resource/JOB_AICPU/data/DATA_PREPROCESS.dev.AICPU.0.slice_0 index ea5e36177e236fe9adbc82925686f4449f50d397..30fe89d6e98cd5d06cdd0492853db30cc1657947 100644 Binary files a/tests/utils/resource/JOB_AICPU/data/DATA_PREPROCESS.dev.AICPU.0.slice_0 and b/tests/utils/resource/JOB_AICPU/data/DATA_PREPROCESS.dev.AICPU.0.slice_0 differ diff --git a/tests/utils/resource/JOB_AICPU/expect/output_data_preprocess_aicpu_0.txt b/tests/utils/resource/JOB_AICPU/expect/output_data_preprocess_aicpu_0.txt index 7f3d739bac7bae23fb3f20486ac360f62bbeebd2..072eb79083c8840235f06ceac3cc5d414906f56d 100644 --- a/tests/utils/resource/JOB_AICPU/expect/output_data_preprocess_aicpu_0.txt +++ b/tests/utils/resource/JOB_AICPU/expect/output_data_preprocess_aicpu_0.txt @@ -2,4 +2,5 @@ serial_number node_type_name total_time(us) dispatch_time(us) ---------------------- ---------------- ---------------- ------------------- ----------- ---------- 1 InitData 1567 100 2298200409 2298200538 2 GetNext 989 87 2302769932 2302769980 -AI CPU Total Time(us): 2556 +3 TruncatedNormal 1566 105 4098200409 4098200538 +AI CPU Total Time(us): 4122