diff --git a/graphengine b/graphengine index 45ca7863ac6410c8e2f83168481ddc6b43bcea33..c54db4343f83cb0c15cc3b5c9755926de27fa3af 160000 --- a/graphengine +++ b/graphengine @@ -1 +1 @@ -Subproject commit 45ca7863ac6410c8e2f83168481ddc6b43bcea33 +Subproject commit c54db4343f83cb0c15cc3b5c9755926de27fa3af diff --git a/mindspore/ccsrc/device/ascend/ascend_kernel_runtime.cc b/mindspore/ccsrc/device/ascend/ascend_kernel_runtime.cc index 0333c796fd90b92d612a072e7fdf3ffb4968ac8f..0f9ee725533e0ee8a91bee28e195950e620a94e3 100644 --- a/mindspore/ccsrc/device/ascend/ascend_kernel_runtime.cc +++ b/mindspore/ccsrc/device/ascend/ascend_kernel_runtime.cc @@ -367,7 +367,8 @@ bool AscendKernelRuntime::LoadTask(const session::KernelGraph *graph) { } if (ProfilingManager::GetInstance().IsProfiling()) { auto task_ids = ge::model_runner::ModelRunner::Instance().GetTaskIdList(model_iter->first); - ProfilingUtils::ReportProfilingData(task_ids, NOT_NULL(graph)); + auto stream_ids = ge::model_runner::ModelRunner::Instance().GetStreamIdList(model_iter->first); + ProfilingUtils::ReportProfilingData(task_ids, stream_ids, NOT_NULL(graph)); } return true; } diff --git a/mindspore/ccsrc/device/ascend/profiling/profiling_utils.cc b/mindspore/ccsrc/device/ascend/profiling/profiling_utils.cc index 62e18793b2cff182b3dd7210c3f1f035bec71e31..1d137a5d72237025e716824fbb11ddf69cb12845 100644 --- a/mindspore/ccsrc/device/ascend/profiling/profiling_utils.cc +++ b/mindspore/ccsrc/device/ascend/profiling/profiling_utils.cc @@ -302,7 +302,7 @@ bool ProfilingUtils::ValidComputeGraph(NotNull gra return false; } -void ProfilingUtils::ReportProfilingData(const std::vector &task_ids, +void ProfilingUtils::ReportProfilingData(const std::vector &task_ids, const std::vector &stream_ids, NotNull graph) { if (!ValidComputeGraph(graph)) { MS_LOG(WARNING) << "Not a valid compute graph:" << graph->graph_id(); @@ -319,6 +319,7 @@ void ProfilingUtils::ReportProfilingData(const std::vector &task_ids, MS_EXCEPTION_IF_NULL(context); TaskDescReporter task_reporter(context->device_id(), "vm.task_desc_info", ret->second); task_reporter.set_task_ids(task_ids); + task_reporter.set_stream_ids(stream_ids); task_reporter.ReportData(); GraphDescReporter graph_reporter(context->device_id(), "vm.graph_desc_info", ret->second); diff --git a/mindspore/ccsrc/device/ascend/profiling/profiling_utils.h b/mindspore/ccsrc/device/ascend/profiling/profiling_utils.h index 39ea80a2e9a76564ce65653cc4bdbbf771e766a4..982ff15d60d14dff77573c02713ea051889389fc 100644 --- a/mindspore/ccsrc/device/ascend/profiling/profiling_utils.h +++ b/mindspore/ccsrc/device/ascend/profiling/profiling_utils.h @@ -87,7 +87,8 @@ class ProfilingUtils { // Mapping task_id and kernel name for device to generate the time cost of specific kernel. // Device calculate the time cost of the task which is marked by task id. // But we need data of (kernel name , time cost) - static void ReportProfilingData(const std::vector &task_ids, NotNull graph); + static void ReportProfilingData(const std::vector &task_ids, const std::vector &stream_ids, + NotNull graph); // Get profiling trace point from envs. // export PROFILING_FP_START='full name of the first cnode to execute' diff --git a/mindspore/ccsrc/device/ascend/profiling/reporter/task_desc_reporter.cc b/mindspore/ccsrc/device/ascend/profiling/reporter/task_desc_reporter.cc index 8f59e72613718c8578fe977e9bae958146f85c96..0f2068eb04ee288d4111a2fc4755dd4e6459719a 100644 --- a/mindspore/ccsrc/device/ascend/profiling/reporter/task_desc_reporter.cc +++ b/mindspore/ccsrc/device/ascend/profiling/reporter/task_desc_reporter.cc @@ -40,12 +40,22 @@ void TaskDescReporter::ReportData() { auto ascend_kernel_mod = dynamic_cast(kernel_mod); MS_EXCEPTION_IF_NULL(node); MS_EXCEPTION_IF_NULL(ascend_kernel_mod); - auto desc_ptr = std::make_shared(node->fullname_with_scope(), task_ids_[task_index++], - ascend_kernel_mod->block_dim(), ascend_kernel_mod->stream_id()); + // Check task_id and stream_id valid + CheckStreamTaskValid(task_index, task_index); + auto desc_ptr = std::make_shared(node->fullname_with_scope(), task_ids_[task_index], + ascend_kernel_mod->block_dim(), stream_ids_[task_index]); prof_desc_.emplace_back(desc_ptr); + ++task_index; } DescReporter::ReportData(); } + +void TaskDescReporter::CheckStreamTaskValid(uint32_t task_id, uint32_t stream_id) { + if (task_id >= task_ids_.size() || stream_id >= stream_ids_.size()) { + MS_LOG(EXCEPTION) << "Index invalid. task_id:" << task_id << ", task_ids.size:" << task_ids_.size() + << ", stream_id:" << stream_id << ", stream_ids.size:" << stream_ids_.size(); + } +} } // namespace ascend } // namespace device } // namespace mindspore diff --git a/mindspore/ccsrc/device/ascend/profiling/reporter/task_desc_reporter.h b/mindspore/ccsrc/device/ascend/profiling/reporter/task_desc_reporter.h index c1f70cacaf017e50c2b03851dc88bf6f830252b8..21a17b84fb9ee35f5fb4d6c7002cbe6ed32a5183 100644 --- a/mindspore/ccsrc/device/ascend/profiling/reporter/task_desc_reporter.h +++ b/mindspore/ccsrc/device/ascend/profiling/reporter/task_desc_reporter.h @@ -32,9 +32,12 @@ class TaskDescReporter : public DescReporter { ~TaskDescReporter() override = default; void ReportData() override; void set_task_ids(const std::vector &task_ids) { task_ids_ = task_ids; } + void set_stream_ids(const std::vector &stream_ids) { stream_ids_ = stream_ids; } private: std::vector task_ids_; + std::vector stream_ids_; + void CheckStreamTaskValid(uint32_t task_id, uint32_t stream_id); }; } // namespace ascend } // namespace device diff --git a/tests/ut/cpp/stub/ge/ge_task_launch_stub.cc b/tests/ut/cpp/stub/ge/ge_task_launch_stub.cc index b77b83c7fe757416c128808a8046493cafd0c3c2..a3a991247cc2a65222c08c5d15cff98c1ae855a3 100644 --- a/tests/ut/cpp/stub/ge/ge_task_launch_stub.cc +++ b/tests/ut/cpp/stub/ge/ge_task_launch_stub.cc @@ -40,6 +40,11 @@ const std::vector &ModelRunner::GetTaskIdList(uint32_t model_id) const static std::vector task_id_list; return task_id_list; } + +const std::vector &ModelRunner::GetStreamIdList(uint32_t model_id) const { + static std::vector stream_id_list; + return stream_id_list; +} } // namespace model_runner } // namespace ge