diff --git a/mindinsight/backend/profiler/profile_api.py b/mindinsight/backend/profiler/profile_api.py index 50dced5569cf6a42da63437672bb81b21169412b..dfaded8eca631ecd0fd4283012ad89da5170a682 100644 --- a/mindinsight/backend/profiler/profile_api.py +++ b/mindinsight/backend/profiler/profile_api.py @@ -23,13 +23,15 @@ import os from flask import Blueprint from flask import jsonify from flask import request +from flask import Response from marshmallow import ValidationError from mindinsight.conf import settings from mindinsight.datavisual.utils.tools import get_train_id, get_profiler_dir, \ - unquote_args, to_int + unquote_args, to_int, get_device_id from mindinsight.profiler.analyser.analyser_factory import AnalyserFactory from mindinsight.profiler.analyser.minddata_analyser import MinddataAnalyser +from mindinsight.profiler.proposer.compose_proposer import ComposeProposal from mindinsight.profiler.common.util import analyse_device_list_from_profiler_dir from mindinsight.profiler.common.validator.validate import validate_condition, \ validate_ui_proc, validate_minddata_pipeline_condition @@ -276,6 +278,44 @@ def get_profiler_abs_dir(requests): return profiler_dir_abs +@BLUEPRINT.route("/profile/summary/propose", methods=["GET"]) +def get_profile_summary_proposal(): + """ + Get summary profiling proposal. + + Returns: + str, the summary profiling proposal. + + Raises: + ParamValueError: If the parameters contain some errors. + + Examples: + >>> GET http://xxxx/v1/mindinsight/profile/summary/propose + """ + profiler_dir = get_profiler_dir(request) + train_id = get_train_id(request) + device_id = get_device_id(request) + if not profiler_dir or not train_id: + raise ParamValueError("No profiler_dir or train_id.") + + profiler_dir_abs = os.path.join(settings.SUMMARY_BASE_DIR, train_id, profiler_dir) + try: + profiler_dir_abs = validate_and_normalize_path(profiler_dir_abs, "profiler") + except ValidationError: + raise ParamValueError("Invalid profiler dir") + + step_trace_condition = {"filter_condition": {"mode": "proc", + "proc_name": "iteration_interval", + "step_id": 0}} + options = {'step_trace': {"iter_interval": step_trace_condition}} + + proposal_type_list = ['step_trace', 'minddata', 'minddata_pipeline', 'common'] + proposal_obj = ComposeProposal(profiler_dir_abs, device_id, proposal_type_list) + proposal_info = proposal_obj.get_proposal(options) + # Use json.dumps for orderly return + return Response(json.dumps(proposal_info), mimetype='application/json') + + @BLUEPRINT.route("/profile/minddata-pipeline/op-queue", methods=["POST"]) def get_minddata_pipeline_op_queue_info(): """ diff --git a/mindinsight/datavisual/utils/tools.py b/mindinsight/datavisual/utils/tools.py index 813bc686a1b96d56b77573b7187b7f4275914527..343a49d4b8db820217823c4a195af183d659ec69 100644 --- a/mindinsight/datavisual/utils/tools.py +++ b/mindinsight/datavisual/utils/tools.py @@ -176,6 +176,27 @@ def unquote_args(request, arg_name): return arg_value +def get_device_id(request): + """ + Get device ID from requst query string and unquote content. + + Args: + request (FlaskRequest): Http request instance. + + Returns: + str, unquoted device ID. + """ + device_id = request.args.get('device_id') + if device_id is not None: + try: + device_id = unquote(device_id, errors='strict') + except UnicodeDecodeError: + raise exceptions.UrlDecodeError('Unquote train id error with strict mode') + else: + device_id = "0" + return device_id + + def if_nan_inf_to_none(name, value): """ Transform value to None if it is NaN or Inf. @@ -197,6 +218,7 @@ def if_nan_inf_to_none(name, value): class Counter: """Count accumulator with limit checking.""" + def __init__(self, max_count=None, init_count=0): self._count = init_count self._max_count = max_count diff --git a/mindinsight/profiler/common/util.py b/mindinsight/profiler/common/util.py index ebb0826d39cb449df8ca13d538d28708579b984b..d7910ba1948889bdf8abccde835a0ecdcaafd700 100644 --- a/mindinsight/profiler/common/util.py +++ b/mindinsight/profiler/common/util.py @@ -144,3 +144,8 @@ def get_field_value(row_info, field_name, header, time_type='realtime'): value = to_millisecond(value) return value + +def get_options(options): + if options is None: + options = {} + return options diff --git a/mindinsight/profiler/proposer/__init__.py b/mindinsight/profiler/proposer/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..0809f3530a1cdea2ca3df6e1e5e40ee17ec73224 --- /dev/null +++ b/mindinsight/profiler/proposer/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ diff --git a/mindinsight/profiler/proposer/allproposers/__init__.py b/mindinsight/profiler/proposer/allproposers/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..93c78a43ba0c0785eb3969e9f31a56ffe01da981 --- /dev/null +++ b/mindinsight/profiler/proposer/allproposers/__init__.py @@ -0,0 +1,20 @@ +# Copyright 2019 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""All proposers.""" +from mindinsight.profiler.proposer.allproposers.common_proposer import CommonProposer +from mindinsight.profiler.proposer.allproposers.step_trace_proposer import StepTraceProposer + + +__all__ = ["CommonProposer", "StepTraceProposer"] diff --git a/mindinsight/profiler/proposer/allproposers/base_proposer.py b/mindinsight/profiler/proposer/allproposers/base_proposer.py new file mode 100644 index 0000000000000000000000000000000000000000..bcaf93498cae7f63ea7e18e499b097487b0353a7 --- /dev/null +++ b/mindinsight/profiler/proposer/allproposers/base_proposer.py @@ -0,0 +1,44 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""The proposer base class.""" +from abc import ABC, abstractmethod + +from mindinsight.profiler.common.log import logger +from mindinsight.profiler.analyser.analyser_factory import AnalyserFactory +from mindinsight.utils.exceptions import MindInsightException + + +class Proposer(ABC): + """The proposer base class.""" + + def __init__(self, profiling_path, device_id): + self.profiling_path = profiling_path + self.device_id = device_id + + def get_analyser_result(self, analyser_type, condition=None): + logger.debug("The Proposer 'analyser_type' is %s, 'options' is %s", str(analyser_type), str(condition)) + analyser_result = {} + try: + analyser = AnalyserFactory.instance().get_analyser(analyser_type, self.profiling_path, self.device_id) + analyser_result = analyser.query(condition) + logger.debug("The 'analyser_result' is %s, the 'condition' is %s.", str(analyser_result), str(condition)) + except MindInsightException as e: + logger.warning(e) + return analyser_result + + @abstractmethod + def analyze(self, options=None): + """analysis and get proposal.""" + raise NotImplementedError("Must define analyze function to inherit Class Propose") diff --git a/mindinsight/profiler/proposer/allproposers/common_proposer.py b/mindinsight/profiler/proposer/allproposers/common_proposer.py new file mode 100644 index 0000000000000000000000000000000000000000..8b8d333d72509c063df57bd63e6602624d41848d --- /dev/null +++ b/mindinsight/profiler/proposer/allproposers/common_proposer.py @@ -0,0 +1,42 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""The common proposer.""" +from collections import OrderedDict + +from mindinsight.profiler.common.util import get_options +from mindinsight.profiler.proposer.allproposers.base_proposer import Proposer +from mindinsight.profiler.common.log import logger + + +class CommonProposer(Proposer): + """The common proposer.""" + def __init__(self, profiling_dir, device_id): + super().__init__(profiling_dir, device_id) + self.__proposal_dict = OrderedDict() + self.__proposal_dict["common-profiler_tutorial"] = None + + def analyze(self, options=None): + """ + Get the proposal from proposer. + + Args: + options: options for proposer analysis + + Returns: + dict, the proposal from proposer instance. + """ + logger.info('The CommonProposer is running') + options = get_options(options) + return self.__proposal_dict diff --git a/mindinsight/profiler/proposer/allproposers/step_trace_proposer.py b/mindinsight/profiler/proposer/allproposers/step_trace_proposer.py new file mode 100644 index 0000000000000000000000000000000000000000..507876f975e0b2b7c4f037bef79b4eac9d7d1492 --- /dev/null +++ b/mindinsight/profiler/proposer/allproposers/step_trace_proposer.py @@ -0,0 +1,76 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""The step trace proposer.""" +from collections import OrderedDict + +from mindinsight.profiler.common.util import get_options +from mindinsight.profiler.proposer.allproposers.base_proposer import Proposer +from mindinsight.profiler.common.log import logger + + +class StepTraceProposer(Proposer): + """The step trace proposer.""" + + def __init__(self, profiling_dir, device_id): + super().__init__(profiling_dir, device_id) + self.__step_trace_iter_interval_threshold = 0.5 + self.__proposer_type = "step_trace" + self.__proposal_dict = OrderedDict() + self.__iter_interval_label = "step_trace-iter_interval" + + def analyze(self, options=None): + """ + Get the proposal from proposer. + + Args: + options (dict): options for proposer analysis. + - step_trace: include optional parameters for step trace,The dictionary key is iter_interval + used to get the analyser options for iteration interval time. + + Returns: + dict, the proposal from proposer instance,the dictionary key is a language internationalization + label, and the value is used to format the value in the language internationalization string. + + Examples: + >>> proposer_type = 'step_trace' + >>> proposer = ProposerFactory.instance().get_proposer(proposer_type, self.profiling_dir, self.device_id) + >>> result = proposer.analyze(options) + """ + logger.info("The StepTraceProposer is running") + + options = get_options(options) + logger.debug("The StepTraceProposer 'options' is %s", str(options)) + step_trace_condition = options.get("step_trace", {}) + # Get the proposals of iteration interval. + self._iter_interval_analyze(step_trace_condition) + return self.__proposal_dict + + def _iter_interval_analyze(self, step_trace_condition): + """Get the proposals of iteration interval.""" + iter_interval_dict = OrderedDict() + default_iter_interval_lst = [0] + iter_interval_condition = step_trace_condition.get("iter_interval", {}) + analyser_result = self.get_analyser_result(self.__proposer_type, condition=iter_interval_condition) + iter_interval_length_lst = analyser_result.get("info", {}).get("iteration_interval", + default_iter_interval_lst) + logger.debug("The 'iter_interval_length_lst' is %s", str(iter_interval_length_lst)) + # Check the iter_interval_length_lst. + if not isinstance(iter_interval_length_lst, list) or not iter_interval_length_lst: + logger.warning("The 'iter_interval_length_lst' is %s, it is null or not a list", + str(iter_interval_length_lst)) + else: + if iter_interval_length_lst[0] > self.__step_trace_iter_interval_threshold: + iter_interval_dict[self.__iter_interval_label] = [str(self.__step_trace_iter_interval_threshold)] + self.__proposal_dict.update(iter_interval_dict) diff --git a/mindinsight/profiler/proposer/compose_proposer.py b/mindinsight/profiler/proposer/compose_proposer.py new file mode 100644 index 0000000000000000000000000000000000000000..d1ae2fc021981f0457a6ca2c07bbc14074cfee68 --- /dev/null +++ b/mindinsight/profiler/proposer/compose_proposer.py @@ -0,0 +1,84 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""The Compose Proposals.""" +from collections import OrderedDict + +from mindinsight.profiler.common.log import logger +from mindinsight.profiler.common.util import get_options +from mindinsight.profiler.proposer.proposer_factory import ProposerFactory + + +class ComposeProposal: + """Get the proposals from multiple different proposers.""" + def __init__(self, profiling_path, device_id, type_list=None): + self.profiling_path = profiling_path + self.device_id = device_id + self.compose_proposer_type_list = type_list + # Postfix of category label, used for UI to identify the label as category label. + self.type_label_postfix = "-proposer_type_label" + + def get_proposal(self, options=None): + """ + Get compose proposals. + + Args: + options (dict): options for composed proposal. + - compose_proposal_result: execution results of the already running proposers. + - step_trace: include optional parameters for step trace,The dictionary key is iter_interval + used to get the analyser options for iteration interval time. + + Returns: + dict, the proposals from multiple different proposers. + + Examples: + >>> type_list = ['common', 'step_trace'] + >>> condition = {"filter_condition": {'mode': "proc", "proc_name": "iteration_interval"}} + >>> options = {'step_trace': {"iter_interval": condition}} + >>> cp = ComposeProposal(self.profiling_dir, self.device_id, type_list) + >>> result_proposal = cp.get_proposal(options=options) + """ + logger.info("The ComposeProposal is running") + options = get_options(options) + logger.debug("The 'options' is %s", str(options)) + # The flag whether to write category label. + type_label_flag = options.get("type_label_flag", True) + compose_proposal_result = OrderedDict() + logger.debug("The 'compose_proposer_type_list' is %s", str(self.compose_proposer_type_list)) + for proposer_type in self.compose_proposer_type_list: + proposer = ProposerFactory.instance().get_proposer(proposer_type, self.profiling_path, self.device_id) + + if proposer is None: + continue + + # Write the result of proposals to option for other proposer to get. + options["compose_proposal_result"] = compose_proposal_result + + result = proposer.analyze(options) + # Insert category label. + if result and type_label_flag: + proposer_type_label = proposer_type + "-type_label" + # Get the name of the category label, the default is the same as the proposer type. + type_label_name = options.get(proposer_type_label, proposer_type) + # Add postfix to category label name + type_proposal_label = type_label_name + self.type_label_postfix + compose_proposal_result[type_proposal_label] = None + # Merge results to the proposals dictionary. + compose_proposal_result.update(result) + elif result and not type_label_flag: + # Merge results to the proposals dictionary. + compose_proposal_result.update(result) + + logger.debug("The 'compose_proposal_result' is %s", str(compose_proposal_result)) + return compose_proposal_result diff --git a/mindinsight/profiler/proposer/proposer_factory.py b/mindinsight/profiler/proposer/proposer_factory.py new file mode 100644 index 0000000000000000000000000000000000000000..d542769ee8a8f359f9b3c2117e4417b7c380bd51 --- /dev/null +++ b/mindinsight/profiler/proposer/proposer_factory.py @@ -0,0 +1,67 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""The proposer factory.""" +import threading + +import mindinsight.profiler.proposer.allproposers as proposer_module +from mindinsight.profiler.common.log import logger + + +class ProposerFactory: + """The Proposer factory is used to create Proposer special instance.""" + _lock = threading.Lock() + _instance = None + + def __new__(cls, *args, **kwargs): + if cls._instance is None: + with cls._lock: + if cls._instance is None: + cls._instance = super().__new__(cls, *args, **kwargs) + return cls._instance + + @classmethod + def instance(cls): + """The factory instance.""" + if cls._instance is None: + cls._instance = cls() + return cls._instance + + def get_proposer(self, proposer_type, *args): + """ + Get the specified proposer according to the proposer type. + + Args: + proposer_type (str): The proposer type. + args (list): The parameters required for the specific proposer class. + + Returns: + Proposer, the specified proposer instance. + + Examples: + >>> proposer_type = 'step_trace' + >>> proposer = ProposerFactory.instance().get_proposer(proposer_type, self.profiling_dir, self.device_id) + + """ + logger.debug("The 'proposer_type' is %s,The 'args' is %s", proposer_type, str(args)) + proposer_instance = None + sub_name = proposer_type.split('_') + proposer_class_name = ''.join([name.capitalize() for name in sub_name]) + proposer_class_name += 'Proposer' + + if hasattr(proposer_module, proposer_class_name): + proposer_instance = getattr(proposer_module, proposer_class_name)(*args) + else: + logger.warning("The proposer class %s does not exist.", proposer_class_name) + return proposer_instance diff --git a/tests/ut/profiler/proposer/__init__.py b/tests/ut/profiler/proposer/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e30774307ca2107b3a81c071ad33c042ef924790 --- /dev/null +++ b/tests/ut/profiler/proposer/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ diff --git a/tests/ut/profiler/proposer/test_proposer.py b/tests/ut/profiler/proposer/test_proposer.py new file mode 100644 index 0000000000000000000000000000000000000000..e31681d8ce331cf52a07374403ce40888dd59b7b --- /dev/null +++ b/tests/ut/profiler/proposer/test_proposer.py @@ -0,0 +1,117 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Test the proposer module.""" +import os +from unittest import TestCase +from collections import OrderedDict + +from mindinsight.profiler.proposer.proposer_factory import ProposerFactory +from mindinsight.profiler.proposer.compose_proposer import ComposeProposal + + +class TestPropose(TestCase): + """Test the class of Proposer.""" + + def setUp(self) -> None: + """Initialization before test case execution.""" + self.profiling_dir = os.path.realpath(os.path.join(os.path.dirname(__file__), + '../../../utils/resource/profiler')) + self.device_id = 0 + self.common_proposal_dict = OrderedDict() + self.common_proposal_dict["common-profiler_tutorial"] = None + self.step_trace_proposal_dict = OrderedDict() + self.step_trace_proposal_dict["step_trace-iter_interval"] = ['0.5'] + + def test_propose_compose(self): + """Test the class of ComposeProposal.""" + proposal_dict = OrderedDict() + proposal_dict["step_trace-proposer_type_label"] = None + proposal_dict.update(self.step_trace_proposal_dict) + proposal_dict["common-proposer_type_label"] = None + proposal_dict.update(self.common_proposal_dict) + type_list = ['step_trace', 'common'] + condition = {"filter_condition": {'mode': "proc", + "proc_name": "iteration_interval", + "step_id": 0}} + options = {'step_trace': {"iter_interval": condition}} + cp = ComposeProposal(self.profiling_dir, self.device_id, type_list) + result = cp.get_proposal(options=options) + self.assertDictEqual(proposal_dict, result) + + def test_propose_compose_exception(self): + """Test the class of ComposeProposal.""" + profiling_dir = os.path.realpath(os.path.join(os.path.dirname(__file__), + '../../../utils/resource/test')) + proposal_dict = OrderedDict() + proposal_dict["common-proposer_type_label"] = None + proposal_dict.update(self.common_proposal_dict) + type_list = ['step_trace', 'common'] + condition = {"filter_condition": {'mode': "proc", + "proc_name": "iteration_interval", + "step_id": 0}} + options = {'step_trace': {"iter_interval": condition}} + cp = ComposeProposal(profiling_dir, self.device_id, type_list) + result = cp.get_proposal(options=options) + self.assertDictEqual(proposal_dict, result) + + def test_propose_compose_type_label(self): + """Test the class of ComposeProposal.""" + proposal_dict = OrderedDict() + proposal_dict["test_label-proposer_type_label"] = None + proposal_dict.update(self.step_trace_proposal_dict) + proposal_dict["common-proposer_type_label"] = None + proposal_dict.update(self.common_proposal_dict) + type_list = ['step_trace', 'common'] + condition = {"filter_condition": {'mode': "proc", + "proc_name": "iteration_interval", + "step_id": 0}} + options = {'step_trace': {"iter_interval": condition}, "step_trace-type_label": "test_label"} + cp = ComposeProposal(self.profiling_dir, self.device_id, type_list) + result = cp.get_proposal(options=options) + self.assertDictEqual(proposal_dict, result) + + def test_propose_compose_type_label_flag(self): + """Test the class of ComposeProposal.""" + proposal_dict = OrderedDict() + proposal_dict.update(self.step_trace_proposal_dict) + proposal_dict.update(self.common_proposal_dict) + + type_list = ['step_trace', 'common'] + condition = {"filter_condition": {'mode': "proc", + "proc_name": "iteration_interval", + "step_id": 0}} + options = {'step_trace': {"iter_interval": condition}, + "step_trace-type_label": "test_label", + "type_label_flag": False} + cp = ComposeProposal(self.profiling_dir, self.device_id, type_list) + result = cp.get_proposal(options=options) + self.assertDictEqual(proposal_dict, result) + + def test_propose_common(self): + proposer_type = 'common' + proposer = ProposerFactory.instance().get_proposer(proposer_type, self.profiling_dir, self.device_id) + result = proposer.analyze() + self.assertDictEqual(self.common_proposal_dict, result) + + def test_propose_step_trace(self): + """Test the class of step trace.""" + proposer_type = 'step_trace' + proposer = ProposerFactory.instance().get_proposer(proposer_type, self.profiling_dir, self.device_id) + condition = {"filter_condition": {'mode': "proc", + "proc_name": "iteration_interval", + "step_id": 0}} + options = {'step_trace': {"iter_interval": condition}} + result = proposer.analyze(options) + self.assertDictEqual(self.step_trace_proposal_dict, result) diff --git a/tests/utils/resource/profiler/step_trace_raw_0_detail_time.csv b/tests/utils/resource/profiler/step_trace_raw_0_detail_time.csv index 4953c6abf9e690b855660cd9e912e34bafac8397..0e923ccad8471e9d5d7a02b4edbc2c4984a58998 100755 --- a/tests/utils/resource/profiler/step_trace_raw_0_detail_time.csv +++ b/tests/utils/resource/profiler/step_trace_raw_0_detail_time.csv @@ -1,22 +1,22 @@ -step_num,start_point,end_point,total,fp_point,bp_point,iteration_interval,fp_and_bp,tail,stream_10_parallel_0_start_point,stream_10_parallel_0_end_point,stream_10_parallel_0,stream_10_parallel_1_start_point,stream_10_parallel_1_end_point,stream_10_parallel_1,stream_10_parallel_2_start_point,stream_10_parallel_2_end_point,stream_10_parallel_2,stream_11_parallel_0_start_point,stream_11_parallel_0_end_point,stream_11_parallel_0 -1,45000030081,45004033128,4003047,45000030081,45001733025,0,1702944,2300103,45000042679,45000060275,17596,45001048152,45001346254,298102,45002247411,45002448354,200943,45000049687,45000075987,26300 -2,45004033128,45017085658,13052530,45013070937,45014785314,9037809,1714377,2300344,45013085379,45013105429,20050,45014087119,45014385136,298017,45015297166,45015504449,207283,45013084925,45013118334,33409 -3,45017085658,45030119392,13033734,45026116231,45027818443,9030573,1702212,2300949,45026131909,45026150554,18645,45027134392,45027430418,296026,45028337093,45028537767,200674,45026129217,45026160937,31720 -4,45030119392,45043158607,13039215,45039152348,45040856975,9032956,1704627,2301632,45039169890,45039188966,19076,45040169338,45040466770,297432,45041374122,45041567754,193632,45039171681,45039193865,22184 -5,45043158607,45056198128,13039521,45052190932,45053898028,9032325,1707096,2300100,45052207675,45052222642,14967,45053204442,45053505540,301098,45054413207,45054616536,203329,45052201931,45052237599,35668 -6,45056198128,45069239564,13041436,45065233106,45066939463,9034978,1706357,2300101,45065245482,45065272534,27052,45066248423,45066546419,297996,45067455113,45067659145,204032,45065245817,45065279896,34079 -7,45069239564,45082281383,13041819,45078274997,45079980193,9035433,1705196,2301190,45078293910,45078312935,19025,45079287754,45079593841,306087,45080492957,45080691395,198438,45078292067,45078322277,30210 -8,45082281383,45095336378,13054995,45091321488,45093036084,9040105,1714596,2300294,45091338628,45091359138,20510,45092338469,45092638994,300525,45093554195,45093747470,193275,45091341356,45091369667,28311 -9,45095336378,45108372225,13035847,45104363079,45106071009,9026701,1707930,2301216,45104374524,45104400088,25564,45105378751,45105683029,304278,45106587481,45106785336,197855,45104382131,45104410852,28721 -10,45108372225,45121412413,13040188,45117401873,45119111301,9029648,1709428,2301112,45117417721,45117439668,21947,45118413083,45118718050,304967,45119629347,45119829996,200649,45117421502,45117446718,25216 -11,45121412413,45134477662,13065249,45130459598,45132175723,9047185,1716125,2301939,45130478168,45130498936,20768,45131477957,45131775220,297263,45132691645,45132893707,202062,45130470285,45130501652,31367 -12,45134477662,45147533298,13055636,45143521860,45145232553,9044198,1710693,2300745,45143533787,45143557293,23506,45144533554,45144841545,307991,45145744997,45145952255,207258,45143537383,45143563466,26083 -13,45147533298,45160588134,13054836,45156570201,45158286694,9036903,1716493,2301440,45156581069,45156609506,28437,45157581617,45157880841,299224,45158806166,45158999875,193709,45156589050,45156615664,26614 -14,45160588134,45173640064,13051930,45169625906,45171339426,9037772,1713520,2300638,45169637432,45169661754,24322,45170639482,45170940949,301467,45171853721,45172056606,202885,45169644605,45169673410,28805 -15,45173640064,45186671634,13031570,45182666696,45184371430,9026632,1704734,2300204,45182678355,45182698471,20116,45183679568,45183981082,301514,45184887156,45185083035,195879,45182680062,45182708455,28393 -16,45186671634,45199720448,13048814,45195714716,45197420410,9043082,1705694,2300038,45195728993,45195754646,25653,45196732493,45197028048,295555,45197934921,45198139237,204316,45195733069,45195764102,31033 -17,45199720448,45212762605,13042157,45208758416,45210460864,9037968,1702448,2301741,45208771010,45208790367,19357,45209773548,45210074988,301440,45210978277,45211173577,195300,45208773143,45208803280,30137 -18,45212762605,45225814601,13051996,45221801814,45223514580,9039209,1712766,2300021,45221815911,45221839644,23733,45222819211,45223114544,295333,45224031469,45224234043,202574,45221812106,45221849103,36997 -19,45225814601,45238848430,13033829,45234842015,45236548356,9027414,1706341,2300074,45234855444,45234876469,21025,45235853358,45236160825,307467,45237063061,45237260964,197903,45234857141,45234882976,25835 -20,45238848430,45251899738,13051308,45247879385,45249598280,9030955,1718895,2301458,45247896725,45247917316,20591,45248896361,45249193681,297320,45250117916,45250315651,197735,45247894228,45247926723,32495 --,45121436513,45134482124,13045611,45130471874,45132181322,9035360,1709449,2300802,45130486422,45130508229,21808,45131486785,45131787364,300579,45132697369,45132897305,199936,45130487458,45130517315,29857 +step_num,start_point,end_point,total,fp_point,bp_point,iteration_interval,fp_and_bp,tail,stream_10_parallel_0_start_point,stream_10_parallel_0_end_point,stream_10_parallel_0,stream_10_parallel_1_start_point,stream_10_parallel_1_end_point,stream_10_parallel_1,stream_10_parallel_2_start_point,stream_10_parallel_2_end_point,stream_10_parallel_2,stream_11_parallel_0_start_point,stream_11_parallel_0_end_point,stream_11_parallel_0 +1,45000030081,45004033128,4003047,45000030081,45001733025,0,1702944,2300103,45000042679,45000060275,17596,45001048152,45001346254,298102,45002247411,45002448354,200943,45000049687,45000075987,26300 +2,45004033128,45017085658,13052530,45013070937,45014785314,9037809,1714377,2300344,45013085379,45013105429,20050,45014087119,45014385136,298017,45015297166,45015504449,207283,45013084925,45013118334,33409 +3,45017085658,45030119392,13033734,45026116231,45027818443,9030573,1702212,2300949,45026131909,45026150554,18645,45027134392,45027430418,296026,45028337093,45028537767,200674,45026129217,45026160937,31720 +4,45030119392,45043158607,13039215,45039152348,45040856975,9032956,1704627,2301632,45039169890,45039188966,19076,45040169338,45040466770,297432,45041374122,45041567754,193632,45039171681,45039193865,22184 +5,45043158607,45056198128,13039521,45052190932,45053898028,9032325,1707096,2300100,45052207675,45052222642,14967,45053204442,45053505540,301098,45054413207,45054616536,203329,45052201931,45052237599,35668 +6,45056198128,45069239564,13041436,45065233106,45066939463,9034978,1706357,2300101,45065245482,45065272534,27052,45066248423,45066546419,297996,45067455113,45067659145,204032,45065245817,45065279896,34079 +7,45069239564,45082281383,13041819,45078274997,45079980193,9035433,1705196,2301190,45078293910,45078312935,19025,45079287754,45079593841,306087,45080492957,45080691395,198438,45078292067,45078322277,30210 +8,45082281383,45095336378,13054995,45091321488,45093036084,9040105,1714596,2300294,45091338628,45091359138,20510,45092338469,45092638994,300525,45093554195,45093747470,193275,45091341356,45091369667,28311 +9,45095336378,45108372225,13035847,45104363079,45106071009,9026701,1707930,2301216,45104374524,45104400088,25564,45105378751,45105683029,304278,45106587481,45106785336,197855,45104382131,45104410852,28721 +10,45108372225,45121412413,13040188,45117401873,45119111301,9029648,1709428,2301112,45117417721,45117439668,21947,45118413083,45118718050,304967,45119629347,45119829996,200649,45117421502,45117446718,25216 +11,45121412413,45134477662,13065249,45130459598,45132175723,9047185,1716125,2301939,45130478168,45130498936,20768,45131477957,45131775220,297263,45132691645,45132893707,202062,45130470285,45130501652,31367 +12,45134477662,45147533298,13055636,45143521860,45145232553,9044198,1710693,2300745,45143533787,45143557293,23506,45144533554,45144841545,307991,45145744997,45145952255,207258,45143537383,45143563466,26083 +13,45147533298,45160588134,13054836,45156570201,45158286694,9036903,1716493,2301440,45156581069,45156609506,28437,45157581617,45157880841,299224,45158806166,45158999875,193709,45156589050,45156615664,26614 +14,45160588134,45173640064,13051930,45169625906,45171339426,9037772,1713520,2300638,45169637432,45169661754,24322,45170639482,45170940949,301467,45171853721,45172056606,202885,45169644605,45169673410,28805 +15,45173640064,45186671634,13031570,45182666696,45184371430,9026632,1704734,2300204,45182678355,45182698471,20116,45183679568,45183981082,301514,45184887156,45185083035,195879,45182680062,45182708455,28393 +16,45186671634,45199720448,13048814,45195714716,45197420410,9043082,1705694,2300038,45195728993,45195754646,25653,45196732493,45197028048,295555,45197934921,45198139237,204316,45195733069,45195764102,31033 +17,45199720448,45212762605,13042157,45208758416,45210460864,9037968,1702448,2301741,45208771010,45208790367,19357,45209773548,45210074988,301440,45210978277,45211173577,195300,45208773143,45208803280,30137 +18,45212762605,45225814601,13051996,45221801814,45223514580,9039209,1712766,2300021,45221815911,45221839644,23733,45222819211,45223114544,295333,45224031469,45224234043,202574,45221812106,45221849103,36997 +19,45225814601,45238848430,13033829,45234842015,45236548356,9027414,1706341,2300074,45234855444,45234876469,21025,45235853358,45236160825,307467,45237063061,45237260964,197903,45234857141,45234882976,25835 +20,45238848430,45251899738,13051308,45247879385,45249598280,9030955,1718895,2301458,45247896725,45247917316,20591,45248896361,45249193681,297320,45250117916,45250315651,197735,45247894228,45247926723,32495 +-,45121436513,45134482124,13045611,45130471874,45132181322,9035360,1709449,2300802,45130486422,45130508229,21808,45131486785,45131787364,300579,45132697369,45132897305,199936,45130487458,45130517315,29857