!269 Support proposer for profiler

Merge pull request !269 from 治愈系潇洒哥/master

mindinsight/backend/profiler/profile_api.py

@@ -23,13 +23,15 @@ import os
 from flask import Blueprint
 from flask import jsonify
 from flask import request
+from flask import Response
 from marshmallow import ValidationError
 
 from mindinsight.conf import settings
 from mindinsight.datavisual.utils.tools import get_train_id, get_profiler_dir, \
-    unquote_args, to_int
+    unquote_args, to_int, get_device_id
 from mindinsight.profiler.analyser.analyser_factory import AnalyserFactory
 from mindinsight.profiler.analyser.minddata_analyser import MinddataAnalyser
+from mindinsight.profiler.proposer.compose_proposer import ComposeProposal
 from mindinsight.profiler.common.util import analyse_device_list_from_profiler_dir
 from mindinsight.profiler.common.validator.validate import validate_condition, \
     validate_ui_proc, validate_minddata_pipeline_condition
@@ -276,6 +278,44 @@ def get_profiler_abs_dir(requests):
     return profiler_dir_abs
 
 
+@BLUEPRINT.route("/profile/summary/propose", methods=["GET"])
+def get_profile_summary_proposal():
+    """
+    Get summary profiling proposal.
+
+    Returns:
+        str, the summary profiling proposal.
+
+    Raises:
+        ParamValueError: If the parameters contain some errors.
+
+    Examples:
+        >>> GET http://xxxx/v1/mindinsight/profile/summary/propose
+    """
+    profiler_dir = get_profiler_dir(request)
+    train_id = get_train_id(request)
+    device_id = get_device_id(request)
+    if not profiler_dir or not train_id:
+        raise ParamValueError("No profiler_dir or train_id.")
+
+    profiler_dir_abs = os.path.join(settings.SUMMARY_BASE_DIR, train_id, profiler_dir)
+    try:
+        profiler_dir_abs = validate_and_normalize_path(profiler_dir_abs, "profiler")
+    except ValidationError:
+        raise ParamValueError("Invalid profiler dir")
+
+    step_trace_condition = {"filter_condition": {"mode": "proc",
+                                                 "proc_name": "iteration_interval",
+                                                 "step_id": 0}}
+    options = {'step_trace': {"iter_interval": step_trace_condition}}
+
+    proposal_type_list = ['step_trace', 'minddata', 'minddata_pipeline', 'common']
+    proposal_obj = ComposeProposal(profiler_dir_abs, device_id, proposal_type_list)
+    proposal_info = proposal_obj.get_proposal(options)
+    # Use json.dumps for orderly return
+    return Response(json.dumps(proposal_info), mimetype='application/json')
+
+
 @BLUEPRINT.route("/profile/minddata-pipeline/op-queue", methods=["POST"])
 def get_minddata_pipeline_op_queue_info():
     """

mindinsight/datavisual/utils/tools.py

@@ -176,6 +176,27 @@ def unquote_args(request, arg_name):
     return arg_value
 
 
+def get_device_id(request):
+    """
+    Get device ID from requst query string and unquote content.
+
+    Args:
+        request (FlaskRequest): Http request instance.
+
+    Returns:
+        str, unquoted device ID.
+    """
+    device_id = request.args.get('device_id')
+    if device_id is not None:
+        try:
+            device_id = unquote(device_id, errors='strict')
+        except UnicodeDecodeError:
+            raise exceptions.UrlDecodeError('Unquote train id error with strict mode')
+    else:
+        device_id = "0"
+    return device_id
+
+
 def if_nan_inf_to_none(name, value):
     """
     Transform value to None if it is NaN or Inf.
@@ -197,6 +218,7 @@ def if_nan_inf_to_none(name, value):
 
 class Counter:
     """Count accumulator with limit checking."""
+
     def __init__(self, max_count=None, init_count=0):
         self._count = init_count
         self._max_count = max_count

mindinsight/profiler/common/util.py

@@ -144,3 +144,8 @@ def get_field_value(row_info, field_name, header, time_type='realtime'):
         value = to_millisecond(value)
 
     return value
+
+def get_options(options):
+    if options is None:
+        options = {}
+    return options

mindinsight/profiler/proposer/__init__.py

+# Copyright 2019 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================

mindinsight/profiler/proposer/allproposers/__init__.py

+# Copyright 2019 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+"""All proposers."""
+from mindinsight.profiler.proposer.allproposers.common_proposer import CommonProposer
+from mindinsight.profiler.proposer.allproposers.step_trace_proposer import StepTraceProposer
+
+
+__all__ = ["CommonProposer", "StepTraceProposer"]

mindinsight/profiler/proposer/allproposers/base_proposer.py

+# Copyright 2020 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+"""The proposer base class."""
+from abc import ABC, abstractmethod
+
+from mindinsight.profiler.common.log import logger
+from mindinsight.profiler.analyser.analyser_factory import AnalyserFactory
+from mindinsight.utils.exceptions import MindInsightException
+
+
+class Proposer(ABC):
+    """The proposer base class."""
+
+    def __init__(self, profiling_path, device_id):
+        self.profiling_path = profiling_path
+        self.device_id = device_id
+
+    def get_analyser_result(self, analyser_type, condition=None):
+        logger.debug("The Proposer 'analyser_type' is %s, 'options' is %s", str(analyser_type), str(condition))
+        analyser_result = {}
+        try:
+            analyser = AnalyserFactory.instance().get_analyser(analyser_type, self.profiling_path, self.device_id)
+            analyser_result = analyser.query(condition)
+            logger.debug("The 'analyser_result' is %s, the 'condition' is %s.", str(analyser_result), str(condition))
+        except MindInsightException as e:
+            logger.warning(e)
+        return analyser_result
+
+    @abstractmethod
+    def analyze(self, options=None):
+        """analysis and get proposal."""
+        raise NotImplementedError("Must define analyze function to inherit Class Propose")

mindinsight/profiler/proposer/allproposers/common_proposer.py

+# Copyright 2020 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+"""The common proposer."""
+from collections import OrderedDict
+
+from mindinsight.profiler.common.util import get_options
+from mindinsight.profiler.proposer.allproposers.base_proposer import Proposer
+from mindinsight.profiler.common.log import logger
+
+
+class CommonProposer(Proposer):
+    """The common proposer."""
+    def __init__(self, profiling_dir, device_id):
+        super().__init__(profiling_dir, device_id)
+        self.__proposal_dict = OrderedDict()
+        self.__proposal_dict["common-profiler_tutorial"] = None
+
+    def analyze(self, options=None):
+        """
+        Get the proposal from proposer.
+
+        Args:
+            options: options for proposer analysis
+
+        Returns:
+            dict, the proposal from proposer instance.
+        """
+        logger.info('The CommonProposer　is running')
+        options = get_options(options)
+        return self.__proposal_dict

mindinsight/profiler/proposer/allproposers/step_trace_proposer.py

+# Copyright 2020 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+"""The step trace proposer."""
+from collections import OrderedDict
+
+from mindinsight.profiler.common.util import get_options
+from mindinsight.profiler.proposer.allproposers.base_proposer import Proposer
+from mindinsight.profiler.common.log import logger
+
+
+class StepTraceProposer(Proposer):
+    """The step trace proposer."""
+
+    def __init__(self, profiling_dir, device_id):
+        super().__init__(profiling_dir, device_id)
+        self.__step_trace_iter_interval_threshold = 0.5
+        self.__proposer_type = "step_trace"
+        self.__proposal_dict = OrderedDict()
+        self.__iter_interval_label = "step_trace-iter_interval"
+
+    def analyze(self, options=None):
+        """
+        Get the proposal from proposer.
+
+        Args:
+            options (dict): options for proposer analysis.
+                - step_trace: include optional parameters for step trace，The dictionary key is iter_interval
+                  used to get the analyser options for iteration interval time.
+
+        Returns:
+            dict, the proposal from proposer instance，the dictionary key is a language internationalization
+            label, and the value is used to format the value in the language internationalization string.
+
+        Examples:
+            >>> proposer_type = 'step_trace'
+            >>> proposer = ProposerFactory.instance().get_proposer(proposer_type, self.profiling_dir, self.device_id)
+            >>> result = proposer.analyze(options)
+        """
+        logger.info("The StepTraceProposer is running")
+
+        options = get_options(options)
+        logger.debug("The StepTraceProposer 'options' is %s", str(options))
+        step_trace_condition = options.get("step_trace", {})
+        # Get the proposals of iteration interval.
+        self._iter_interval_analyze(step_trace_condition)
+        return self.__proposal_dict
+
+    def _iter_interval_analyze(self, step_trace_condition):
+        """Get the proposals of iteration interval."""
+        iter_interval_dict = OrderedDict()
+        default_iter_interval_lst = [0]
+        iter_interval_condition = step_trace_condition.get("iter_interval", {})
+        analyser_result = self.get_analyser_result(self.__proposer_type, condition=iter_interval_condition)
+        iter_interval_length_lst = analyser_result.get("info", {}).get("iteration_interval",
+                                                                       default_iter_interval_lst)
+        logger.debug("The 'iter_interval_length_lst' is %s", str(iter_interval_length_lst))
+        # Check the iter_interval_length_lst.
+        if not isinstance(iter_interval_length_lst, list) or not iter_interval_length_lst:
+            logger.warning("The 'iter_interval_length_lst' is %s,　it is null or not a list",
+                           str(iter_interval_length_lst))
+        else:
+            if iter_interval_length_lst[0] > self.__step_trace_iter_interval_threshold:
+                iter_interval_dict[self.__iter_interval_label] = [str(self.__step_trace_iter_interval_threshold)]
+                self.__proposal_dict.update(iter_interval_dict)

mindinsight/profiler/proposer/compose_proposer.py

+# Copyright 2020 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+"""The Compose Proposals."""
+from collections import OrderedDict
+
+from mindinsight.profiler.common.log import logger
+from mindinsight.profiler.common.util import get_options
+from mindinsight.profiler.proposer.proposer_factory import ProposerFactory
+
+
+class ComposeProposal:
+    """Get the proposals from multiple different proposers."""
+    def __init__(self, profiling_path, device_id, type_list=None):
+        self.profiling_path = profiling_path
+        self.device_id = device_id
+        self.compose_proposer_type_list = type_list
+        # Postfix of category label, used for UI to identify the label as category label.
+        self.type_label_postfix = "-proposer_type_label"
+
+    def get_proposal(self, options=None):
+        """
+        Get compose proposals.
+
+        Args:
+            options (dict): options for composed proposal.
+                - compose_proposal_result: execution results of the already running proposers.
+                - step_trace: include optional parameters for step trace，The dictionary key is iter_interval
+                  used to get the analyser options for iteration interval time.
+
+        Returns:
+            dict, the proposals from multiple different proposers.
+
+        Examples:
+            >>> type_list = ['common', 'step_trace']
+            >>> condition = {"filter_condition": {'mode': "proc", "proc_name": "iteration_interval"}}
+            >>> options = {'step_trace': {"iter_interval": condition}}
+            >>> cp = ComposeProposal(self.profiling_dir, self.device_id, type_list)
+            >>> result_proposal = cp.get_proposal(options=options)
+        """
+        logger.info("The ComposeProposal is running")
+        options = get_options(options)
+        logger.debug("The 'options'　is %s", str(options))
+        # The flag whether to write category label.
+        type_label_flag = options.get("type_label_flag", True)
+        compose_proposal_result = OrderedDict()
+        logger.debug("The 'compose_proposer_type_list'　is %s", str(self.compose_proposer_type_list))
+        for proposer_type in self.compose_proposer_type_list:
+            proposer = ProposerFactory.instance().get_proposer(proposer_type, self.profiling_path, self.device_id)
+
+            if proposer is None:
+                continue
+
+            # Write the result of proposals to option for other proposer to get.
+            options["compose_proposal_result"] = compose_proposal_result
+
+            result = proposer.analyze(options)
+            # Insert category label.
+            if result and type_label_flag:
+                proposer_type_label = proposer_type + "-type_label"
+                # Get the name of the category label, the default is the same as the proposer type.
+                type_label_name = options.get(proposer_type_label, proposer_type)
+                # Add postfix to category label name
+                type_proposal_label = type_label_name + self.type_label_postfix
+                compose_proposal_result[type_proposal_label] = None
+                # Merge results to the proposals dictionary.
+                compose_proposal_result.update(result)
+            elif result and not type_label_flag:
+                # Merge results to the proposals dictionary.
+                compose_proposal_result.update(result)
+
+        logger.debug("The 'compose_proposal_result' is %s", str(compose_proposal_result))
+        return compose_proposal_result

mindinsight/profiler/proposer/proposer_factory.py

+# Copyright 2020 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+"""The proposer factory."""
+import threading
+
+import mindinsight.profiler.proposer.allproposers as proposer_module
+from mindinsight.profiler.common.log import logger
+
+
+class ProposerFactory:
+    """The Proposer factory is used to create Proposer special instance."""
+    _lock = threading.Lock()
+    _instance = None
+
+    def __new__(cls, *args, **kwargs):
+        if cls._instance is None:
+            with cls._lock:
+                if cls._instance is None:
+                    cls._instance = super().__new__(cls, *args, **kwargs)
+        return cls._instance
+
+    @classmethod
+    def instance(cls):
+        """The factory instance."""
+        if cls._instance is None:
+            cls._instance = cls()
+        return cls._instance
+
+    def get_proposer(self, proposer_type, *args):
+        """
+        Get the specified proposer according to the proposer type.
+
+        Args:
+            proposer_type (str): The proposer type.
+            args (list): The parameters required for the specific proposer class.
+
+        Returns:
+            Proposer, the specified proposer instance.
+
+        Examples:
+            >>> proposer_type = 'step_trace'
+            >>> proposer = ProposerFactory.instance().get_proposer(proposer_type, self.profiling_dir, self.device_id)
+
+        """
+        logger.debug("The 'proposer_type' is %s,The 'args' is %s", proposer_type, str(args))
+        proposer_instance = None
+        sub_name = proposer_type.split('_')
+        proposer_class_name = ''.join([name.capitalize() for name in sub_name])
+        proposer_class_name += 'Proposer'
+
+        if hasattr(proposer_module, proposer_class_name):
+            proposer_instance = getattr(proposer_module, proposer_class_name)(*args)
+        else:
+            logger.warning("The proposer class %s does not exist.", proposer_class_name)
+        return proposer_instance

tests/ut/profiler/proposer/__init__.py

+# Copyright 2020 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================

tests/ut/profiler/proposer/test_proposer.py

+# Copyright 2020 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+"""Test the proposer module."""
+import os
+from unittest import TestCase
+from collections import OrderedDict
+
+from mindinsight.profiler.proposer.proposer_factory import ProposerFactory
+from mindinsight.profiler.proposer.compose_proposer import ComposeProposal
+
+
+class TestPropose(TestCase):
+    """Test the class of Proposer."""
+
+    def setUp(self) -> None:
+        """Initialization before test case execution."""
+        self.profiling_dir = os.path.realpath(os.path.join(os.path.dirname(__file__),
+                                                           '../../../utils/resource/profiler'))
+        self.device_id = 0
+        self.common_proposal_dict = OrderedDict()
+        self.common_proposal_dict["common-profiler_tutorial"] = None
+        self.step_trace_proposal_dict = OrderedDict()
+        self.step_trace_proposal_dict["step_trace-iter_interval"] = ['0.5']
+
+    def test_propose_compose(self):
+        """Test the class of ComposeProposal."""
+        proposal_dict = OrderedDict()
+        proposal_dict["step_trace-proposer_type_label"] = None
+        proposal_dict.update(self.step_trace_proposal_dict)
+        proposal_dict["common-proposer_type_label"] = None
+        proposal_dict.update(self.common_proposal_dict)
+        type_list = ['step_trace', 'common']
+        condition = {"filter_condition": {'mode': "proc",
+                                          "proc_name": "iteration_interval",
+                                          "step_id": 0}}
+        options = {'step_trace': {"iter_interval": condition}}
+        cp = ComposeProposal(self.profiling_dir, self.device_id, type_list)
+        result = cp.get_proposal(options=options)
+        self.assertDictEqual(proposal_dict, result)
+
+    def test_propose_compose_exception(self):
+        """Test the class of ComposeProposal."""
+        profiling_dir = os.path.realpath(os.path.join(os.path.dirname(__file__),
+                                                      '../../../utils/resource/test'))
+        proposal_dict = OrderedDict()
+        proposal_dict["common-proposer_type_label"] = None
+        proposal_dict.update(self.common_proposal_dict)
+        type_list = ['step_trace', 'common']
+        condition = {"filter_condition": {'mode': "proc",
+                                          "proc_name": "iteration_interval",
+                                          "step_id": 0}}
+        options = {'step_trace': {"iter_interval": condition}}
+        cp = ComposeProposal(profiling_dir, self.device_id, type_list)
+        result = cp.get_proposal(options=options)
+        self.assertDictEqual(proposal_dict, result)
+
+    def test_propose_compose_type_label(self):
+        """Test the class of ComposeProposal."""
+        proposal_dict = OrderedDict()
+        proposal_dict["test_label-proposer_type_label"] = None
+        proposal_dict.update(self.step_trace_proposal_dict)
+        proposal_dict["common-proposer_type_label"] = None
+        proposal_dict.update(self.common_proposal_dict)
+        type_list = ['step_trace', 'common']
+        condition = {"filter_condition": {'mode': "proc",
+                                          "proc_name": "iteration_interval",
+                                          "step_id": 0}}
+        options = {'step_trace': {"iter_interval": condition}, "step_trace-type_label": "test_label"}
+        cp = ComposeProposal(self.profiling_dir, self.device_id, type_list)
+        result = cp.get_proposal(options=options)
+        self.assertDictEqual(proposal_dict, result)
+
+    def test_propose_compose_type_label_flag(self):
+        """Test the class of ComposeProposal."""
+        proposal_dict = OrderedDict()
+        proposal_dict.update(self.step_trace_proposal_dict)
+        proposal_dict.update(self.common_proposal_dict)
+
+        type_list = ['step_trace', 'common']
+        condition = {"filter_condition": {'mode': "proc",
+                                          "proc_name": "iteration_interval",
+                                          "step_id": 0}}
+        options = {'step_trace': {"iter_interval": condition},
+                   "step_trace-type_label": "test_label",
+                   "type_label_flag": False}
+        cp = ComposeProposal(self.profiling_dir, self.device_id, type_list)
+        result = cp.get_proposal(options=options)
+        self.assertDictEqual(proposal_dict, result)
+
+    def test_propose_common(self):
+        proposer_type = 'common'
+        proposer = ProposerFactory.instance().get_proposer(proposer_type, self.profiling_dir, self.device_id)
+        result = proposer.analyze()
+        self.assertDictEqual(self.common_proposal_dict, result)
+
+    def test_propose_step_trace(self):
+        """Test the class of step trace."""
+        proposer_type = 'step_trace'
+        proposer = ProposerFactory.instance().get_proposer(proposer_type, self.profiling_dir, self.device_id)
+        condition = {"filter_condition": {'mode': "proc",
+                                          "proc_name": "iteration_interval",
+                                          "step_id": 0}}
+        options = {'step_trace': {"iter_interval": condition}}
+        result = proposer.analyze(options)
+        self.assertDictEqual(self.step_trace_proposal_dict, result)