!125 Add profiler restful api.

Merge pull request !125 from yuximiao/master

mindinsight/backend/profiler/__init__.py

+# Copyright 2020 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+"""
+module init file.
+"""
+from mindinsight.backend.profiler.profile_api import init_module as init_profiler_module
+
+
+def init_module(app):
+    """
+    Init module entry.
+
+    Args:
+        app: Flask. A Flask instance.
+
+    Returns:
+
+    """
+    init_profiler_module(app)

mindinsight/backend/profiler/profile_api.py

+# Copyright 2020 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+"""
+Profile api.
+
+This module provides the interfaces to profile functions.
+"""
+import json
+import os
+
+from flask import Blueprint
+from flask import request
+from flask import jsonify
+from marshmallow import ValidationError
+
+from mindinsight.conf import settings
+from mindinsight.datavisual.utils.tools import get_train_id, get_profiler_dir
+from mindinsight.profiler.analyser.analyser_factory import AnalyserFactory
+
+from mindinsight.lineagemgr.common.validator.validate_path import validate_and_normalize_path
+from mindinsight.profiler.common.util import analyse_device_list_from_profiler_dir
+from mindinsight.profiler.common.validator.validate import validate_condition
+from mindinsight.utils.exceptions import ParamValueError
+
+BLUEPRINT = Blueprint("profile", __name__, url_prefix=settings.URL_PREFIX)
+
+
+@BLUEPRINT.route("/profile/ops/search", methods=["POST"])
+def get_profile_op_info():
+    """
+    Get operation profiling info.
+
+    Returns:
+        str, the operation profiling information.
+
+    Raises:
+        ParamValueError: If the search condition contains some errors.
+
+    Examples:
+        >>> POST http://xxxx/v1/mindinsight/profile/op
+    """
+    profiler_dir = get_profiler_dir(request)
+    train_id = get_train_id(request)
+
+    search_condition = request.stream.read()
+    try:
+        search_condition = json.loads(search_condition if search_condition else "{}")
+    except Exception:
+        raise ParamValueError("Json data parse failed.")
+    validate_condition(search_condition)
+
+    device_id = search_condition.get("device_id", "0")
+    profiler_dir_abs = os.path.join(settings.SUMMARY_BASE_DIR, train_id, profiler_dir)
+    try:
+        profiler_dir_abs = validate_and_normalize_path(profiler_dir_abs, "profiler")
+    except ValidationError:
+        raise ParamValueError("Invalid profiler dir")
+
+    op_type = search_condition.get("op_type")
+
+    analyser = AnalyserFactory.instance().get_analyser(
+        op_type, profiler_dir_abs, device_id
+    )
+
+    op_info = analyser.query(search_condition)
+    return jsonify(op_info)
+
+
+@BLUEPRINT.route("/profile/devices", methods=["GET"])
+def get_profile_device_list():
+    """
+    Get profile device list.
+
+    Returns:
+        list, the available device list.
+
+    Raises:
+        ParamValueError: If the search condition contains some errors.
+
+    Examples:
+        >>> POST http://xxxx/v1/mindinsight/profile/device_list
+    """
+    profiler_dir = get_profiler_dir(request)
+    train_id = get_train_id(request)
+    profiler_dir_abs = os.path.join(settings.SUMMARY_BASE_DIR, train_id, profiler_dir)
+    try:
+        profiler_dir_abs = validate_and_normalize_path(profiler_dir_abs, "profiler")
+    except ValidationError:
+        raise ParamValueError("Invalid profiler dir")
+
+    device_list = analyse_device_list_from_profiler_dir(profiler_dir_abs)
+    return jsonify(device_list)
+
+
+def init_module(app):
+    """
+    Init module entry.
+
+    Args:
+        app: the application obj.
+
+    """
+    app.register_blueprint(BLUEPRINT)

mindinsight/datavisual/utils/tools.py

@@ -137,6 +137,25 @@ def get_train_id(request):
     return train_id
 
 
+def get_profiler_dir(request):
+    """
+    Get train ID from requst query string and unquote content.
+
+    Args:
+        request (FlaskRequest): Http request instance.
+
+    Returns:
+        str, unquoted train ID.
+    """
+    profiler_dir = request.args.get('profile')
+    if profiler_dir is not None:
+        try:
+            profiler_dir = unquote(profiler_dir, errors='strict')
+        except UnicodeDecodeError:
+            raise exceptions.UrlDecodeError('Unquote profiler_dir error with strict mode')
+    return profiler_dir
+
+
 def if_nan_inf_to_none(name, value):
     """
     Transform value to None if it is NaN or Inf.

mindinsight/profiler/common/exceptions/error_code.py

@@ -41,6 +41,11 @@ class ProfilerErrors(ProfilerMgrErrors):
     # analyser error code
     COLUMN_NOT_EXIST_ERROR = 0 | _ANALYSER_MASK
     ANALYSER_NOT_EXIST_ERROR = 1 | _ANALYSER_MASK
+    DEVICE_ID_ERROR = 2 | _ANALYSER_MASK
+    OP_TYPE_ERROR = 3 | _ANALYSER_MASK
+    GROUP_CONDITION_ERROR = 4 | _ANALYSER_MASK
+    SORT_CONDITION_ERROR = 5 | _ANALYSER_MASK
+    FILTER_CONDITION_ERROR = 6 | _ANALYSER_MASK
 
 
 @unique
@@ -61,3 +66,8 @@ class ProfilerErrorMsg(Enum):
     # analyser error msg
     COLUMN_NOT_EXIST_ERROR = 'The column {} does not exist.'
     ANALYSER_NOT_EXIST_ERROR = 'The analyser {} does not exist.'
+    DEIVICE_ID_ERROR = 'The device_id in search_condition error, {}'
+    FILTER_CONDITION_ERROR = 'The filter_condition in search_condition error, {}'
+    OP_TYPE_ERROR = 'The op_type in search_condition error, {}'
+    GROUP_CONDITION_ERROR = 'The group_condition in search_condition error, {}'
+    SORT_CONDITION_ERROR = 'The sort_condition in search_condition error, {}'

mindinsight/profiler/common/exceptions/exceptions.py

@@ -126,3 +126,58 @@ class ProfilerAnalyserNotExistException(MindInsightException):
             message=ProfilerErrorMsg.ANALYSER_NOT_EXIST_ERROR.value.format(msg),
             http_code=400
         )
+
+
+class ProfilerDeviceIdException(MindInsightException):
+    """The parameter device_id error in profiler module."""
+
+    def __init__(self, msg):
+        super(ProfilerDeviceIdException, self).__init__(
+            error=ProfilerErrors.DEVICE_ID_ERROR,
+            message=ProfilerErrorMsg.DEIVICE_ID_ERROR.value.format(msg),
+            http_code=400
+        )
+
+
+class ProfilerOpTypeException(MindInsightException):
+    """The parameter op_type error in profiler module."""
+
+    def __init__(self, msg):
+        super(ProfilerOpTypeException, self).__init__(
+            error=ProfilerErrors.OP_TYPE_ERROR,
+            message=ProfilerErrorMsg.OP_TYPE_ERROR.value.format(msg),
+            http_code=400
+        )
+
+
+class ProfilerSortConditionException(MindInsightException):
+    """The parameter sort_condition error in profiler module."""
+
+    def __init__(self, msg):
+        super(ProfilerSortConditionException, self).__init__(
+            error=ProfilerErrors.SORT_CONDITION_ERROR,
+            message=ProfilerErrorMsg.SORT_CONDITION_ERROR.value.format(msg),
+            http_code=400
+        )
+
+
+class ProfilerFilterConditionException(MindInsightException):
+    """The parameter filer_condition error in profiler module."""
+
+    def __init__(self, msg):
+        super(ProfilerFilterConditionException, self).__init__(
+            error=ProfilerErrors.FILTER_CONDITION_ERROR,
+            message=ProfilerErrorMsg.FILTER_CONDITION_ERROR.value.format(msg),
+            http_code=400
+        )
+
+
+class ProfilerGroupConditionException(MindInsightException):
+    """The parameter group_condition error in profiler module."""
+
+    def __init__(self, msg):
+        super(ProfilerGroupConditionException, self).__init__(
+            error=ProfilerErrors.GROUP_CONDITION_ERROR,
+            message=ProfilerErrorMsg.GROUP_CONDITION_ERROR.value.format(msg),
+            http_code=400
+        )

mindinsight/profiler/common/util.py

+# Copyright 2020 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+"""
+Profiler util.
+
+This module provides the utils.
+"""
+import os
+
+
+def analyse_device_list_from_profiler_dir(profiler_dir):
+    """
+    Analyse device list from profiler dir.
+
+    Args:
+        profiler_dir (str): The profiler data dir.
+
+    Returns:
+        list, the device_id list.
+    """
+    device_id_list = set()
+    for _, _, filenames in os.walk(profiler_dir):
+        for filename in filenames:
+            profiler_file_prefix = ["output_op_compute_time", "output_data_preprocess_aicpu"]
+            items = filename.split("_")
+            device_num = items[-1].split(".")[0] if items[-1].split(".") else ""
+            if device_num.isdigit() and '_'.join(items[:-1]) in profiler_file_prefix:
+                device_id_list.add(device_num)
+
+    return list(device_id_list)

mindinsight/profiler/common/validator/validate.py

+# Copyright 2020 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+"""Validate the profiler parameters."""
+from mindinsight.profiler.common.exceptions.exceptions import ProfilerParamTypeErrorException, \
+    ProfilerParamValueErrorException, ProfilerDeviceIdException, ProfilerOpTypeException, \
+    ProfilerSortConditionException, ProfilerFilterConditionException, ProfilerGroupConditionException
+from mindinsight.profiler.common.log import logger as log
+
+AICORE_TYPE_COL = ["op_type", "execution_time", "execution_frequency", "precent"]
+AICORE_DETAIL_COL = ["op_name", "op_type", "execution_time", "subgraph", "full_op_name"]
+AICPU_COL = ["serial_number", "op_name", "total_time", "dispatch_time", "RunV2_start",
+             "compute_start", "memcpy_start", "memcpy_end", "RunV2_end"]
+
+
+def validate_condition(search_condition):
+    """
+    Verify the param in search_condition is valid or not.
+
+    Args:
+        search_condition (dict): The search condition.
+
+    Raises:
+        ProfilerParamTypeErrorException: If the type of the param in search_condition is invalid.
+        ProfilerDeviceIdException: If the device_id param in search_condition is invalid.
+        ProfilerOpTypeException: If the op_type param in search_condition is invalid.
+        ProfilerGroupConditionException: If the group_condition param in search_condition is invalid.
+        ProfilerSortConditionException: If the sort_condition param in search_condition is invalid.
+        ProfilerFilterConditionException: If the filter_condition param in search_condition is invalid.
+    """
+    if not isinstance(search_condition, dict):
+        log.error("Invalid search_condition type, it should be dict.")
+        raise ProfilerParamTypeErrorException(
+            "Invalid search_condition type, it should be dict.")
+
+    if "device_id" in search_condition:
+        device_id = search_condition.get("device_id")
+        if not isinstance(device_id, str):
+            raise ProfilerDeviceIdException("Invalid device_id type, it should be str.")
+
+    if "op_type" in search_condition:
+        op_type = search_condition.get("op_type")
+        if op_type == "aicpu":
+            search_scope = AICPU_COL
+        elif op_type == "aicore_type":
+            search_scope = AICORE_TYPE_COL
+        elif op_type == "aicore_detail":
+            search_scope = AICORE_DETAIL_COL
+        else:
+            raise ProfilerOpTypeException("The op_type must in ['aicpu', 'aicore_type', 'aicore_detail']")
+    else:
+        raise ProfilerOpTypeException("The op_type must in ['aicpu', 'aicore_type', 'aicore_detail']")
+
+    if "group_condition" in search_condition:
+        group_condition = search_condition.get("group_condition")
+        if not isinstance(group_condition, dict):
+            raise ProfilerGroupConditionException("The group condition must be dict.")
+        if "limit" in group_condition:
+            limit = group_condition.get("limit", 0)
+            if isinstance(limit, bool) \
+                    or not isinstance(group_condition.get("limit"), int):
+                log.error("The limit must be int.")
+                raise ProfilerGroupConditionException("The limit must be int.")
+            if limit < 1 or limit > 100:
+                raise ProfilerGroupConditionException("The limit must in [1, 100].")
+
+        if "offset" in group_condition:
+            offset = group_condition.get("offset", 0)
+            if isinstance(offset, bool) \
+                    or not isinstance(group_condition.get("offset"), int):
+                log.error("The offset must be int.")
+                raise ProfilerGroupConditionException("The offset must be int.")
+            if offset < 0:
+                raise ProfilerGroupConditionException("The offset must ge 0.")
+
+            if offset > 1000000:
+                raise ProfilerGroupConditionException("The offset must le 1000000.")
+
+    if "sort_condition" in search_condition:
+        sort_condition = search_condition.get("sort_condition")
+        if not isinstance(sort_condition, dict):
+            raise ProfilerSortConditionException("The sort condition must be dict.")
+        if "name" in sort_condition:
+            sorted_name = sort_condition.get("name", "")
+            err_msg = "The sorted_name must be in {}".format(search_scope)
+            if not isinstance(sorted_name, str):
+                log.error("Wrong sorted name type.")
+                raise ProfilerSortConditionException("Wrong sorted name type.")
+            if sorted_name not in search_scope:
+                log.error(err_msg)
+                raise ProfilerSortConditionException(err_msg)
+
+        if "type" in sort_condition:
+            sorted_type_param = ['ascending', 'descending']
+            sorted_type = sort_condition.get("type")
+            if sorted_type not in sorted_type_param:
+                err_msg = "The sorted type must be ascending or descending."
+                log.error(err_msg)
+                raise ProfilerParamValueErrorException(err_msg)
+
+    if "filter_condition" in search_condition:
+        def validate_op_filter_condition(op_condition):
+            if not isinstance(op_condition, dict):
+                raise ProfilerFilterConditionException("Wrong op_type filter condition.")
+            for key, value in op_condition.items():
+                if not isinstance(key, str):
+                    raise ProfilerFilterConditionException("The filter key must be str")
+                if not isinstance(value, list):
+                    raise ProfilerFilterConditionException("The filter value must be list")
+                if key not in filter_key:
+                    raise ProfilerFilterConditionException("The filter key must in {}.".format(filter_key))
+                for item in value:
+                    if not isinstance(item, str):
+                        raise ProfilerFilterConditionException("The item in filter value must be str")
+
+        filter_condition = search_condition.get("filter_condition")
+        if not isinstance(filter_condition, dict):
+            raise ProfilerFilterConditionException("The filter condition must be dict.")
+        filter_key = ["in", "not_in", "partial_match_str_in"]
+        if filter_condition:
+            if "op_type" in filter_condition:
+                op_type_condition = filter_condition.get("op_type")
+                validate_op_filter_condition(op_type_condition)
+            if "op_name" in filter_condition:
+                op_name_condition = filter_condition.get("op_name")
+                validate_op_filter_condition(op_name_condition)
+            if "op_type" not in filter_condition and "op_name" not in filter_condition:
+                raise ProfilerFilterConditionException("The key of filter_condition is not support")