From c91bb084f4e2157dde1fe22ec7741fcf4551cf94 Mon Sep 17 00:00:00 2001 From: LoveAn Date: Tue, 24 Nov 2020 21:17:52 +0800 Subject: [PATCH] Add op benchmark ci pipeline in Paddle repo (#28692) --- paddle/scripts/paddle_build.sh | 7 ++ tools/check_op_benchmark_result.py | 120 ++++++++++++++++++ tools/test_op_benchmark.sh | 187 +++++++++++++++++++++++++++++ 3 files changed, 314 insertions(+) create mode 100644 tools/check_op_benchmark_result.py create mode 100644 tools/test_op_benchmark.sh diff --git a/paddle/scripts/paddle_build.sh b/paddle/scripts/paddle_build.sh index 43faccfff2b..6771228b64a 100755 --- a/paddle/scripts/paddle_build.sh +++ b/paddle/scripts/paddle_build.sh @@ -1609,6 +1609,10 @@ function example() { fi } +function test_op_benchmark() { + bash ${PADDLE_ROOT}/tools/test_op_benchmark.sh +} + function summary_check_problems() { set +x local check_style_code=$1 @@ -1784,6 +1788,9 @@ function main() { api_example) example ;; + test_op_benchmark) + test_op_benchmark + ;; *) print_usage exit 1 diff --git a/tools/check_op_benchmark_result.py b/tools/check_op_benchmark_result.py new file mode 100644 index 00000000000..79ab9a28476 --- /dev/null +++ b/tools/check_op_benchmark_result.py @@ -0,0 +1,120 @@ +# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import json +import logging +import argparse + + +def check_path_exists(path): + """Assert whether file/directory exists. + """ + assert os.path.exists(path), "%s does not exist." % path + + +def parse_log_file(log_file): + """Load one case result from log file. + """ + check_path_exists(log_file) + + result = None + with open(log_file) as f: + for line in f.read().strip().split('\n')[::-1]: + try: + result = json.loads(line) + return result + except ValueError: + pass # do nothing + + assert result != None, "Parse log file fail!" + + +def load_benchmark_result_from_logs_dir(logs_dir): + """Load benchmark result from logs directory. + """ + check_path_exists(logs_dir) + + log_file_path = lambda log_file: os.path.join(logs_dir, log_file) + result_lambda = lambda log_file: (log_file, parse_log_file(log_file_path(log_file))) + + return dict(map(result_lambda, os.listdir(logs_dir))) + + +def compare_benchmark_result(develop_result, pr_result): + """Compare the differences between devlop and pr. + """ + develop_speed = develop_result.get("speed") + pr_speed = pr_result.get("speed") + + assert type(develop_speed) == type( + pr_speed), "The types of comparison results need to be consistent." + + if isinstance(develop_speed, dict) and isinstance(pr_speed, dict): + pr_gpu_time = pr_speed.get("gpu_time") + develop_gpu_time = develop_speed.get("gpu_time") + gpu_time_diff = (pr_gpu_time - develop_gpu_time) / develop_gpu_time + + pr_total_time = pr_speed.get("total") + develop_total_time = develop_speed.get("total") + total_time_diff = ( + pr_total_time - develop_total_time) / develop_total_time + + # TODO(Avin0323): Print all info for making relu of alart. + logging.info("------ OP: %s ------" % pr_result.get("name")) + logging.info("GPU time change: %.5f%% (develop: %.7f -> PR: %.7f)" % + (gpu_time_diff * 100, develop_gpu_time, pr_gpu_time)) + logging.info("Total time change: %.5f%% (develop: %.7f -> PR: %.7f)" % + (total_time_diff * 100, develop_total_time, pr_total_time)) + logging.info("backward: %s" % pr_result.get("backward")) + logging.info("parameters:") + for line in pr_result.get("parameters").strip().split("\n"): + logging.info("\t%s" % line) + else: + # TODO(Avin0323): Accuracy need to add. + pass + + return True + + +if __name__ == "__main__": + """Load result from log directories and compare the differences. + """ + logging.basicConfig( + level=logging.INFO, + format="[%(pathname)s:%(lineno)d] [%(levelname)s] %(message)s") + + parser = argparse.ArgumentParser() + parser.add_argument( + "--develop_logs_dir", + type=str, + required=True, + help="Specify the benchmark result directory of develop branch.") + parser.add_argument( + "--pr_logs_dir", + type=str, + required=True, + help="Specify the benchmark result directory of PR branch.") + args = parser.parse_args() + + develop_result_dict = load_benchmark_result_from_logs_dir( + args.develop_logs_dir) + + check_path_exists(args.pr_logs_dir) + for log_file in os.listdir(args.pr_logs_dir): + develop_result = develop_result_dict.get(log_file) + pr_result = parse_log_file(os.path.join(args.pr_logs_dir, log_file)) + if develop_result is None or pr_result is None: + continue + compare_benchmark_result(develop_result, pr_result) diff --git a/tools/test_op_benchmark.sh b/tools/test_op_benchmark.sh new file mode 100644 index 00000000000..955446b5c2a --- /dev/null +++ b/tools/test_op_benchmark.sh @@ -0,0 +1,187 @@ +#!/bin/bash + +# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set +ex + +[ -z "$PADDLE_ROOT" ] && PADDLE_ROOT=$(cd $(dirname ${BASH_SOURCE[0]})/.. && pwd) + +# Paddle repo file name -> op name +declare -A PADDLE_FILENAME_OP_MAP +PADDLE_FILENAME_OP_MAP=( + ["arg_min_max_op_base.h"]="arg_min arg_max" + ["arg_min_max_op_base.cu.h"]="arg_min arg_max" + ["activation_op.cu"]="leaky_relu elu sqrt square pow exp abs log" + ["activation_op.h"]="relu leaky_relu elu sqrt square pow exp abs log" + ["activation_op.cc"]="relu leaky_relu elu sqrt square pow exp abs log" +) + +# Benchmark repo name -> op name +declare -A BENCHMARK_APINAME_OP_MAP +BENCHMARK_APINAME_OP_MAP=( + ["argmin"]="arg_min" + ["argmax"]="arg_max" +) + +# ops that will run benchmark test +declare -A CHANGE_OP_MAP + +# ops that benchmark repo has +declare -A BENCHMARK_OP_MAP + +# ops that benchmark repo missing +declare -A BENCHMARK_MISS_OP_MAP + +function LOG { + echo "[$0:${BASH_LINENO[0]}] $*" >&2 +} + +# Load ops that will run benchmark test +function load_CHANGE_OP_MAP { + local op_name change_file change_file_name + for change_file in $(git diff --name-only origin/develop) + do + # match directory limit + [[ "$change_file" =~ "paddle/fluid/operators/" ]] || continue + LOG "[INFO] Found \"${change_file}\" changed." + change_file_name=${change_file#*paddle/fluid/operators/} + if [ -n "${PADDLE_FILENAME_OP_MAP[$change_file_name]}" ] + then + for op_name in ${PADDLE_FILENAME_OP_MAP[$change_file_name]} + do + LOG "[INFO] Load op: \"${op_name}\"." + CHANGE_OP_MAP[${op_name}]="dummy" + done + else + LOG "[INFO] Load op: \"${change_file_name%_op*}\"." + CHANGE_OP_MAP[${change_file_name%_op*}]="dummy" + fi + done + [ ${#CHANGE_OP_MAP[*]} -eq 0 ] && LOG "[INFO] No op to test, skip this ci." && exit 0 +} + +# Clone benchmark repo +function prepare_benchmark_environment { + LOG "[INFO] Clone benchmark repo ..." + git clone https://github.com/PaddlePaddle/benchmark.git + [ $? -ne 0 ] && LOG "[FATAL] Clone benchmark repo fail." && exit -1 + LOG "[INFO] Collect api info ..." + python benchmark/api/deploy/collect_api_info.py \ + --test_module_name tests_v2 \ + --info_file api_info.txt >& 2 + [ $? -ne 0 ] && LOG "[FATAL] Collect api info fail." && exit -1 +} + +# Load ops that will +function load_BENCHMARK_OP_MAP { + local line op_name api_name + prepare_benchmark_environment + for line in $(cat api_info.txt) + do + api_name=${line%%,*} + if [ -n "${BENCHMARK_APINAME_OP_MAP[$api_name]}" ] + then + op_name=${BENCHMARK_APINAME_OP_MAP[$api_name]} + else + op_name=$api_name + fi + if [ -n "${CHANGE_OP_MAP[$op_name]}" ] + then + LOG "[INFO] Load benchmark settings with op \"${op_name}\"." + BENCHMARK_OP_MAP[$op_name]=$line + fi + done +} + +# compile and install paddlepaddle +function compile_install_paddlepaddle { + LOG "[DEBUG] Compiling install package ..." + export WITH_GPU=ON + export WITH_AVX=ON + export WITH_MKL=ON + export RUN_TEST=OFF + export WITH_PYTHON=ON + export WITH_TESTING=OFF + export BUILD_TYPE=Release + export WITH_DISTRIBUTE=OFF + export PYTHON_ABI=cp37-cp37m + export CMAKE_BUILD_TYPE=Release + [ -d build ] && rm -rf build + bash paddle/scripts/paddle_build.sh build + [ $? -ne 0 ] && LOG "[FATAL] compile fail." && exit 7 + LOG "[DEBUG] Uninstall Paddle ..." + pip uninstall -y paddlepaddle paddlepaddle_gpu + LOG "[DEBUG] Install Paddle ..." + pip install build/python/dist/paddlepaddle_gpu-0.0.0-cp37-cp37m-linux_x86_64.whl +} + +# run op benchmark test +function run_op_benchmark_test { + local logs_dir op_name branch_name api_info_file + api_info_file="$(pwd)/api_info.txt" + [ -f "$api_info_file" ] && rm -f $api_info_file + for api_info in ${BENCHMARK_OP_MAP[*]} + do + echo "$api_info" >> $api_info_file + done + LOG "[INFO] Uninstall " + for branch_name in "develop" "test_pr" + do + git checkout $branch_name + [ $? -ne 0 ] && LOG "[FATAL] Missing branh ${branch_name}." && exit 7 + LOG "[INFO] Now branch name is ${branch_name}." + compile_install_paddlepaddle + logs_dir="$(pwd)/logs-${branch_name}" + [ -d $logs_dir ] && rm -rf $logs_dir/* || mkdir -p $logs_dir + [ -z "$VISIBLE_DEVICES" ] && export VISIBLE_DEVICES=0 + pushd benchmark/api > /dev/null + bash deploy/main_control.sh tests_v2 \ + tests_v2/configs \ + $logs_dir \ + $VISIBLE_DEVICES \ + "gpu" \ + "speed" \ + $api_info_file \ + "paddle" + popd > /dev/null + done +} + +# diff benchmakr result and miss op +function summary_problems { + local op_name + python ${PADDLE_ROOT}/tools/check_op_benchmark_result.py \ + --develop_logs_dir $(pwd)/logs-develop \ + --pr_logs_dir $(pwd)/logs-test_pr + for op_name in ${!CHANGE_OP_MAP[@]} + do + if [ -z "${BENCHMARK_OP_MAP[$op_name]}" ] + then + LOG "[WARNING] Missing test script of \"${op_name}\" in benchmark." + fi + done +} + +function main { + LOG "[INFO] Start run op benchmark test ..." + load_CHANGE_OP_MAP + load_BENCHMARK_OP_MAP + run_op_benchmark_test + summary_problems + LOG "[INFO] Op benchmark run success and no error!" + exit 0 +} + +main -- GitLab