From 40f9dbd26e6cda139795307c35e194270358d063 Mon Sep 17 00:00:00 2001 From: LoveAn Date: Thu, 3 Dec 2020 11:22:37 +0800 Subject: [PATCH] Optimize OP match rule of op benchmark ci (#29263) --- tools/check_op_benchmark_result.py | 5 +- tools/test_op_benchmark.sh | 122 ++++++++++++++++------------- 2 files changed, 72 insertions(+), 55 deletions(-) diff --git a/tools/check_op_benchmark_result.py b/tools/check_op_benchmark_result.py index 2fa85f0a74c..6eb2c9ecb57 100644 --- a/tools/check_op_benchmark_result.py +++ b/tools/check_op_benchmark_result.py @@ -38,7 +38,10 @@ def parse_log_file(log_file): except ValueError: pass # do nothing - assert result != None, "Parse log file fail!" + if result is None: + logging.warning("Parse %s fail!" % log_file) + + return result def load_benchmark_result_from_logs_dir(logs_dir): diff --git a/tools/test_op_benchmark.sh b/tools/test_op_benchmark.sh index 01e7895a01f..8d7d7bb777f 100644 --- a/tools/test_op_benchmark.sh +++ b/tools/test_op_benchmark.sh @@ -18,33 +18,8 @@ set +ex [ -z "$PADDLE_ROOT" ] && PADDLE_ROOT=$(cd $(dirname ${BASH_SOURCE[0]})/.. && pwd) -# Paddle repo file name -> op name -declare -A PADDLE_FILENAME_OP_MAP -PADDLE_FILENAME_OP_MAP=( - ["arg_min_max_op_base.h"]="arg_min arg_max" - ["arg_min_max_op_base.cu.h"]="arg_min arg_max" - ["activation_op.cu"]="leaky_relu elu sqrt square pow exp abs log" - ["activation_op.h"]="relu leaky_relu elu sqrt square pow exp abs log" - ["activation_op.cc"]="relu leaky_relu elu sqrt square pow exp abs log" - ["interpolate_op.h"]="bilinear_interp nearest_interp trilinear_interp bicubic_interp linear_interp" - ["interpolate_op.cc"]="bilinear_interp nearest_interp trilinear_interp bicubic_interp linear_interp" - ["interpolate_op.cu"]="bilinear_interp nearest_interp trilinear_interp bicubic_interp linear_interp" -) - -# Benchmark repo name -> op name -declare -A BENCHMARK_APINAME_OP_MAP -BENCHMARK_APINAME_OP_MAP=( - ["argmin"]="arg_min" - ["argmax"]="arg_max" - ["cos_sim"]="cosine_similarity" - ["elementwise_max"]="maximum" - ["elementwise_min"]="minimum" - ["bilinear_interp"]="interp_bilinear" - ["nearest_interp"]="interp_nearest" - ["trilinear_interp"]="interp_trilinear" - ["bicubic_interp"]="interp_bicubic" - ["linear_interp"]="interp_linear" -) +# PR modify op source files +CHANGE_OP_FILES=() # ops that will run benchmark test declare -A CHANGE_OP_MAP @@ -52,38 +27,46 @@ declare -A CHANGE_OP_MAP # ops that benchmark repo has declare -A BENCHMARK_OP_MAP -# ops that benchmark repo missing -declare -A BENCHMARK_MISS_OP_MAP - function LOG { echo "[$0:${BASH_LINENO[0]}] $*" >&2 } -# Load ops that will run benchmark test -function load_CHANGE_OP_MAP { - local op_name change_file change_file_name +# Load op files by header file +function load_CHANGE_OP_FILES_by_header_file { + local change_file + for change_file in $(grep -rl "${1}" paddle/fluid/operators) + do + if [[ "$change_file" =~ "_op.cu" ]] + then + LOG "[INFO] Found \"${1}\" include by \"${change_file}\"." + CHANGE_OP_FILES[${#CHANGE_OP_FILES[@]}]="$change_file" + elif [[ "$change_file" =~ ".h" ]] + then + LOG "[INFO] Found \"${1}\" include by \"${change_file}\", keep searching." + load_CHANGE_OP_FILES_by_header_file $change_file + fi + done +} + +# Load op files that PR changes +function load_CHANGE_OP_FILES { + local change_file for change_file in $(git diff --name-only origin/develop) do # match directory limit [[ "$change_file" =~ "paddle/fluid/operators/" ]] || continue # match file name limit - [[ "$change_file" =~ "_op." ]] || continue - LOG "[INFO] Found \"${change_file}\" changed." - change_file_name=${change_file#*paddle/fluid/operators/} - if [ -n "${PADDLE_FILENAME_OP_MAP[$change_file_name]}" ] + if [[ "$change_file" =~ "_op.cu" ]] then - for op_name in ${PADDLE_FILENAME_OP_MAP[$change_file_name]} - do - LOG "[INFO] Load op: \"${op_name}\"." - CHANGE_OP_MAP[${op_name}]="$change_file" - done - else - change_file_name=${change_file_name##*/} - LOG "[INFO] Load op: \"${change_file_name%_op*}\"." - CHANGE_OP_MAP[${change_file_name%_op*}]="$change_file" + LOG "[INFO] Found \"${change_file}\" changed." + CHANGE_OP_FILES[${#CHANGE_OP_FILES[@]}]="$change_file" + elif [[ "$change_file" =~ ".h" ]] + then + LOG "[INFO] Found \"${change_file}\" changed, keep searching." + load_CHANGE_OP_FILES_by_header_file $change_file fi done - [ ${#CHANGE_OP_MAP[*]} -eq 0 ] && LOG "[INFO] No op to test, skip this ci." && exit 0 + [ ${#CHANGE_OP_FILES[@]} -eq 0 ] && LOG "[INFO] No op to test, skip this ci." && exit 0 } # Clone benchmark repo @@ -96,12 +79,35 @@ function prepare_benchmark_environment { --test_module_name tests_v2 \ --info_file api_info.txt >& 2 [ $? -ne 0 ] && LOG "[FATAL] Collect api info fail." && exit -1 + [ ! -f benchmark/ci/scripts/op_benchmark.config ] && LOG "[FATAL] Missing op_benchmark.config!" && exit -1 } -# Load ops that will +# Load unique op name from CHANGE_OP_FILES +function load_CHANGE_OP_MAP { + local op_name change_file change_file_name + source benchmark/ci/scripts/op_benchmark.config + for change_file in ${CHANGE_OP_FILES[@]} + do + change_file_name=${change_file#*paddle/fluid/operators/} + if [ -n "${PADDLE_FILENAME_OP_MAP[$change_file_name]}" ] + then + for op_name in ${PADDLE_FILENAME_OP_MAP[$change_file_name]} + do + LOG "[INFO] Load op: \"${op_name}\"." + CHANGE_OP_MAP[${op_name}]="$change_file" + done + else + change_file_name=${change_file_name##*/} + LOG "[INFO] Load op: \"${change_file_name%_op*}\"." + CHANGE_OP_MAP[${change_file_name%_op*}]="$change_file" + fi + done +} + +# Load ops that will run benchmark test function load_BENCHMARK_OP_MAP { local line op_name api_name - prepare_benchmark_environment + source benchmark/ci/scripts/op_benchmark.config for line in $(cat api_info.txt) do api_name=${line%%,*} @@ -129,6 +135,7 @@ function compile_install_paddlepaddle { export WITH_PYTHON=ON export WITH_TESTING=OFF export BUILD_TYPE=Release + export CUDA_ARCH_NAME=Auto export WITH_DISTRIBUTE=OFF export PYTHON_ABI=cp37-cp37m export CMAKE_BUILD_TYPE=Release @@ -145,6 +152,8 @@ function compile_install_paddlepaddle { function run_op_benchmark_test { [ ${#BENCHMARK_OP_MAP[*]} -eq 0 ] && return local logs_dir op_name branch_name api_info_file + [ -z "$VISIBLE_DEVICES" ] && export VISIBLE_DEVICES=0 + [ "$BENCHMARK_PRINT_FAIL_LOG" != "1" ] && export BENCHMARK_PRINT_FAIL_LOG=1 api_info_file="$(pwd)/api_info.txt" [ -f "$api_info_file" ] && rm -f $api_info_file for api_info in ${BENCHMARK_OP_MAP[*]} @@ -159,7 +168,6 @@ function run_op_benchmark_test { compile_install_paddlepaddle logs_dir="$(pwd)/logs-${branch_name}" [ -d $logs_dir ] && rm -rf $logs_dir/* || mkdir -p $logs_dir - [ -z "$VISIBLE_DEVICES" ] && export VISIBLE_DEVICES=0 pushd benchmark/api > /dev/null bash deploy/main_control.sh tests_v2 \ tests_v2/configs \ @@ -176,10 +184,14 @@ function run_op_benchmark_test { # diff benchmakr result and miss op function summary_problems { local op_name exit_code - python ${PADDLE_ROOT}/tools/check_op_benchmark_result.py \ - --develop_logs_dir $(pwd)/logs-develop \ - --pr_logs_dir $(pwd)/logs-test_pr - exit_code=$? + exit_code=0 + if [ ${#BENCHMARK_OP_MAP[*]} -ne 0 ] + then + python ${PADDLE_ROOT}/tools/check_op_benchmark_result.py \ + --develop_logs_dir $(pwd)/logs-develop \ + --pr_logs_dir $(pwd)/logs-test_pr + exit_code=$? + fi for op_name in ${!CHANGE_OP_MAP[@]} do if [ -z "${BENCHMARK_OP_MAP[$op_name]}" ] @@ -193,6 +205,8 @@ function summary_problems { function main { LOG "[INFO] Start run op benchmark test ..." + load_CHANGE_OP_FILES + prepare_benchmark_environment load_CHANGE_OP_MAP load_BENCHMARK_OP_MAP run_op_benchmark_test -- GitLab