Skip to content

P
Paddle

PaddlePaddle / Paddle
大约 1 年 前同步成功

通知 2298
Star 20931
Fork 5422
P
Paddle
未验证 提交 8424cf28 编写于 作者: Y Yiqun Liu 提交者： GitHub
浏览文件
操作

Optimize the log of broadcast and decrease the log level. (#48327) 

* Optimize the log of broadcast and decrease the log level.

* Remove the redundant brackets.

* Change op benchmark ci to test the tests module.

* Remove the observe of elementwise and reduce_ops sub-directory.
上级 df82fd35
隐藏空白更改
内联 并排
Showing with 35 addition and 365 deletion
paddle/phi/kernels/funcs/broadcast_function.h
浏览文件 @ 8424cf28
......@@ -14,6 +14,7 @@ limitations under the License. */
#pragma once
#include <sstream>
#include "paddle/phi/kernels/funcs/elementwise_base.h"
#if defined(__NVCC__) || defined(__HIPCC__) || defined(__xpu__)
......@@ -776,6 +777,21 @@ struct LaunchBroadcastKernelWithInt64IndexHelper<InT,
};
#endif
template <typename T>
static std::string ReversedVectorToString(const std::vector<T> &reversed_v) {
std::stringstream ss;
bool is_last = true;
for (int i = reversed_v.size() - 1; i >= 0; --i) {
if (is_last) {
ss << reversed_v[i];
is_last = false;
} else {
ss << ", " << reversed_v[i];
}
}
return ss.str();
}
template <ElementwiseType ET,
typename InT,
typename OutT,
......@@ -877,15 +893,16 @@ void BroadcastKernelForDifferentVecSize(
// mergedim and get vec_size
const auto dims_simplifier =
BroadcastDimsSimplifier(ins, (*outs)[0]->dims(), axis);
if (VLOG_IS_ON(4)) {
if (VLOG_IS_ON(6)) {
for (size_t i = 0; i < ins.size(); ++i) {
VLOG(4) << "input i=" << i << ": origin_dims={" << ins[i]->dims()
VLOG(6) << "input i=" << i << ": origin_dims={" << ins[i]->dims()
<< "}, simplied_dims={"
<< phi::make_ddim(dims_simplifier.in_dims[i]) << "}";
<< ReversedVectorToString<int64_t>(dims_simplifier.in_dims[i])
<< "}";
}
VLOG(4) << "output: origin_dims={" << (*outs)[0]->dims()
<< "}, simplied_dims={" << phi::make_ddim(dims_simplifier.out_dims)
<< "}";
VLOG(6) << "output: origin_dims={" << (*outs)[0]->dims()
<< "}, simplied_dims={"
<< ReversedVectorToString<int64_t>(dims_simplifier.out_dims) << "}";
}
phi::Array<kps::details::BroadcastConfig, kArity> configs;
......
tools/ci_op_benchmark.sh
浏览文件 @ 8424cf28
......@@ -39,10 +39,9 @@ function match_cu_file_directory {
LOG "[INFO] run function match_cu_file_directory"
local sub_dir cu_file_dir
cu_file_dir=$(dirname ${1})
for sub_dir in "" "/elementwise" "/reduce_ops"
do
[ "${cu_file_dir}" == "paddle/fluid/operators${sub_dir}" ] && return 0
done
# the operators under paddle/fluid/operators directory
[ "${cu_file_dir}" == "paddle/fluid/operators" ] && return 0
# the operators under paddle/phi/kernels directory
for sub_dir in "" "/gpu" "/gpudnn" "/sparse/gpu"
do
[ "${cu_file_dir}" == "paddle/phi/kernels${sub_dir}" ] && return 0
......@@ -115,13 +114,13 @@ function load_CHANGE_OP_FILES {
}
# Clone benchmark repo
function prepare_benchmark_environment {
function clone_and_collect_op_info {
LOG "[INFO] Clone benchmark repo ..."
git clone https://github.com/PaddlePaddle/benchmark.git
[ $? -ne 0 ] && LOG "[FATAL] Clone benchmark repo fail." && exit -1
LOG "[INFO] Collect api info ..."
python benchmark/api/deploy/collect_api_info.py \
--test_module_name dynamic_tests_v2 \
--test_module_name tests \
--info_file api_info.txt >& 2
[ $? -ne 0 ] && LOG "[FATAL] Collect api info fail." && exit -1
[ ! -f benchmark/ci/scripts/op_benchmark.config ] && LOG "[FATAL] Missing op_benchmark.config!" && exit -1
......@@ -204,7 +203,7 @@ function run_op_benchmark_test {
logs_dir="$(pwd)/logs-${branch_name}"
[ -d $logs_dir ] && rm -rf $logs_dir/* || mkdir -p $logs_dir
pushd benchmark/api > /dev/null
bash deploy/main_control.sh dynamic_tests_v2 \
bash deploy/main_control.sh tests \
tests_v2/configs \
$logs_dir \
$VISIBLE_DEVICES \
......@@ -231,7 +230,7 @@ function check_op_benchmark_result {
# there is no need to recompile and install paddle
LOG "[INFO] retry ${retry_time} times ..."
pushd benchmark/api > /dev/null
bash deploy/main_control.sh dynamic_tests_v2 \
bash deploy/main_control.sh tests \
tests_v2/configs \
${logs_dir} \
$VISIBLE_DEVICES \
......@@ -285,13 +284,13 @@ function summary_problems {
}
function cpu_op_benchmark {
LOG "[INFO] Start run op benchmark cpu test ..."
function prepare_env {
LOG "[INFO] Start preparing op benchmark environment ..."
load_CHANGE_OP_FILES
prepare_benchmark_environment
clone_and_collect_op_info
load_CHANGE_OP_MAP
load_BENCHMARK_OP_MAP
LOG "[INFO] Op benchmark run success and no error!"
LOG "[INFO] Op benchmark environment is prepared success!"
}
......@@ -321,7 +320,7 @@ fi
case $1 in
run_op_benchmark)
cpu_op_benchmark
prepare_env
gpu_op_benchmark
;;
esac
tools/test_ci_op_benchmark.sh 已删除 100644 → 0
浏览文件 @ df82fd35
#!/bin/bash
# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set +ex
[ -z "$PADDLE_ROOT" ] && PADDLE_ROOT=$(cd $(dirname ${BASH_SOURCE[0]})/.. && pwd)
# PR modify op source files
CHANGE_OP_FILES=()
# ops that will run benchmark test
declare -A CHANGE_OP_MAP
# ops that benchmark repo has
declare -A BENCHMARK_OP_MAP
# searched header files
declare -A INCLUDE_SEARCH_MAP
function LOG {
echo "[$0:${BASH_LINENO[0]}] $*" >&2
}
# Limit cu file directory
function match_cu_file_directory {
LOG "[INFO] run function match_cu_file_directory"
local sub_dir cu_file_dir
cu_file_dir=$(dirname ${1})
for sub_dir in "" "/elementwise" "/reduce_ops"
do
[ "${cu_file_dir}" == "paddle/fluid/operators${sub_dir}" ] && return 0
done
return 1
}
# Load op files by header file
function load_CHANGE_OP_FILES_by_header_file {
LOG "[INFO] run function load_CHANGE_OP_FILES_by_header_file"
local change_file
for change_file in $(grep -rl "${1}" paddle/fluid/operators)
do
if [[ "$change_file" =~ "_op.cu" ]]
then
# match cu file directory limit
match_cu_file_directory $change_file || continue
LOG "[INFO] Found \"${1}\" include by \"${change_file}\"."
CHANGE_OP_FILES[${#CHANGE_OP_FILES[@]}]="$change_file"
elif [[ "$change_file" =~ ".h" ]]
then
[ -n "${INCLUDE_SEARCH_MAP[$change_file]}" ] && continue
LOG "[INFO] Found \"${1}\" include by \"${change_file}\", keep searching."
INCLUDE_SEARCH_MAP[$change_file]="searched"
load_CHANGE_OP_FILES_by_header_file $change_file
fi
done
}
# Load op files that PR changes
function load_CHANGE_OP_FILES {
LOG "[INFO] run function load_CHANGE_OP_FILES"
local sub_dir change_file
# TODO(Avin0323): Need to filter the files added by the new OP.
for change_file in $(git diff --name-only origin/develop)
do
# match directory limit
[[ "$change_file" =~ "paddle/fluid/operators/" ]] || continue
# match file name limit
if [[ "$change_file" =~ "_op.cu" ]]
then
# match cu file directory limit
match_cu_file_directory $change_file || continue
LOG "[INFO] Found \"${change_file}\" changed."
CHANGE_OP_FILES[${#CHANGE_OP_FILES[@]}]="$change_file"
elif [[ "$change_file" =~ ".h" ]]
then
LOG "[INFO] Found \"${change_file}\" changed, keep searching."
INCLUDE_SEARCH_MAP[${change_file}]="searched"
load_CHANGE_OP_FILES_by_header_file $change_file
fi
done
[ ${#CHANGE_OP_FILES[@]} -eq 0 ] && LOG "[INFO] No op to test, skip this ci." && \
echo "cpu_benchmark=ON" >${cfs_dir}/op_benchmark/${AGILE_PULL_ID}/${AGILE_REVISION}/pass.txt && \
exit 0
}
# Clone benchmark repo
function prepare_benchmark_environment {
LOG "[INFO] Clone benchmark repo ..."
git clone https://github.com/PaddlePaddle/benchmark.git
[ $? -ne 0 ] && LOG "[FATAL] Clone benchmark repo fail." && exit -1
LOG "[INFO] Collect api info ..."
python benchmark/api/deploy/collect_api_info.py \
--test_module_name tests_v2 \
--info_file api_info.txt >& 2
[ $? -ne 0 ] && LOG "[FATAL] Collect api info fail." && exit -1
[ ! -f benchmark/ci/scripts/op_benchmark.config ] && LOG "[FATAL] Missing op_benchmark.config!" && exit -1
}
# Load unique op name from CHANGE_OP_FILES
function load_CHANGE_OP_MAP {
LOG "[INFO] run function load_CHANGE_OP_MAP"
local op_name change_file change_file_name
source benchmark/ci/scripts/op_benchmark.config
for change_file in ${CHANGE_OP_FILES[@]}
do
change_file_name=${change_file#*paddle/fluid/operators/}
if [ -n "${PADDLE_FILENAME_OP_MAP[$change_file_name]}" ]
then
for op_name in ${PADDLE_FILENAME_OP_MAP[$change_file_name]}
do
LOG "[INFO] Load op: \"${op_name}\"."
CHANGE_OP_MAP[${op_name}]="$change_file"
done
else
op_name=${change_file_name##*/}
op_name=${op_name%_cudnn_op*}
op_name=${op_name%_op*}
[ -n "${SKIP_OP_MAP[$op_name]}" ] && continue
LOG "[INFO] Load op: \"${op_name}\"."
CHANGE_OP_MAP[${op_name}]="$change_file"
fi
done
}
# Load ops that will run benchmark test
function load_BENCHMARK_OP_MAP {
LOG "[INFO] run function load_BENCHMARK_OP_MAP"
local line op_name api_name
source benchmark/ci/scripts/op_benchmark.config
for line in $(cat api_info.txt)
do
api_name=${line%%,*}
if [ -n "${BENCHMARK_APINAME_OP_MAP[$api_name]}" ]
then
op_name=${BENCHMARK_APINAME_OP_MAP[$api_name]}
else
op_name=$api_name
fi
if [ -n "${CHANGE_OP_MAP[$op_name]}" ]
then
LOG "[INFO] Load benchmark settings with op \"${op_name}\"."
BENCHMARK_OP_MAP[$op_name]=$line
fi
done
}
# compile and install paddlepaddle
function compile_install_paddlepaddle {
LOG "[INFO] Compiling install package ..."
export WITH_GPU=ON
export WITH_AVX=ON
export WITH_MKL=ON
export RUN_TEST=OFF
export WITH_PYTHON=ON
export WITH_TESTING=OFF
export BUILD_TYPE=Release
export CUDA_ARCH_NAME=${CUDA_ARCH_NAME:-Auto}
export WITH_DISTRIBUTE=OFF
export CMAKE_BUILD_TYPE=Release
[ -d build ] && rm -rf build
bash paddle/scripts/paddle_build.sh build_only $(nproc)
[ $? -ne 0 ] && LOG "[FATAL] compile fail." && exit 7
LOG "[INFO] Build fineshed"
mkdir -p build_whl/${branch_name} && cp build/python/dist/paddlepaddle_gpu-0.0.0-cp37-cp37m-linux_x86_64.whl build_whl/${branch_name}/
}
function build_whl {
LOG "[INFO] run function build_whl"
for branch_name in "develop" "test"
do
git checkout ${branch_name}
[ $? -ne 0 ] && LOG "[FATAL] Missing branch ${branch_name}." && exit 7
LOG "[INFO] Now branch name is ${branch_name}."
compile_install_paddlepaddle
done
}
# run op benchmark test
function run_op_benchmark_test {
LOG "[INFO] run function run_op_benchmark_test"
[ ${#BENCHMARK_OP_MAP[*]} -eq 0 ] && return
local logs_dir op_name branch_name api_info_file
[ -z "$VISIBLE_DEVICES" ] && export VISIBLE_DEVICES=0
[ "$BENCHMARK_PRINT_FAIL_LOG" != "1" ] && export BENCHMARK_PRINT_FAIL_LOG=1
api_info_file="$(pwd)/api_info.txt"
[ -f "$api_info_file" ] && rm -f $api_info_file
for api_info in ${BENCHMARK_OP_MAP[*]}
do
echo "$api_info" >> $api_info_file
done
# install tensorflow for testing accuary
pip install tensorflow==2.3.0 tensorflow-probability
for branch_name in "develop" "test"
do
LOG "[INFO] Uninstall Paddle ..."
pip uninstall -y paddlepaddle paddlepaddle_gpu
LOG "[INFO] Install Paddle ..."
pip install build_whl/${branch_name}/paddlepaddle_gpu-0.0.0-cp37-cp37m-linux_x86_64.whl
logs_dir="$(pwd)/logs-${branch_name}"
[ -d $logs_dir ] && rm -rf $logs_dir/* || mkdir -p $logs_dir
pushd benchmark/api > /dev/null
bash deploy/main_control.sh tests_v2 \
tests_v2/configs \
$logs_dir \
$VISIBLE_DEVICES \
"gpu" \
"both" \
$api_info_file \
"paddle"
popd > /dev/null
done
}
# check benchmark result
function check_op_benchmark_result {
LOG "[INFO] run function check_op_benchmark_result"
local logs_dir api_info_file check_status_code
# default 3 times
[ -z "${RETRY_TIMES}" ] && RETRY_TIMES=3
logs_dir=$(pwd)/logs-test_pr
api_info_file=$(pwd)/api_info.txt
for retry_time in $(seq 0 ${RETRY_TIMES})
do
if [ $retry_time -gt 0 ]; then
# run op benchmark speed test
# there is no need to recompile and install paddle
LOG "[INFO] retry ${retry_time} times ..."
pushd benchmark/api > /dev/null
bash deploy/main_control.sh tests_v2 \
tests_v2/configs \
${logs_dir} \
$VISIBLE_DEVICES \
"gpu" \
"speed" \
${api_info_file} \
"paddle"
popd > /dev/null
fi
# check current result and update the file to benchmark test
python ${PADDLE_ROOT}/tools/check_op_benchmark_result.py \
--develop_logs_dir $(pwd)/logs-develop \
--pr_logs_dir $(pwd)/logs-test_pr \
--api_info_file ${api_info_file}
check_status_code=$?
# TODO(Avin0323): retry only if the performance check fails
[ $check_status_code -eq 0 ] && break
done
return $check_status_code
}
function check_CHANGE_OP_MAP {
LOG "[INFO] run function check_CHANGE_OP_MAP"
for op_name in ${!CHANGE_OP_MAP[@]}
do
if [ -z "${BENCHMARK_OP_MAP[$op_name]}" ]
then
exit_code=8
LOG "[ERROR] Missing test script of \"${op_name}\"(${CHANGE_OP_MAP[$op_name]}) in benchmark."
fi
done
if [ $exit_code -ne 0 ]; then
LOG "[INFO] See https://github.com/PaddlePaddle/Paddle/wiki/PR-CI-OP-benchmark-Manual for details."
LOG "[INFO] Or you can apply for one RD (ZzSean(Recommend), JamesLim-sy, Xreki, luotao1) approval to pass this PR."
exit $exit_code
fi
}
# diff benchmakr result and miss op
function summary_problems {
LOG "[INFO] run function summary_problems"
local op_name exit_code
exit_code=0
if [ ${#BENCHMARK_OP_MAP[*]} -ne 0 ]
then
check_op_benchmark_result
exit_code=$?
fi
check_CHANGE_OP_MAP
}
function cpu_op_benchmark {
LOG "[INFO] Start run op benchmark cpu test ..."
load_CHANGE_OP_FILES
prepare_benchmark_environment
load_CHANGE_OP_MAP
load_BENCHMARK_OP_MAP
# check_CHANGE_OP_MAP
build_whl
LOG "[INFO] Op benchmark run success and no error!"
exit 0
}
function gpu_op_benchmark {
LOG "[INFO] Start run op benchmark gpu test ..."
load_CHANGE_OP_FILES
load_CHANGE_OP_MAP
load_BENCHMARK_OP_MAP
run_op_benchmark_test
summary_problems
LOG "[INFO] Op benchmark run success and no error!"
exit 0
}
# The PR will pass quickly when get approval from specific person.
# Xreki 12538138, luotao1 6836917, ZzSean 32410583, JamesLim-sy 61349199
set +x
approval_line=$(curl -H "Authorization: token ${GITHUB_API_TOKEN}" https://api.github.com/repos/PaddlePaddle/Paddle/pulls/${GIT_PR_ID}/reviews?per_page=10000)
if [ -n "${approval_line}" ]; then
APPROVALS=$(echo ${approval_line} | python ${PADDLE_ROOT}/tools/check_pr_approval.py 1 32410583 12538138 6836917 61349199)
LOG "[INFO] current pr ${GIT_PR_ID} got approvals: ${APPROVALS}"
if [ "${APPROVALS}" == "TRUE" ]; then
LOG "[INFO] ==================================="
LOG "[INFO] current pr ${GIT_PR_ID} has got approvals. So, Pass CI directly!"
LOG "[INFO] ==================================="
exit 0
fi
fi
set -x
case $1 in
cpu_op_benchmark)
cpu_op_benchmark
;;
gpu_op_benchmark)
gpu_op_benchmark
;;
*)
cpu_op_benchmark
gpu_op_benchmark
;;
esac
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或