未验证 提交 8f416eb1 编写于 作者: C Chen Weihang 提交者: GitHub

Add grep enforce shell & polish other enforce count shell (#24897)

* add grep enforce shell, test=develop, test=document_fix

* add copyright desc, test=develop, test=document_fix

* refactor four scripts, test=develop, test=document_fix

* change white list var, test=develop, test=document_fix

* add args for count_enforce_by_file, test=develop, test=document_fix
上级 b174b997
#!/bin/bash #!/bin/bash
# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# This script is used to count all PADDLE checks in the paddle/fluid directory, # This script is used to count all PADDLE checks in the paddle/fluid directory,
# including the total PADDLE check number, the valid check number and the # including the total PADDLE check number, the valid check number and the
# invalid check number under paddle/fluid and its main sub-directories. # invalid check number under paddle/fluid and its main sub-directories.
# Usage: bash count_invalid_enforce.sh (run in tools directory) # Usage: bash count_all_enforce.sh (run in tools directory)
# Result Example: # Result Example:
...@@ -29,7 +43,7 @@ ROOT_DIR=../paddle/fluid ...@@ -29,7 +43,7 @@ ROOT_DIR=../paddle/fluid
ALL_PADDLE_CHECK_CNT=0 ALL_PADDLE_CHECK_CNT=0
VALID_PADDLE_CHECK_CNT=0 VALID_PADDLE_CHECK_CNT=0
function enforce_scan(){ function enforce_count(){
paddle_check=`grep -r -zoE "(PADDLE_ENFORCE[A-Z_]{0,9}|PADDLE_THROW)\(.[^,\);]*.[^;]*\);\s" $1 || true` paddle_check=`grep -r -zoE "(PADDLE_ENFORCE[A-Z_]{0,9}|PADDLE_THROW)\(.[^,\);]*.[^;]*\);\s" $1 || true`
total_check_cnt=`echo "$paddle_check" | grep -cE "(PADDLE_ENFORCE|PADDLE_THROW)" || true` total_check_cnt=`echo "$paddle_check" | grep -cE "(PADDLE_ENFORCE|PADDLE_THROW)" || true`
valid_check_cnt=`echo "$paddle_check" | grep -zoE '(PADDLE_ENFORCE[A-Z_]{0,9}|PADDLE_THROW)\((.[^,;]+,)*.[^";]*(errors::).[^"]*".[^";]{20,}.[^;]*\);\s' | grep -cE "(PADDLE_ENFORCE|PADDLE_THROW)" || true` valid_check_cnt=`echo "$paddle_check" | grep -zoE '(PADDLE_ENFORCE[A-Z_]{0,9}|PADDLE_THROW)\((.[^,;]+,)*.[^";]*(errors::).[^"]*".[^";]{20,}.[^;]*\);\s' | grep -cE "(PADDLE_ENFORCE|PADDLE_THROW)" || true`
...@@ -37,28 +51,34 @@ function enforce_scan(){ ...@@ -37,28 +51,34 @@ function enforce_scan(){
eval $3=$valid_check_cnt eval $3=$valid_check_cnt
} }
function walk_dir(){ function count_dir_recursively(){
for file in `ls $1` for file in `ls $1`
do do
if [ -d $1"/"$file ];then if [ -d $1"/"$file ];then
level=$(($2+1)) level=$(($2+1))
if [ $level -le 1 ]; then if [ $level -le 1 ]; then
enforce_scan $1"/"$file total_check_cnt valid_check_cnt enforce_count $1"/"$file total_check_cnt valid_check_cnt
dir_name=$1 dir_name=$1
echo "${dir_name#../}/"$file" | ${total_check_cnt} | ${valid_check_cnt} | $(($total_check_cnt-$valid_check_cnt))" echo "${dir_name#../}/"$file" | ${total_check_cnt} | ${valid_check_cnt} | $(($total_check_cnt-$valid_check_cnt))"
ALL_PADDLE_CHECK_CNT=$(($ALL_PADDLE_CHECK_CNT+$total_check_cnt)) ALL_PADDLE_CHECK_CNT=$(($ALL_PADDLE_CHECK_CNT+$total_check_cnt))
VALID_PADDLE_CHECK_CNT=$(($VALID_PADDLE_CHECK_CNT+$valid_check_cnt)) VALID_PADDLE_CHECK_CNT=$(($VALID_PADDLE_CHECK_CNT+$valid_check_cnt))
walk_dir $1"/"$file $level count_dir_recursively $1"/"$file $level
fi fi
fi fi
done done
} }
walk_dir $ROOT_DIR 0 main() {
count_dir_recursively $ROOT_DIR 0
echo "----------------------------"
echo "PADDLE ENFORCE & THROW COUNT"
echo "----------------------------"
echo "All PADDLE_ENFORCE{_**} & PADDLE_THROW Count: ${ALL_PADDLE_CHECK_CNT}"
echo "Valid PADDLE_ENFORCE{_**} & PADDLE_THROW Count: ${VALID_PADDLE_CHECK_CNT}"
echo "Invalid PADDLE_ENFORCE{_**} & PADDLE_THROW Count: $(($ALL_PADDLE_CHECK_CNT-$VALID_PADDLE_CHECK_CNT))"
}
echo "----------------------------" if [ "${1}" != "--source-only" ]; then
echo "PADDLE ENFORCE & THROW COUNT" main "${@}"
echo "----------------------------" fi
echo "All PADDLE_ENFORCE{_**} & PADDLE_THROW Count: ${ALL_PADDLE_CHECK_CNT}"
echo "Valid PADDLE_ENFORCE{_**} & PADDLE_THROW Count: ${VALID_PADDLE_CHECK_CNT}"
echo "Invalid PADDLE_ENFORCE{_**} & PADDLE_THROW Count: $(($ALL_PADDLE_CHECK_CNT-$VALID_PADDLE_CHECK_CNT))"
#!/bin/bash #!/bin/bash
# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# This script is used to count detail PADDLE checks in the paddle/fluid directory, # This script is used to count detail PADDLE checks in the paddle/fluid directory,
# contains the number of PADDLE checks under each folder, the statistical data # contains the number of PADDLE checks under each folder, the statistical data
# does not include subdirectories, only covers all files under the current directory. # does not include subdirectories, only covers all files under the current directory.
...@@ -7,7 +21,7 @@ ...@@ -7,7 +21,7 @@
# The three columns of data are: total number, valid number, invalid number. # The three columns of data are: total number, valid number, invalid number.
# The output format is easy to display as a markdown table. # The output format is easy to display as a markdown table.
# Usage: bash detail_invalid_enforce.sh (run in tools directory) # Usage: bash count_enforce_by_dir.sh (run in tools directory)
# Result Example: # Result Example:
...@@ -43,27 +57,21 @@ ...@@ -43,27 +57,21 @@
# paddle/fluid/operators/tensorrt | 7 | 4 | 3 # paddle/fluid/operators/tensorrt | 7 | 4 | 3
# paddle/fluid/operators | 2144 | 999 | 1145 # paddle/fluid/operators | 2144 | 999 | 1145
ROOT_DIR=../paddle/fluid . ./count_all_enforce.sh --source-only
function enforce_scan(){ ROOT_DIR=../paddle/fluid
paddle_check=`grep -r -zoE "(PADDLE_ENFORCE[A-Z_]{0,9}|PADDLE_THROW)\(.[^,\);]*.[^;]*\);\s" $1 || true`
total_check_cnt=`echo "$paddle_check" | grep -cE "(PADDLE_ENFORCE|PADDLE_THROW)" || true`
valid_check_cnt=`echo "$paddle_check" | grep -zoE '(PADDLE_ENFORCE[A-Z_]{0,9}|PADDLE_THROW)\((.[^,;]+,)*.[^";]*(errors::).[^"]*".[^";]{20,}.[^;]*\);\s' | grep -cE "(PADDLE_ENFORCE|PADDLE_THROW)" || true`
eval $2=$total_check_cnt
eval $3=$valid_check_cnt
}
function walk_dir(){ function count_dir_independently(){
local sub_dir_total_check_cnt=0 local sub_dir_total_check_cnt=0
local sub_dir_valid_check_cnt=0 local sub_dir_valid_check_cnt=0
for file in `ls $1` for file in `ls $1`
do do
if [ -d $1"/"$file ];then if [ -d $1"/"$file ];then
enforce_scan $1"/"$file dir_total_check_cnt dir_valid_check_cnt enforce_count $1"/"$file dir_total_check_cnt dir_valid_check_cnt
sub_dir_total_check_cnt=$(($sub_dir_total_check_cnt+$dir_total_check_cnt)) sub_dir_total_check_cnt=$(($sub_dir_total_check_cnt+$dir_total_check_cnt))
sub_dir_valid_check_cnt=$(($sub_dir_valid_check_cnt+$dir_valid_check_cnt)) sub_dir_valid_check_cnt=$(($sub_dir_valid_check_cnt+$dir_valid_check_cnt))
walk_dir $1"/"$file $dir_total_check_cnt $dir_valid_check_cnt count_dir_independently $1"/"$file $dir_total_check_cnt $dir_valid_check_cnt
fi fi
done done
total_check_cnt=$(($2-$sub_dir_total_check_cnt)) total_check_cnt=$(($2-$sub_dir_total_check_cnt))
...@@ -72,4 +80,10 @@ function walk_dir(){ ...@@ -72,4 +80,10 @@ function walk_dir(){
echo "${dir_name#../} | ${total_check_cnt} | ${valid_check_cnt} | $(($total_check_cnt-$valid_check_cnt))" echo "${dir_name#../} | ${total_check_cnt} | ${valid_check_cnt} | $(($total_check_cnt-$valid_check_cnt))"
} }
walk_dir $ROOT_DIR 0 0 main() {
count_dir_independently $ROOT_DIR 0 0
}
if [ "${1}" != "--source-only" ]; then
main "${@}"
fi
#!/bin/bash #!/bin/bash
# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# This script is used to count PADDLE checks by files in the paddle/fluid/operators directory, # This script is used to count PADDLE checks by files in the paddle/fluid/operators directory,
# contains the number of PADDLE checks under each file. # contains the number of PADDLE checks under each file.
# #
# The three columns of data are: total number, valid number, invalid number. # The three columns of data are: total number, valid number, invalid number.
# The output format is easy to display as a markdown table. # The output format is easy to display as a markdown table.
# Usage: bash file_invalid_enforce.sh (run in tools directory) # Usage: bash count_enforce_by_file.sh [target directory or file] (run in tools directory)
# - The default check path is paddle/fluid/operators
# Result Example: # Result Example:
...@@ -27,9 +42,15 @@ ...@@ -27,9 +42,15 @@
# - math_function.cu | 4 | 0 | 4 # - math_function.cu | 4 | 0 | 4
# - math_function_impl.h | 10 | 0 | 10 # - math_function_impl.h | 10 | 0 | 10
. ./count_all_enforce.sh --source-only
ROOT_DIR=../paddle/fluid/operators ROOT_DIR=../paddle/fluid/operators
white_list_str = "\ if [ "$1" != "" ]; then
ROOT_DIR=$1
fi
FILE_WHITE_LIST="\
layer_norm_op.cc \ layer_norm_op.cc \
box_clip_op.cc \ box_clip_op.cc \
box_clip_op.h \ box_clip_op.h \
...@@ -38,15 +59,7 @@ white_list_str = "\ ...@@ -38,15 +59,7 @@ white_list_str = "\
fused_elemwise_activation_op.cc \ fused_elemwise_activation_op.cc \
auc_op.cu" auc_op.cu"
function enforce_scan(){ function count_file_recursively(){
paddle_check=`grep -r -zoE "(PADDLE_ENFORCE[A-Z_]{0,9}|PADDLE_THROW)\(.[^,\);]*.[^;]*\);\s" $1 || true`
total_check_cnt=`echo "$paddle_check" | grep -cE "(PADDLE_ENFORCE|PADDLE_THROW)" || true`
valid_check_cnt=`echo "$paddle_check" | grep -zoE '(PADDLE_ENFORCE[A-Z_]{0,9}|PADDLE_THROW)\((.[^,;]+,)*.[^";]*(errors::).[^"]*".[^";]{20,}.[^;]*\);\s' | grep -cE "(PADDLE_ENFORCE|PADDLE_THROW)" || true`
eval $2=$total_check_cnt
eval $3=$valid_check_cnt
}
function walk_dir(){
dir_name=$1 dir_name=$1
echo "**${dir_name#../}** | **$2** | **$3** | **$(($2-$3))**" echo "**${dir_name#../}** | **$2** | **$3** | **$(($2-$3))**"
local i=0 local i=0
...@@ -54,9 +67,9 @@ function walk_dir(){ ...@@ -54,9 +67,9 @@ function walk_dir(){
for file in `ls $1` for file in `ls $1`
do do
if [ -f $1"/"$file ];then if [ -f $1"/"$file ];then
in_white_list=$(echo $white_list_str | grep "${file}") in_white_list=$(echo $FILE_WHITE_LIST | grep "${file}")
if [[ "$in_white_list" == "" ]];then if [[ "$in_white_list" == "" ]];then
enforce_scan $1"/"$file file_total_check_cnt file_valid_check_cnt enforce_count $1"/"$file file_total_check_cnt file_valid_check_cnt
file_invalid_check_cnt=$(($total_check_cnt-$valid_check_cnt)) file_invalid_check_cnt=$(($total_check_cnt-$valid_check_cnt))
if [ $file_invalid_check_cnt -gt 0 ];then if [ $file_invalid_check_cnt -gt 0 ];then
echo "- $file | ${file_total_check_cnt} | ${file_valid_check_cnt} | ${file_invalid_check_cnt}" echo "- $file | ${file_total_check_cnt} | ${file_valid_check_cnt} | ${file_invalid_check_cnt}"
...@@ -70,9 +83,15 @@ function walk_dir(){ ...@@ -70,9 +83,15 @@ function walk_dir(){
done done
for sub_dir_name in ${dir_array[@]} for sub_dir_name in ${dir_array[@]}
do do
enforce_scan $sub_dir_name dir_total_check_cnt dir_valid_check_cnt enforce_count $sub_dir_name dir_total_check_cnt dir_valid_check_cnt
walk_dir $sub_dir_name $dir_total_check_cnt $dir_valid_check_cnt count_file_recursively $sub_dir_name $dir_total_check_cnt $dir_valid_check_cnt
done done
} }
walk_dir $ROOT_DIR 0 0 main() {
count_file_recursively $ROOT_DIR 0 0
}
if [ "${1}" != "--source-only" ]; then
main "${@}"
fi
#!/bin/bash
# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# This script is used to grep invalid PADDLE checks by directory or file in the paddle/fluid/,
# the result show all invalid PADDLE checks in specified directory or file.
# Usage:
# - bash grep_invalid_enforce.sh [target directory or file] (run in tools directory)
# - The default check path is paddle/fluid/operators
# Result Examples:
# 1. grep invalid PADDLE checks in directory
# - Command: /work/paddle/tools {develop} bash grep_invalid_enforce.sh ../paddle/fluid/imperative
# - Results:
# - paddle/fluid/imperative/gradient_accumulator.cc
# PADDLE_ENFORCE_EQ(dst_tensor->numel() == numel, true,
# "dst_numel %d vs. src_numel %d", dst_tensor->numel(),
# numel);
# - paddle/fluid/imperative/nccl_context.cc
# PADDLE_ENFORCE_EQ(addr.size(), 2UL,
# "The endpoint should contain host and port: %s", ep);
# PADDLE_THROW("create server fd failed");
# PADDLE_THROW("set socket opt failed");
# PADDLE_THROW("binding failed on ep: %s", ep);
# PADDLE_THROW("listen on server fd failed");
# PADDLE_THROW("accept the new socket fd failed");
# PADDLE_THROW("reading the ncclUniqueId from socket failed");
# PADDLE_ENFORCE_EQ(addr.size(), 2UL,
# "The endpoint should contain host and port: %s", ep);
# PADDLE_THROW("create socket failed");
# PADDLE_THROW("invalied address: %s", ep);
# - paddle/fluid/imperative/jit/program_desc_tracer.cc
# PADDLE_ENFORCE_NOT_NULL(new_var);
# PADDLE_ENFORCE_EQ(inner_var.IsInitialized(), true);
# PADDLE_THROW("Not support variable type %s",
# framework::ToTypeName(inner_var.Type()));
# 2. grep invalid PADDLE checks in file
# - Command: /work/paddle/tools {develop} bash grep_invalid_enforce.sh ../paddle/fluid/pybind/reader_py.cc
# - Results:
# - paddle/fluid/pybind/reader_py.cc
# PADDLE_THROW(
# "Place cannot be CUDAPlace when use_double_buffer is False");
# PADDLE_ENFORCE_NOT_NULL(exceptions_[i]);
# PADDLE_ENFORCE_EQ(status, Status::kException);
# PADDLE_ENFORCE_EQ(status, Status::kSuccess);
. ./count_enforce_by_file.sh --source-only
ROOT_DIR=../paddle/fluid/operators
if [ "$1" != "" ]; then
ROOT_DIR=$1
fi
function enforce_grep(){
paddle_check=`grep -zoE "(PADDLE_ENFORCE[A-Z_]{0,9}|PADDLE_THROW)\(.[^,\);]*.[^;]*\);\s" $1 || true`
valid_check=`echo "$paddle_check" | grep -zoE '(PADDLE_ENFORCE[A-Z_]{0,9}|PADDLE_THROW)\((.[^,;]+,)*.[^";]*(errors::).[^"]*".[^";]{20,}.[^;]*\);\s' || true`
invalid_check=`echo "$paddle_check" | grep -vxF "$valid_check" || true`
if [ "${invalid_check}" != "" ];then
file_path=$1
echo -e "\n- ${file_path#../}"
echo "${invalid_check}"
fi
}
function grep_file_recursively(){
local i=0
local dir_array
for file in `ls $1`
do
if [ -f $1"/"$file ];then
in_white_list=$(echo $FILE_WHITE_LIST | grep "${file}")
if [[ "$in_white_list" == "" ]];then
enforce_grep $1"/"$file
fi
fi
if [ -d $1"/"$file ];then
dir_array[$i]=$1"/"$file
((i++))
fi
done
for sub_dir_name in ${dir_array[@]}
do
grep_file_recursively $sub_dir_name
done
}
function grep_file(){
file_path=$1
file_name=`echo ${file_path##*/} `
if [ -f $file_path ];then
in_white_list=$(echo $FILE_WHITE_LIST | grep "${file_name}")
if [[ "$in_white_list" == "" ]];then
enforce_grep $file_path
fi
fi
}
main() {
if [ -f $ROOT_DIR ];then
grep_file $ROOT_DIR
else
grep_file_recursively $ROOT_DIR
fi
}
if [ "${1}" != "--source-only" ]; then
main "${@}"
fi
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册