From 8f416eb1b4bbdd1e0d284baa6251e57feded11b5 Mon Sep 17 00:00:00 2001 From: Chen Weihang Date: Thu, 4 Jun 2020 21:02:39 +0800 Subject: [PATCH] Add grep enforce shell & polish other enforce count shell (#24897) * add grep enforce shell, test=develop, test=document_fix * add copyright desc, test=develop, test=document_fix * refactor four scripts, test=develop, test=document_fix * change white list var, test=develop, test=document_fix * add args for count_enforce_by_file, test=develop, test=document_fix --- ...nvalid_enforce.sh => count_all_enforce.sh} | 44 ++++-- ...lid_enforce.sh => count_enforce_by_dir.sh} | 40 ++++-- ...id_enforce.sh => count_enforce_by_file.sh} | 51 ++++--- tools/grep_invalid_enforce.sh | 127 ++++++++++++++++++ 4 files changed, 221 insertions(+), 41 deletions(-) rename tools/{count_invalid_enforce.sh => count_all_enforce.sh} (62%) rename tools/{detail_invalid_enforce.sh => count_enforce_by_dir.sh} (72%) rename tools/{file_invalid_enforce.sh => count_enforce_by_file.sh} (57%) create mode 100644 tools/grep_invalid_enforce.sh diff --git a/tools/count_invalid_enforce.sh b/tools/count_all_enforce.sh similarity index 62% rename from tools/count_invalid_enforce.sh rename to tools/count_all_enforce.sh index 0294132e25d..c1b7508de03 100644 --- a/tools/count_invalid_enforce.sh +++ b/tools/count_all_enforce.sh @@ -1,10 +1,24 @@ #!/bin/bash +# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # This script is used to count all PADDLE checks in the paddle/fluid directory, # including the total PADDLE check number, the valid check number and the # invalid check number under paddle/fluid and its main sub-directories. -# Usage: bash count_invalid_enforce.sh (run in tools directory) +# Usage: bash count_all_enforce.sh (run in tools directory) # Result Example: @@ -29,7 +43,7 @@ ROOT_DIR=../paddle/fluid ALL_PADDLE_CHECK_CNT=0 VALID_PADDLE_CHECK_CNT=0 -function enforce_scan(){ +function enforce_count(){ paddle_check=`grep -r -zoE "(PADDLE_ENFORCE[A-Z_]{0,9}|PADDLE_THROW)\(.[^,\);]*.[^;]*\);\s" $1 || true` total_check_cnt=`echo "$paddle_check" | grep -cE "(PADDLE_ENFORCE|PADDLE_THROW)" || true` valid_check_cnt=`echo "$paddle_check" | grep -zoE '(PADDLE_ENFORCE[A-Z_]{0,9}|PADDLE_THROW)\((.[^,;]+,)*.[^";]*(errors::).[^"]*".[^";]{20,}.[^;]*\);\s' | grep -cE "(PADDLE_ENFORCE|PADDLE_THROW)" || true` @@ -37,28 +51,34 @@ function enforce_scan(){ eval $3=$valid_check_cnt } -function walk_dir(){ +function count_dir_recursively(){ for file in `ls $1` do if [ -d $1"/"$file ];then level=$(($2+1)) if [ $level -le 1 ]; then - enforce_scan $1"/"$file total_check_cnt valid_check_cnt + enforce_count $1"/"$file total_check_cnt valid_check_cnt dir_name=$1 echo "${dir_name#../}/"$file" | ${total_check_cnt} | ${valid_check_cnt} | $(($total_check_cnt-$valid_check_cnt))" ALL_PADDLE_CHECK_CNT=$(($ALL_PADDLE_CHECK_CNT+$total_check_cnt)) VALID_PADDLE_CHECK_CNT=$(($VALID_PADDLE_CHECK_CNT+$valid_check_cnt)) - walk_dir $1"/"$file $level + count_dir_recursively $1"/"$file $level fi fi done } -walk_dir $ROOT_DIR 0 +main() { + count_dir_recursively $ROOT_DIR 0 + + echo "----------------------------" + echo "PADDLE ENFORCE & THROW COUNT" + echo "----------------------------" + echo "All PADDLE_ENFORCE{_**} & PADDLE_THROW Count: ${ALL_PADDLE_CHECK_CNT}" + echo "Valid PADDLE_ENFORCE{_**} & PADDLE_THROW Count: ${VALID_PADDLE_CHECK_CNT}" + echo "Invalid PADDLE_ENFORCE{_**} & PADDLE_THROW Count: $(($ALL_PADDLE_CHECK_CNT-$VALID_PADDLE_CHECK_CNT))" +} -echo "----------------------------" -echo "PADDLE ENFORCE & THROW COUNT" -echo "----------------------------" -echo "All PADDLE_ENFORCE{_**} & PADDLE_THROW Count: ${ALL_PADDLE_CHECK_CNT}" -echo "Valid PADDLE_ENFORCE{_**} & PADDLE_THROW Count: ${VALID_PADDLE_CHECK_CNT}" -echo "Invalid PADDLE_ENFORCE{_**} & PADDLE_THROW Count: $(($ALL_PADDLE_CHECK_CNT-$VALID_PADDLE_CHECK_CNT))" +if [ "${1}" != "--source-only" ]; then + main "${@}" +fi diff --git a/tools/detail_invalid_enforce.sh b/tools/count_enforce_by_dir.sh similarity index 72% rename from tools/detail_invalid_enforce.sh rename to tools/count_enforce_by_dir.sh index 7424107e237..03233d417ac 100644 --- a/tools/detail_invalid_enforce.sh +++ b/tools/count_enforce_by_dir.sh @@ -1,5 +1,19 @@ #!/bin/bash +# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # This script is used to count detail PADDLE checks in the paddle/fluid directory, # contains the number of PADDLE checks under each folder, the statistical data # does not include subdirectories, only covers all files under the current directory. @@ -7,7 +21,7 @@ # The three columns of data are: total number, valid number, invalid number. # The output format is easy to display as a markdown table. -# Usage: bash detail_invalid_enforce.sh (run in tools directory) +# Usage: bash count_enforce_by_dir.sh (run in tools directory) # Result Example: @@ -43,27 +57,21 @@ # paddle/fluid/operators/tensorrt | 7 | 4 | 3 # paddle/fluid/operators | 2144 | 999 | 1145 -ROOT_DIR=../paddle/fluid +. ./count_all_enforce.sh --source-only -function enforce_scan(){ - paddle_check=`grep -r -zoE "(PADDLE_ENFORCE[A-Z_]{0,9}|PADDLE_THROW)\(.[^,\);]*.[^;]*\);\s" $1 || true` - total_check_cnt=`echo "$paddle_check" | grep -cE "(PADDLE_ENFORCE|PADDLE_THROW)" || true` - valid_check_cnt=`echo "$paddle_check" | grep -zoE '(PADDLE_ENFORCE[A-Z_]{0,9}|PADDLE_THROW)\((.[^,;]+,)*.[^";]*(errors::).[^"]*".[^";]{20,}.[^;]*\);\s' | grep -cE "(PADDLE_ENFORCE|PADDLE_THROW)" || true` - eval $2=$total_check_cnt - eval $3=$valid_check_cnt -} +ROOT_DIR=../paddle/fluid -function walk_dir(){ +function count_dir_independently(){ local sub_dir_total_check_cnt=0 local sub_dir_valid_check_cnt=0 for file in `ls $1` do if [ -d $1"/"$file ];then - enforce_scan $1"/"$file dir_total_check_cnt dir_valid_check_cnt + enforce_count $1"/"$file dir_total_check_cnt dir_valid_check_cnt sub_dir_total_check_cnt=$(($sub_dir_total_check_cnt+$dir_total_check_cnt)) sub_dir_valid_check_cnt=$(($sub_dir_valid_check_cnt+$dir_valid_check_cnt)) - walk_dir $1"/"$file $dir_total_check_cnt $dir_valid_check_cnt + count_dir_independently $1"/"$file $dir_total_check_cnt $dir_valid_check_cnt fi done total_check_cnt=$(($2-$sub_dir_total_check_cnt)) @@ -72,4 +80,10 @@ function walk_dir(){ echo "${dir_name#../} | ${total_check_cnt} | ${valid_check_cnt} | $(($total_check_cnt-$valid_check_cnt))" } -walk_dir $ROOT_DIR 0 0 +main() { + count_dir_independently $ROOT_DIR 0 0 +} + +if [ "${1}" != "--source-only" ]; then + main "${@}" +fi diff --git a/tools/file_invalid_enforce.sh b/tools/count_enforce_by_file.sh similarity index 57% rename from tools/file_invalid_enforce.sh rename to tools/count_enforce_by_file.sh index 3feb10cfb97..1858bd0fd17 100644 --- a/tools/file_invalid_enforce.sh +++ b/tools/count_enforce_by_file.sh @@ -1,12 +1,27 @@ #!/bin/bash +# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # This script is used to count PADDLE checks by files in the paddle/fluid/operators directory, # contains the number of PADDLE checks under each file. # # The three columns of data are: total number, valid number, invalid number. # The output format is easy to display as a markdown table. -# Usage: bash file_invalid_enforce.sh (run in tools directory) +# Usage: bash count_enforce_by_file.sh [target directory or file] (run in tools directory) +# - The default check path is paddle/fluid/operators # Result Example: @@ -27,9 +42,15 @@ # - math_function.cu | 4 | 0 | 4 # - math_function_impl.h | 10 | 0 | 10 +. ./count_all_enforce.sh --source-only + ROOT_DIR=../paddle/fluid/operators -white_list_str = "\ +if [ "$1" != "" ]; then + ROOT_DIR=$1 +fi + +FILE_WHITE_LIST="\ layer_norm_op.cc \ box_clip_op.cc \ box_clip_op.h \ @@ -38,15 +59,7 @@ white_list_str = "\ fused_elemwise_activation_op.cc \ auc_op.cu" -function enforce_scan(){ - paddle_check=`grep -r -zoE "(PADDLE_ENFORCE[A-Z_]{0,9}|PADDLE_THROW)\(.[^,\);]*.[^;]*\);\s" $1 || true` - total_check_cnt=`echo "$paddle_check" | grep -cE "(PADDLE_ENFORCE|PADDLE_THROW)" || true` - valid_check_cnt=`echo "$paddle_check" | grep -zoE '(PADDLE_ENFORCE[A-Z_]{0,9}|PADDLE_THROW)\((.[^,;]+,)*.[^";]*(errors::).[^"]*".[^";]{20,}.[^;]*\);\s' | grep -cE "(PADDLE_ENFORCE|PADDLE_THROW)" || true` - eval $2=$total_check_cnt - eval $3=$valid_check_cnt -} - -function walk_dir(){ +function count_file_recursively(){ dir_name=$1 echo "**${dir_name#../}** | **$2** | **$3** | **$(($2-$3))**" local i=0 @@ -54,9 +67,9 @@ function walk_dir(){ for file in `ls $1` do if [ -f $1"/"$file ];then - in_white_list=$(echo $white_list_str | grep "${file}") + in_white_list=$(echo $FILE_WHITE_LIST | grep "${file}") if [[ "$in_white_list" == "" ]];then - enforce_scan $1"/"$file file_total_check_cnt file_valid_check_cnt + enforce_count $1"/"$file file_total_check_cnt file_valid_check_cnt file_invalid_check_cnt=$(($total_check_cnt-$valid_check_cnt)) if [ $file_invalid_check_cnt -gt 0 ];then echo "- $file | ${file_total_check_cnt} | ${file_valid_check_cnt} | ${file_invalid_check_cnt}" @@ -70,9 +83,15 @@ function walk_dir(){ done for sub_dir_name in ${dir_array[@]} do - enforce_scan $sub_dir_name dir_total_check_cnt dir_valid_check_cnt - walk_dir $sub_dir_name $dir_total_check_cnt $dir_valid_check_cnt + enforce_count $sub_dir_name dir_total_check_cnt dir_valid_check_cnt + count_file_recursively $sub_dir_name $dir_total_check_cnt $dir_valid_check_cnt done } -walk_dir $ROOT_DIR 0 0 +main() { + count_file_recursively $ROOT_DIR 0 0 +} + +if [ "${1}" != "--source-only" ]; then + main "${@}" +fi diff --git a/tools/grep_invalid_enforce.sh b/tools/grep_invalid_enforce.sh new file mode 100644 index 00000000000..04243bfb9af --- /dev/null +++ b/tools/grep_invalid_enforce.sh @@ -0,0 +1,127 @@ +#!/bin/bash + +# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script is used to grep invalid PADDLE checks by directory or file in the paddle/fluid/, +# the result show all invalid PADDLE checks in specified directory or file. + +# Usage: +# - bash grep_invalid_enforce.sh [target directory or file] (run in tools directory) +# - The default check path is paddle/fluid/operators + +# Result Examples: +# 1. grep invalid PADDLE checks in directory + +# - Command: /work/paddle/tools {develop} bash grep_invalid_enforce.sh ../paddle/fluid/imperative +# - Results: +# - paddle/fluid/imperative/gradient_accumulator.cc +# PADDLE_ENFORCE_EQ(dst_tensor->numel() == numel, true, +# "dst_numel %d vs. src_numel %d", dst_tensor->numel(), +# numel); + +# - paddle/fluid/imperative/nccl_context.cc +# PADDLE_ENFORCE_EQ(addr.size(), 2UL, +# "The endpoint should contain host and port: %s", ep); +# PADDLE_THROW("create server fd failed"); +# PADDLE_THROW("set socket opt failed"); +# PADDLE_THROW("binding failed on ep: %s", ep); +# PADDLE_THROW("listen on server fd failed"); +# PADDLE_THROW("accept the new socket fd failed"); +# PADDLE_THROW("reading the ncclUniqueId from socket failed"); +# PADDLE_ENFORCE_EQ(addr.size(), 2UL, +# "The endpoint should contain host and port: %s", ep); +# PADDLE_THROW("create socket failed"); +# PADDLE_THROW("invalied address: %s", ep); + +# - paddle/fluid/imperative/jit/program_desc_tracer.cc +# PADDLE_ENFORCE_NOT_NULL(new_var); +# PADDLE_ENFORCE_EQ(inner_var.IsInitialized(), true); +# PADDLE_THROW("Not support variable type %s", +# framework::ToTypeName(inner_var.Type())); + +# 2. grep invalid PADDLE checks in file + +# - Command: /work/paddle/tools {develop} bash grep_invalid_enforce.sh ../paddle/fluid/pybind/reader_py.cc +# - Results: +# - paddle/fluid/pybind/reader_py.cc +# PADDLE_THROW( +# "Place cannot be CUDAPlace when use_double_buffer is False"); +# PADDLE_ENFORCE_NOT_NULL(exceptions_[i]); +# PADDLE_ENFORCE_EQ(status, Status::kException); +# PADDLE_ENFORCE_EQ(status, Status::kSuccess); + +. ./count_enforce_by_file.sh --source-only + +ROOT_DIR=../paddle/fluid/operators + +if [ "$1" != "" ]; then + ROOT_DIR=$1 +fi + +function enforce_grep(){ + paddle_check=`grep -zoE "(PADDLE_ENFORCE[A-Z_]{0,9}|PADDLE_THROW)\(.[^,\);]*.[^;]*\);\s" $1 || true` + valid_check=`echo "$paddle_check" | grep -zoE '(PADDLE_ENFORCE[A-Z_]{0,9}|PADDLE_THROW)\((.[^,;]+,)*.[^";]*(errors::).[^"]*".[^";]{20,}.[^;]*\);\s' || true` + invalid_check=`echo "$paddle_check" | grep -vxF "$valid_check" || true` + if [ "${invalid_check}" != "" ];then + file_path=$1 + echo -e "\n- ${file_path#../}" + echo "${invalid_check}" + fi +} + +function grep_file_recursively(){ + local i=0 + local dir_array + for file in `ls $1` + do + if [ -f $1"/"$file ];then + in_white_list=$(echo $FILE_WHITE_LIST | grep "${file}") + if [[ "$in_white_list" == "" ]];then + enforce_grep $1"/"$file + fi + fi + if [ -d $1"/"$file ];then + dir_array[$i]=$1"/"$file + ((i++)) + fi + done + for sub_dir_name in ${dir_array[@]} + do + grep_file_recursively $sub_dir_name + done +} + +function grep_file(){ + file_path=$1 + file_name=`echo ${file_path##*/} ` + if [ -f $file_path ];then + in_white_list=$(echo $FILE_WHITE_LIST | grep "${file_name}") + if [[ "$in_white_list" == "" ]];then + enforce_grep $file_path + fi + fi +} + +main() { + if [ -f $ROOT_DIR ];then + grep_file $ROOT_DIR + else + grep_file_recursively $ROOT_DIR + fi +} + +if [ "${1}" != "--source-only" ]; then + main "${@}" +fi -- GitLab