diff --git a/tools/count_invalid_enforce.sh b/tools/count_invalid_enforce.sh index fe99674f6bec6b06437f8383bb1289b48c239243..927d96e9a08d7f95973285fe6d0b49143963a88a 100644 --- a/tools/count_invalid_enforce.sh +++ b/tools/count_invalid_enforce.sh @@ -8,16 +8,16 @@ # Result Example: -# ../paddle/fluid/framework - total: 1065, valid: 267, invalid: 798 -# ../paddle/fluid/imperative - total: 135, valid: 118, invalid: 17 -# ../paddle/fluid/inference - total: 449, valid: 158, invalid: 291 -# ../paddle/fluid/memory - total: 60, valid: 10, invalid: 50 -# ../paddle/fluid/operators - total: 4225, valid: 1061, invalid: 3164 -# ../paddle/fluid/platform - total: 240, valid: 39, invalid: 201 -# ../paddle/fluid/pybind - total: 98, valid: 53, invalid: 45 -# ../paddle/fluid/string - total: 0, valid: 0, invalid: 0 -# ../paddle/fluid/testdata - total: 0, valid: 0, invalid: 0 -# ../paddle/fluid/train - total: 6, valid: 0, invalid: 6 +# paddle/fluid/framework - total: 1065, valid: 267, invalid: 798 +# paddle/fluid/imperative - total: 135, valid: 118, invalid: 17 +# paddle/fluid/inference - total: 449, valid: 158, invalid: 291 +# paddle/fluid/memory - total: 60, valid: 10, invalid: 50 +# paddle/fluid/operators - total: 4225, valid: 1061, invalid: 3164 +# paddle/fluid/platform - total: 240, valid: 39, invalid: 201 +# paddle/fluid/pybind - total: 98, valid: 53, invalid: 45 +# paddle/fluid/string - total: 0, valid: 0, invalid: 0 +# paddle/fluid/testdata - total: 0, valid: 0, invalid: 0 +# paddle/fluid/train - total: 6, valid: 0, invalid: 6 # ---------------------------- # PADDLE ENFORCE & THROW COUNT # ---------------------------- @@ -44,7 +44,8 @@ function walk_dir(){ level=$(($2+1)) if [ $level -le 1 ]; then enforce_scan $1"/"$file total_check_cnt valid_check_cnt - echo $1"/"$file" - total: ${total_check_cnt}, valid: ${valid_check_cnt}, invalid: $(($total_check_cnt-$valid_check_cnt))" + dir_name=$1 + echo "${dir_name#../}"/"$file - total: ${total_check_cnt}, valid: ${valid_check_cnt}, invalid: $(($total_check_cnt-$valid_check_cnt))" ALL_PADDLE_CHECK_CNT=$(($ALL_PADDLE_CHECK_CNT+$total_check_cnt)) VALID_PADDLE_CHECK_CNT=$(($VALID_PADDLE_CHECK_CNT+$valid_check_cnt)) walk_dir $1"/"$file $level diff --git a/tools/detail_invalid_enforce.sh b/tools/detail_invalid_enforce.sh new file mode 100644 index 0000000000000000000000000000000000000000..7424107e23707339981e81a1145a8be4bf766d4d --- /dev/null +++ b/tools/detail_invalid_enforce.sh @@ -0,0 +1,75 @@ +#!/bin/bash + +# This script is used to count detail PADDLE checks in the paddle/fluid directory, +# contains the number of PADDLE checks under each folder, the statistical data +# does not include subdirectories, only covers all files under the current directory. +# +# The three columns of data are: total number, valid number, invalid number. +# The output format is easy to display as a markdown table. + +# Usage: bash detail_invalid_enforce.sh (run in tools directory) + +# Result Example: + +# paddle/fluid/operators/benchmark | 11 | 0 | 11 +# paddle/fluid/operators/collective | 28 | 1 | 27 +# paddle/fluid/operators/controlflow | 60 | 59 | 1 +# paddle/fluid/operators/detail | 2 | 0 | 2 +# paddle/fluid/operators/detection | 276 | 146 | 130 +# paddle/fluid/operators/distributed/brpc | 17 | 0 | 17 +# paddle/fluid/operators/distributed/grpc | 13 | 0 | 13 +# paddle/fluid/operators/distributed | 63 | 10 | 53 +# paddle/fluid/operators/distributed_ops | 88 | 6 | 82 +# paddle/fluid/operators/elementwise/mkldnn | 5 | 5 | 0 +# paddle/fluid/operators/elementwise | 29 | 20 | 9 +# paddle/fluid/operators/fused | 227 | 182 | 45 +# paddle/fluid/operators/jit/gen | 17 | 0 | 17 +# paddle/fluid/operators/jit/more/intrinsic | 0 | 0 | 0 +# paddle/fluid/operators/jit/more/mix | 1 | 0 | 1 +# paddle/fluid/operators/jit/more/mkl | 9 | 0 | 9 +# paddle/fluid/operators/jit/more | 0 | 0 | 0 +# paddle/fluid/operators/jit/refer | 8 | 0 | 8 +# paddle/fluid/operators/jit | 18 | 0 | 18 +# paddle/fluid/operators/lite | 2 | 2 | 0 +# paddle/fluid/operators/math/detail | 0 | 0 | 0 +# paddle/fluid/operators/math | 200 | 7 | 193 +# paddle/fluid/operators/metrics | 38 | 29 | 9 +# paddle/fluid/operators/mkldnn | 107 | 14 | 93 +# paddle/fluid/operators/nccl | 27 | 0 | 27 +# paddle/fluid/operators/optimizers | 214 | 50 | 164 +# paddle/fluid/operators/reader | 40 | 14 | 26 +# paddle/fluid/operators/reduce_ops | 8 | 8 | 0 +# paddle/fluid/operators/sequence_ops | 167 | 47 | 120 +# paddle/fluid/operators/tensorrt | 7 | 4 | 3 +# paddle/fluid/operators | 2144 | 999 | 1145 + +ROOT_DIR=../paddle/fluid + +function enforce_scan(){ + paddle_check=`grep -r -zoE "(PADDLE_ENFORCE[A-Z_]{0,9}|PADDLE_THROW)\(.[^,\);]*.[^;]*\);\s" $1 || true` + total_check_cnt=`echo "$paddle_check" | grep -cE "(PADDLE_ENFORCE|PADDLE_THROW)" || true` + valid_check_cnt=`echo "$paddle_check" | grep -zoE '(PADDLE_ENFORCE[A-Z_]{0,9}|PADDLE_THROW)\((.[^,;]+,)*.[^";]*(errors::).[^"]*".[^";]{20,}.[^;]*\);\s' | grep -cE "(PADDLE_ENFORCE|PADDLE_THROW)" || true` + eval $2=$total_check_cnt + eval $3=$valid_check_cnt +} + +function walk_dir(){ + local sub_dir_total_check_cnt=0 + local sub_dir_valid_check_cnt=0 + for file in `ls $1` + do + if [ -d $1"/"$file ];then + enforce_scan $1"/"$file dir_total_check_cnt dir_valid_check_cnt + sub_dir_total_check_cnt=$(($sub_dir_total_check_cnt+$dir_total_check_cnt)) + sub_dir_valid_check_cnt=$(($sub_dir_valid_check_cnt+$dir_valid_check_cnt)) + + walk_dir $1"/"$file $dir_total_check_cnt $dir_valid_check_cnt + fi + done + total_check_cnt=$(($2-$sub_dir_total_check_cnt)) + valid_check_cnt=$(($3-$sub_dir_valid_check_cnt)) + dir_name=$1 + echo "${dir_name#../} | ${total_check_cnt} | ${valid_check_cnt} | $(($total_check_cnt-$valid_check_cnt))" +} + +walk_dir $ROOT_DIR 0 0 diff --git a/tools/file_invalid_enforce.sh b/tools/file_invalid_enforce.sh new file mode 100644 index 0000000000000000000000000000000000000000..f2c1630a1d43e4ee14b73904d535e7b475f6528e --- /dev/null +++ b/tools/file_invalid_enforce.sh @@ -0,0 +1,67 @@ +#!/bin/bash + +# This script is used to count PADDLE checks by files in the paddle/fluid/operators directory, +# contains the number of PADDLE checks under each file. +# +# The three columns of data are: total number, valid number, invalid number. +# The output format is easy to display as a markdown table. + +# Usage: bash file_invalid_enforce.sh (run in tools directory) + +# Result Example: + +# **paddle/fluid/operators/math** | **200** | **7** | **193** +# - beam_search.cc | 1 | 0 | 1 +# - beam_search.cu | 1 | 0 | 1 +# - blas.cc | 1 | 0 | 1 +# - blas_impl.cu.h | 8 | 1 | 7 +# - blas_impl.h | 15 | 0 | 15 +# - concat_test.cc | 16 | 0 | 16 +# - context_project.h | 1 | 0 | 1 +# - cpu_vec.h | 1 | 0 | 1 +# - cross_entropy.cu | 1 | 0 | 1 +# - cross_entropy.h | 1 | 0 | 1 +# - im2col.cc | 12 | 0 | 12 +# - im2col.cu | 12 | 0 | 12 +# - math_function.cc | 2 | 0 | 2 +# - math_function.cu | 4 | 0 | 4 +# - math_function_impl.h | 10 | 0 | 10 + +ROOT_DIR=../paddle/fluid/operators + +function enforce_scan(){ + paddle_check=`grep -r -zoE "(PADDLE_ENFORCE[A-Z_]{0,9}|PADDLE_THROW)\(.[^,\);]*.[^;]*\);\s" $1 || true` + total_check_cnt=`echo "$paddle_check" | grep -cE "(PADDLE_ENFORCE|PADDLE_THROW)" || true` + valid_check_cnt=`echo "$paddle_check" | grep -zoE '(PADDLE_ENFORCE[A-Z_]{0,9}|PADDLE_THROW)\((.[^,;]+,)*.[^";]*(errors::).[^"]*".[^";]{20,}.[^;]*\);\s' | grep -cE "(PADDLE_ENFORCE|PADDLE_THROW)" || true` + eval $2=$total_check_cnt + eval $3=$valid_check_cnt +} + +function walk_dir(){ + dir_name=$1 + echo "**${dir_name#../}** | **$2** | **$3** | **$(($2-$3))**" + local i=0 + local dir_array + for file in `ls $1` + do + if [ -f $1"/"$file ];then + enforce_scan $1"/"$file file_total_check_cnt file_valid_check_cnt + file_invalid_check_cnt=$(($total_check_cnt-$valid_check_cnt)) + if [ $file_invalid_check_cnt -gt 0 ];then + echo "- $file | ${file_total_check_cnt} | ${file_valid_check_cnt} | ${file_invalid_check_cnt}" + fi + fi + if [ -d $1"/"$file ];then + + dir_array[$i]=$1"/"$file + ((i++)) + fi + done + for sub_dir_name in ${dir_array[@]} + do + enforce_scan $sub_dir_name dir_total_check_cnt dir_valid_check_cnt + walk_dir $sub_dir_name $dir_total_check_cnt $dir_valid_check_cnt + done +} + +walk_dir $ROOT_DIR 0 0