From 7728a20175cb8aa10d8279393b42307131f8a383 Mon Sep 17 00:00:00 2001
From: Chen Weihang <chenweihang@baidu.com>
Date: Sat, 9 May 2020 12:48:46 +0800
Subject: [PATCH] add detail & file enforce count shell, test=develop,
 test=document_fix (#24361)

---
 tools/count_invalid_enforce.sh  | 23 +++++-----
 tools/detail_invalid_enforce.sh | 75 +++++++++++++++++++++++++++++++++
 tools/file_invalid_enforce.sh   | 67 +++++++++++++++++++++++++++++
 3 files changed, 154 insertions(+), 11 deletions(-)
 create mode 100644 tools/detail_invalid_enforce.sh
 create mode 100644 tools/file_invalid_enforce.sh

diff --git a/tools/count_invalid_enforce.sh b/tools/count_invalid_enforce.sh
index fe99674f6be..927d96e9a08 100644
--- a/tools/count_invalid_enforce.sh
+++ b/tools/count_invalid_enforce.sh
@@ -8,16 +8,16 @@
 
 # Result Example:
 
-#     ../paddle/fluid/framework - total: 1065, valid: 267, invalid: 798
-#     ../paddle/fluid/imperative - total: 135, valid: 118, invalid: 17
-#     ../paddle/fluid/inference - total: 449, valid: 158, invalid: 291
-#     ../paddle/fluid/memory - total: 60, valid: 10, invalid: 50
-#     ../paddle/fluid/operators - total: 4225, valid: 1061, invalid: 3164
-#     ../paddle/fluid/platform - total: 240, valid: 39, invalid: 201
-#     ../paddle/fluid/pybind - total: 98, valid: 53, invalid: 45
-#     ../paddle/fluid/string - total: 0, valid: 0, invalid: 0
-#     ../paddle/fluid/testdata - total: 0, valid: 0, invalid: 0
-#     ../paddle/fluid/train - total: 6, valid: 0, invalid: 6
+#     paddle/fluid/framework - total: 1065, valid: 267, invalid: 798
+#     paddle/fluid/imperative - total: 135, valid: 118, invalid: 17
+#     paddle/fluid/inference - total: 449, valid: 158, invalid: 291
+#     paddle/fluid/memory - total: 60, valid: 10, invalid: 50
+#     paddle/fluid/operators - total: 4225, valid: 1061, invalid: 3164
+#     paddle/fluid/platform - total: 240, valid: 39, invalid: 201
+#     paddle/fluid/pybind - total: 98, valid: 53, invalid: 45
+#     paddle/fluid/string - total: 0, valid: 0, invalid: 0
+#     paddle/fluid/testdata - total: 0, valid: 0, invalid: 0
+#     paddle/fluid/train - total: 6, valid: 0, invalid: 6
 #     ----------------------------
 #     PADDLE ENFORCE & THROW COUNT
 #     ----------------------------
@@ -44,7 +44,8 @@ function walk_dir(){
             level=$(($2+1))
             if [ $level -le 1 ]; then
                 enforce_scan $1"/"$file total_check_cnt valid_check_cnt
-                echo $1"/"$file" - total: ${total_check_cnt}, valid: ${valid_check_cnt}, invalid: $(($total_check_cnt-$valid_check_cnt))"
+                dir_name=$1
+                echo "${dir_name#../}"/"$file - total: ${total_check_cnt}, valid: ${valid_check_cnt}, invalid: $(($total_check_cnt-$valid_check_cnt))"
                 ALL_PADDLE_CHECK_CNT=$(($ALL_PADDLE_CHECK_CNT+$total_check_cnt))
                 VALID_PADDLE_CHECK_CNT=$(($VALID_PADDLE_CHECK_CNT+$valid_check_cnt))
                 walk_dir $1"/"$file $level
diff --git a/tools/detail_invalid_enforce.sh b/tools/detail_invalid_enforce.sh
new file mode 100644
index 00000000000..7424107e237
--- /dev/null
+++ b/tools/detail_invalid_enforce.sh
@@ -0,0 +1,75 @@
+#!/bin/bash
+
+# This script is used to count detail PADDLE checks in the paddle/fluid directory,
+#   contains the number of PADDLE checks under each folder, the statistical data 
+#   does not include subdirectories, only covers all files under the current directory.
+#   
+#   The three columns of data are: total number, valid number, invalid number. 
+#   The output format is easy to display as a markdown table.
+
+# Usage: bash detail_invalid_enforce.sh (run in tools directory)
+
+# Result Example:
+
+#     paddle/fluid/operators/benchmark | 11 | 0 | 11
+#     paddle/fluid/operators/collective | 28 | 1 | 27
+#     paddle/fluid/operators/controlflow | 60 | 59 | 1
+#     paddle/fluid/operators/detail | 2 | 0 | 2
+#     paddle/fluid/operators/detection | 276 | 146 | 130
+#     paddle/fluid/operators/distributed/brpc | 17 | 0 | 17
+#     paddle/fluid/operators/distributed/grpc | 13 | 0 | 13
+#     paddle/fluid/operators/distributed | 63 | 10 | 53
+#     paddle/fluid/operators/distributed_ops | 88 | 6 | 82
+#     paddle/fluid/operators/elementwise/mkldnn | 5 | 5 | 0
+#     paddle/fluid/operators/elementwise | 29 | 20 | 9
+#     paddle/fluid/operators/fused | 227 | 182 | 45
+#     paddle/fluid/operators/jit/gen | 17 | 0 | 17
+#     paddle/fluid/operators/jit/more/intrinsic | 0 | 0 | 0
+#     paddle/fluid/operators/jit/more/mix | 1 | 0 | 1
+#     paddle/fluid/operators/jit/more/mkl | 9 | 0 | 9
+#     paddle/fluid/operators/jit/more | 0 | 0 | 0
+#     paddle/fluid/operators/jit/refer | 8 | 0 | 8
+#     paddle/fluid/operators/jit | 18 | 0 | 18
+#     paddle/fluid/operators/lite | 2 | 2 | 0
+#     paddle/fluid/operators/math/detail | 0 | 0 | 0
+#     paddle/fluid/operators/math | 200 | 7 | 193
+#     paddle/fluid/operators/metrics | 38 | 29 | 9
+#     paddle/fluid/operators/mkldnn | 107 | 14 | 93
+#     paddle/fluid/operators/nccl | 27 | 0 | 27
+#     paddle/fluid/operators/optimizers | 214 | 50 | 164
+#     paddle/fluid/operators/reader | 40 | 14 | 26
+#     paddle/fluid/operators/reduce_ops | 8 | 8 | 0
+#     paddle/fluid/operators/sequence_ops | 167 | 47 | 120
+#     paddle/fluid/operators/tensorrt | 7 | 4 | 3
+#     paddle/fluid/operators | 2144 | 999 | 1145
+
+ROOT_DIR=../paddle/fluid
+
+function enforce_scan(){
+    paddle_check=`grep -r -zoE "(PADDLE_ENFORCE[A-Z_]{0,9}|PADDLE_THROW)\(.[^,\);]*.[^;]*\);\s" $1 || true`
+    total_check_cnt=`echo "$paddle_check" | grep -cE "(PADDLE_ENFORCE|PADDLE_THROW)" || true`
+    valid_check_cnt=`echo "$paddle_check" | grep -zoE '(PADDLE_ENFORCE[A-Z_]{0,9}|PADDLE_THROW)\((.[^,;]+,)*.[^";]*(errors::).[^"]*".[^";]{20,}.[^;]*\);\s' | grep -cE "(PADDLE_ENFORCE|PADDLE_THROW)" || true`
+    eval $2=$total_check_cnt
+    eval $3=$valid_check_cnt
+}
+
+function walk_dir(){
+    local sub_dir_total_check_cnt=0
+    local sub_dir_valid_check_cnt=0
+    for file in `ls $1`
+    do
+        if [ -d $1"/"$file ];then
+            enforce_scan $1"/"$file dir_total_check_cnt dir_valid_check_cnt
+            sub_dir_total_check_cnt=$(($sub_dir_total_check_cnt+$dir_total_check_cnt))
+            sub_dir_valid_check_cnt=$(($sub_dir_valid_check_cnt+$dir_valid_check_cnt))
+            
+            walk_dir $1"/"$file $dir_total_check_cnt $dir_valid_check_cnt 
+        fi
+    done
+    total_check_cnt=$(($2-$sub_dir_total_check_cnt))
+    valid_check_cnt=$(($3-$sub_dir_valid_check_cnt))
+    dir_name=$1
+    echo "${dir_name#../} | ${total_check_cnt} | ${valid_check_cnt} | $(($total_check_cnt-$valid_check_cnt))"
+}
+
+walk_dir $ROOT_DIR 0 0
diff --git a/tools/file_invalid_enforce.sh b/tools/file_invalid_enforce.sh
new file mode 100644
index 00000000000..f2c1630a1d4
--- /dev/null
+++ b/tools/file_invalid_enforce.sh
@@ -0,0 +1,67 @@
+#!/bin/bash
+
+# This script is used to count PADDLE checks by files in the paddle/fluid/operators directory,
+#   contains the number of PADDLE checks under each file.
+#   
+#   The three columns of data are: total number, valid number, invalid number. 
+#   The output format is easy to display as a markdown table.
+
+# Usage: bash file_invalid_enforce.sh (run in tools directory)
+
+# Result Example:
+
+#   **paddle/fluid/operators/math** | **200** | **7** | **193**
+#   - beam_search.cc | 1 | 0 | 1
+#   - beam_search.cu | 1 | 0 | 1
+#   - blas.cc | 1 | 0 | 1
+#   - blas_impl.cu.h | 8 | 1 | 7
+#   - blas_impl.h | 15 | 0 | 15
+#   - concat_test.cc | 16 | 0 | 16
+#   - context_project.h | 1 | 0 | 1
+#   - cpu_vec.h | 1 | 0 | 1
+#   - cross_entropy.cu | 1 | 0 | 1
+#   - cross_entropy.h | 1 | 0 | 1
+#   - im2col.cc | 12 | 0 | 12
+#   - im2col.cu | 12 | 0 | 12
+#   - math_function.cc | 2 | 0 | 2
+#   - math_function.cu | 4 | 0 | 4
+#   - math_function_impl.h | 10 | 0 | 10
+
+ROOT_DIR=../paddle/fluid/operators
+
+function enforce_scan(){
+    paddle_check=`grep -r -zoE "(PADDLE_ENFORCE[A-Z_]{0,9}|PADDLE_THROW)\(.[^,\);]*.[^;]*\);\s" $1 || true`
+    total_check_cnt=`echo "$paddle_check" | grep -cE "(PADDLE_ENFORCE|PADDLE_THROW)" || true`
+    valid_check_cnt=`echo "$paddle_check" | grep -zoE '(PADDLE_ENFORCE[A-Z_]{0,9}|PADDLE_THROW)\((.[^,;]+,)*.[^";]*(errors::).[^"]*".[^";]{20,}.[^;]*\);\s' | grep -cE "(PADDLE_ENFORCE|PADDLE_THROW)" || true`
+    eval $2=$total_check_cnt
+    eval $3=$valid_check_cnt
+}
+
+function walk_dir(){
+    dir_name=$1
+    echo "**${dir_name#../}** | **$2** | **$3** | **$(($2-$3))**"
+    local i=0
+    local dir_array
+    for file in `ls $1`
+    do
+        if [ -f $1"/"$file ];then
+            enforce_scan $1"/"$file file_total_check_cnt file_valid_check_cnt
+            file_invalid_check_cnt=$(($total_check_cnt-$valid_check_cnt))
+            if [ $file_invalid_check_cnt -gt 0 ];then
+                echo "- $file | ${file_total_check_cnt} | ${file_valid_check_cnt} | ${file_invalid_check_cnt}"
+            fi
+        fi
+        if [ -d $1"/"$file ];then
+            
+            dir_array[$i]=$1"/"$file
+            ((i++))
+        fi
+    done
+    for sub_dir_name in ${dir_array[@]}
+    do
+        enforce_scan $sub_dir_name dir_total_check_cnt dir_valid_check_cnt
+        walk_dir $sub_dir_name $dir_total_check_cnt $dir_valid_check_cnt
+    done
+}
+
+walk_dir $ROOT_DIR 0 0
-- 
GitLab