Add grep enforce shell & polish other enforce count shell (#24897)

* add grep enforce shell, test=develop, test=document_fix * add copyright desc, test=develop, test=document_fix * refactor four scripts, test=develop, test=document_fix * change white list var, test=develop, test=document_fix * add args for count_enforce_by_file, test=develop, test=document_fix

Add grep enforce shell & polish other enforce count shell (#24897)
* add grep enforce shell, test=develop, test=document_fix * add copyright desc, test=develop, test=document_fix * refactor four scripts, test=develop, test=document_fix * change white list var, test=develop, test=document_fix * add args for count_enforce_by_file, test=develop, test=document_fix
8f416eb1 · Chen Weihang · GitHub · b174b997 · 8f416eb1 · 8f416eb1
4 changed file
--- a/tools/count_invalid_enforce.sh
+++ b/tools/count_invalid_enforce.sh
 #!/bin/bash
+# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
 # This script is used to count all PADDLE checks in the paddle/fluid directory,
 #   including the total PADDLE check number, the valid check number and the
 #   invalid check number under paddle/fluid and its main sub-directories.
-# Usage: bash count_invalid_enforce.sh (run in tools directory)
+# Usage: bash count_all_enforce.sh (run in tools directory)
 # Result Example:
@@ -29,7 +43,7 @@ ROOT_DIR=../paddle/fluid
 ALL_PADDLE_CHECK_CNT=0
 VALID_PADDLE_CHECK_CNT=0
-function enforce_scan(){
+function enforce_count(){
    paddle_check=`grep -r -zoE "(PADDLE_ENFORCE[A-Z_]{0,9}|PADDLE_THROW)\(.[^,\);]*.[^;]*\);\s" $1 || true`
    total_check_cnt=`echo "$paddle_check" | grep -cE "(PADDLE_ENFORCE|PADDLE_THROW)" || true`
    valid_check_cnt=`echo "$paddle_check" | grep -zoE '(PADDLE_ENFORCE[A-Z_]{0,9}|PADDLE_THROW)\((.[^,;]+,)*.[^";]*(errors::).[^"]*".[^";]{20,}.[^;]*\);\s' | grep -cE "(PADDLE_ENFORCE|PADDLE_THROW)" || true`
@@ -37,28 +51,34 @@ function enforce_scan(){
    eval $3=$valid_check_cnt
 }
-function walk_dir(){
+function count_dir_recursively(){
    for file in `ls $1`
    do
        if [ -d $1"/"$file ];then
            level=$(($2+1))
            if [ $level -le 1 ]; then
-                enforce_scan $1"/"$file total_check_cnt valid_check_cnt
+                enforce_count $1"/"$file total_check_cnt valid_check_cnt
                dir_name=$1
                echo "${dir_name#../}/"$file" | ${total_check_cnt} | ${valid_check_cnt} | $(($total_check_cnt-$valid_check_cnt))"
                ALL_PADDLE_CHECK_CNT=$(($ALL_PADDLE_CHECK_CNT+$total_check_cnt))
                VALID_PADDLE_CHECK_CNT=$(($VALID_PADDLE_CHECK_CNT+$valid_check_cnt))
-                walk_dir $1"/"$file $level
+                count_dir_recursively $1"/"$file $level
            fi
        fi
    done
 }
-walk_dir $ROOT_DIR 0
+main() {
+    count_dir_recursively $ROOT_DIR 0
+    echo "----------------------------"
+    echo "PADDLE ENFORCE & THROW COUNT"
+    echo "----------------------------"
+    echo "All PADDLE_ENFORCE{_**} & PADDLE_THROW Count: ${ALL_PADDLE_CHECK_CNT}"
+    echo "Valid PADDLE_ENFORCE{_**} & PADDLE_THROW Count: ${VALID_PADDLE_CHECK_CNT}"
+    echo "Invalid PADDLE_ENFORCE{_**} & PADDLE_THROW Count: $(($ALL_PADDLE_CHECK_CNT-$VALID_PADDLE_CHECK_CNT))"
+}
-echo "----------------------------"
+if [ "${1}" != "--source-only" ]; then
-echo "PADDLE ENFORCE & THROW COUNT"
+    main "${@}"
-echo "----------------------------"
+fi
-echo "All PADDLE_ENFORCE{_**} & PADDLE_THROW Count: ${ALL_PADDLE_CHECK_CNT}"
-echo "Valid PADDLE_ENFORCE{_**} & PADDLE_THROW Count: ${VALID_PADDLE_CHECK_CNT}"
-echo "Invalid PADDLE_ENFORCE{_**} & PADDLE_THROW Count: $(($ALL_PADDLE_CHECK_CNT-$VALID_PADDLE_CHECK_CNT))"
--- a/tools/detail_invalid_enforce.sh
+++ b/tools/detail_invalid_enforce.sh
 #!/bin/bash
+# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
 # This script is used to count detail PADDLE checks in the paddle/fluid directory,
 #   contains the number of PADDLE checks under each folder, the statistical data 
 #   does not include subdirectories, only covers all files under the current directory.
@@ -7,7 +21,7 @@
 #   The three columns of data are: total number, valid number, invalid number. 
 #   The output format is easy to display as a markdown table.
-# Usage: bash detail_invalid_enforce.sh (run in tools directory)
+# Usage: bash count_enforce_by_dir.sh (run in tools directory)
 # Result Example:
@@ -43,27 +57,21 @@
 #     paddle/fluid/operators/tensorrt | 7 | 4 | 3
 #     paddle/fluid/operators | 2144 | 999 | 1145
-ROOT_DIR=../paddle/fluid
+. ./count_all_enforce.sh --source-only
-function enforce_scan(){
+ROOT_DIR=../paddle/fluid
-    paddle_check=`grep -r -zoE "(PADDLE_ENFORCE[A-Z_]{0,9}|PADDLE_THROW)\(.[^,\);]*.[^;]*\);\s" $1 || true`
-    total_check_cnt=`echo "$paddle_check" | grep -cE "(PADDLE_ENFORCE|PADDLE_THROW)" || true`
-    valid_check_cnt=`echo "$paddle_check" | grep -zoE '(PADDLE_ENFORCE[A-Z_]{0,9}|PADDLE_THROW)\((.[^,;]+,)*.[^";]*(errors::).[^"]*".[^";]{20,}.[^;]*\);\s' | grep -cE "(PADDLE_ENFORCE|PADDLE_THROW)" || true`
-    eval $2=$total_check_cnt
-    eval $3=$valid_check_cnt
-}
-function walk_dir(){
+function count_dir_independently(){
    local sub_dir_total_check_cnt=0
    local sub_dir_valid_check_cnt=0
    for file in `ls $1`
    do
        if [ -d $1"/"$file ];then
-            enforce_scan $1"/"$file dir_total_check_cnt dir_valid_check_cnt
+            enforce_count $1"/"$file dir_total_check_cnt dir_valid_check_cnt
            sub_dir_total_check_cnt=$(($sub_dir_total_check_cnt+$dir_total_check_cnt))
            sub_dir_valid_check_cnt=$(($sub_dir_valid_check_cnt+$dir_valid_check_cnt))
-            walk_dir $1"/"$file $dir_total_check_cnt $dir_valid_check_cnt 
+            count_dir_independently $1"/"$file $dir_total_check_cnt $dir_valid_check_cnt 
        fi
    done
    total_check_cnt=$(($2-$sub_dir_total_check_cnt))
@@ -72,4 +80,10 @@ function walk_dir(){
    echo "${dir_name#../} | ${total_check_cnt} | ${valid_check_cnt} | $(($total_check_cnt-$valid_check_cnt))"
 }
-walk_dir $ROOT_DIR 0 0
+main() {
+    count_dir_independently $ROOT_DIR 0 0
+}
+if [ "${1}" != "--source-only" ]; then
+    main "${@}"
+fi
--- a/tools/file_invalid_enforce.sh
+++ b/tools/file_invalid_enforce.sh
 #!/bin/bash
+# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
 # This script is used to count PADDLE checks by files in the paddle/fluid/operators directory,
 #   contains the number of PADDLE checks under each file.
 #   
 #   The three columns of data are: total number, valid number, invalid number. 
 #   The output format is easy to display as a markdown table.
-# Usage: bash file_invalid_enforce.sh (run in tools directory)
+# Usage: bash count_enforce_by_file.sh  [target directory or file] (run in tools directory)
+#   - The default check path is paddle/fluid/operators
 # Result Example:
@@ -27,9 +42,15 @@
 #   - math_function.cu | 4 | 0 | 4
 #   - math_function_impl.h | 10 | 0 | 10
+. ./count_all_enforce.sh --source-only
 ROOT_DIR=../paddle/fluid/operators
-white_list_str = "\
+if [ "$1" != "" ]; then
+    ROOT_DIR=$1
+fi
+FILE_WHITE_LIST="\
    layer_norm_op.cc \
    box_clip_op.cc \
    box_clip_op.h \
@@ -38,15 +59,7 @@ white_list_str = "\
    fused_elemwise_activation_op.cc \
    auc_op.cu"
-function enforce_scan(){
+function count_file_recursively(){
-    paddle_check=`grep -r -zoE "(PADDLE_ENFORCE[A-Z_]{0,9}|PADDLE_THROW)\(.[^,\);]*.[^;]*\);\s" $1 || true`
-    total_check_cnt=`echo "$paddle_check" | grep -cE "(PADDLE_ENFORCE|PADDLE_THROW)" || true`
-    valid_check_cnt=`echo "$paddle_check" | grep -zoE '(PADDLE_ENFORCE[A-Z_]{0,9}|PADDLE_THROW)\((.[^,;]+,)*.[^";]*(errors::).[^"]*".[^";]{20,}.[^;]*\);\s' | grep -cE "(PADDLE_ENFORCE|PADDLE_THROW)" || true`
-    eval $2=$total_check_cnt
-    eval $3=$valid_check_cnt
-}
-function walk_dir(){
    dir_name=$1
    echo "**${dir_name#../}** | **$2** | **$3** | **$(($2-$3))**"
    local i=0
@@ -54,9 +67,9 @@ function walk_dir(){
    for file in `ls $1`
    do
        if [ -f $1"/"$file ];then
-            in_white_list=$(echo $white_list_str | grep "${file}")
+            in_white_list=$(echo $FILE_WHITE_LIST | grep "${file}")
            if [[ "$in_white_list" == "" ]];then
-                enforce_scan $1"/"$file file_total_check_cnt file_valid_check_cnt
+                enforce_count $1"/"$file file_total_check_cnt file_valid_check_cnt
                file_invalid_check_cnt=$(($total_check_cnt-$valid_check_cnt))
                if [ $file_invalid_check_cnt -gt 0 ];then
                    echo "- $file | ${file_total_check_cnt} | ${file_valid_check_cnt} | ${file_invalid_check_cnt}"
@@ -70,9 +83,15 @@ function walk_dir(){
    done
    for sub_dir_name in ${dir_array[@]}
    do
-        enforce_scan $sub_dir_name dir_total_check_cnt dir_valid_check_cnt
+        enforce_count $sub_dir_name dir_total_check_cnt dir_valid_check_cnt
-        walk_dir $sub_dir_name $dir_total_check_cnt $dir_valid_check_cnt
+        count_file_recursively $sub_dir_name $dir_total_check_cnt $dir_valid_check_cnt
    done
 }
-walk_dir $ROOT_DIR 0 0
+main() {
+    count_file_recursively $ROOT_DIR 0 0
+}
+if [ "${1}" != "--source-only" ]; then
+    main "${@}"
+fi
--- a/tools/grep_invalid_enforce.sh
+++ b/tools/grep_invalid_enforce.sh
+#!/bin/bash
+# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# This script is used to grep invalid PADDLE checks by directory or file in the paddle/fluid/,
+#   the result show all invalid PADDLE checks in specified directory or file.
+# Usage: 
+#   - bash grep_invalid_enforce.sh [target directory or file] (run in tools directory)
+#       - The default check path is paddle/fluid/operators
+# Result Examples:
+# 1. grep invalid PADDLE checks in directory
+#     - Command: /work/paddle/tools {develop} bash grep_invalid_enforce.sh ../paddle/fluid/imperative 
+#     - Results:
+#         - paddle/fluid/imperative/gradient_accumulator.cc
+#         PADDLE_ENFORCE_EQ(dst_tensor->numel() == numel, true,
+#                             "dst_numel %d vs. src_numel %d", dst_tensor->numel(),
+#                             numel);
+#         - paddle/fluid/imperative/nccl_context.cc
+#         PADDLE_ENFORCE_EQ(addr.size(), 2UL,
+#                             "The endpoint should contain host and port: %s", ep);
+#         PADDLE_THROW("create server fd failed");
+#         PADDLE_THROW("set socket opt failed");
+#         PADDLE_THROW("binding failed on ep: %s", ep);
+#         PADDLE_THROW("listen on server fd failed");
+#         PADDLE_THROW("accept the new socket fd failed");
+#         PADDLE_THROW("reading the ncclUniqueId from socket failed");
+#         PADDLE_ENFORCE_EQ(addr.size(), 2UL,
+#                             "The endpoint should contain host and port: %s", ep);
+#         PADDLE_THROW("create socket failed");
+#         PADDLE_THROW("invalied address: %s", ep);
+#         - paddle/fluid/imperative/jit/program_desc_tracer.cc
+#         PADDLE_ENFORCE_NOT_NULL(new_var);
+#         PADDLE_ENFORCE_EQ(inner_var.IsInitialized(), true);
+#         PADDLE_THROW("Not support variable type %s",
+#                         framework::ToTypeName(inner_var.Type()));
+# 2. grep invalid PADDLE checks in file
+#     - Command: /work/paddle/tools {develop} bash grep_invalid_enforce.sh ../paddle/fluid/pybind/reader_py.cc
+#     - Results:
+#         - paddle/fluid/pybind/reader_py.cc
+#         PADDLE_THROW(
+#                     "Place cannot be CUDAPlace when use_double_buffer is False");
+#         PADDLE_ENFORCE_NOT_NULL(exceptions_[i]);
+#         PADDLE_ENFORCE_EQ(status, Status::kException);
+#         PADDLE_ENFORCE_EQ(status, Status::kSuccess);    
+. ./count_enforce_by_file.sh --source-only
+ROOT_DIR=../paddle/fluid/operators
+if [ "$1" != "" ]; then
+    ROOT_DIR=$1
+fi
+function enforce_grep(){
+    paddle_check=`grep -zoE "(PADDLE_ENFORCE[A-Z_]{0,9}|PADDLE_THROW)\(.[^,\);]*.[^;]*\);\s" $1 || true`
+    valid_check=`echo "$paddle_check" | grep -zoE '(PADDLE_ENFORCE[A-Z_]{0,9}|PADDLE_THROW)\((.[^,;]+,)*.[^";]*(errors::).[^"]*".[^";]{20,}.[^;]*\);\s' || true`
+    invalid_check=`echo "$paddle_check" | grep -vxF "$valid_check" || true`
+    if [ "${invalid_check}" != "" ];then
+        file_path=$1
+        echo -e "\n- ${file_path#../}"
+        echo "${invalid_check}"
+    fi
+}
+function grep_file_recursively(){
+    local i=0
+    local dir_array
+    for file in `ls $1`
+    do
+        if [ -f $1"/"$file ];then
+            in_white_list=$(echo $FILE_WHITE_LIST | grep "${file}")
+            if [[ "$in_white_list" == "" ]];then
+                enforce_grep $1"/"$file
+            fi
+        fi
+        if [ -d $1"/"$file ];then
+            dir_array[$i]=$1"/"$file
+            ((i++))
+        fi
+    done
+    for sub_dir_name in ${dir_array[@]}
+    do
+        grep_file_recursively $sub_dir_name
+    done
+}
+function grep_file(){
+    file_path=$1
+    file_name=`echo ${file_path##*/} `
+    if [ -f $file_path ];then
+        in_white_list=$(echo $FILE_WHITE_LIST | grep "${file_name}")
+        if [[ "$in_white_list" == "" ]];then
+            enforce_grep $file_path
+        fi
+    fi
+}
+main() {
+    if [ -f $ROOT_DIR ];then
+        grep_file $ROOT_DIR
+    else
+        grep_file_recursively $ROOT_DIR
+    fi
+}
+if [ "${1}" != "--source-only" ]; then
+    main "${@}"
+fi