From f28c11262776691f20a93fc8ad797c0a59c8679f Mon Sep 17 00:00:00 2001 From: Chen Weihang Date: Wed, 24 Aug 2022 00:15:24 -0500 Subject: [PATCH] [CI] Add forward and backward kernel dtypes consistent check (#45313) * add kernel dtypes check * polish details * fix print info, test=document_fix * skip repeated kernel, test=document_fix * remove test kernel, test=document_fix --- paddle/scripts/paddle_build.sh | 4 ++++ tools/check_api_approvals.sh | 6 ++++++ tools/check_op_kernel_same_dtypes.py | 16 ++++++++++++++-- 3 files changed, 24 insertions(+), 2 deletions(-) diff --git a/paddle/scripts/paddle_build.sh b/paddle/scripts/paddle_build.sh index 1705c2680cd..4ee05ba079e 100755 --- a/paddle/scripts/paddle_build.sh +++ b/paddle/scripts/paddle_build.sh @@ -1041,6 +1041,10 @@ function generate_api_spec() { op_type_path=${PADDLE_ROOT}/paddle/fluid/OP_TYPE_${spec_kind}.spec python ${PADDLE_ROOT}/tools/check_op_register_type.py > $op_type_path + # used to log op_register data_type + op_type_path=${PADDLE_ROOT}/paddle/fluid/OP_KERNEL_DTYPE_${spec_kind}.spec + python ${PADDLE_ROOT}/tools/check_op_kernel_same_dtypes.py > $op_type_path + # print all ops desc in dict to op_desc_path op_desc_path=${PADDLE_ROOT}/paddle/fluid/OP_DESC_${spec_kind}.spec python ${PADDLE_ROOT}/tools/print_op_desc.py > $op_desc_path diff --git a/tools/check_api_approvals.sh b/tools/check_api_approvals.sh index 1a454f82890..88b641fe219 100644 --- a/tools/check_api_approvals.sh +++ b/tools/check_api_approvals.sh @@ -75,6 +75,12 @@ if [ "$op_type_spec_diff" != "" ]; then check_approval 1 9301846 7913861 fi +op_kernel_dtype_spec_diff=`python ${PADDLE_ROOT}/tools/check_op_kernel_same_dtypes.py ${PADDLE_ROOT}/paddle/fluid/OP_KERNEL_DTYPE_DEV.spec ${PADDLE_ROOT}/paddle/fluid/OP_KERNEL_DTYPE_PR.spec` +if [ "$op_kernel_dtype_spec_diff" != "" ]; then + echo_line="You have added or modified Op Kernel, resulting in inconsistent data types supported by the forward and backward kernels of the same op, such modifications are not allowed in principle. If it is a mismatch, please request one RD (lanxianghit (Recommend) or chenwhql) review and approve. Including the following kernels:\n${op_kernel_dtype_spec_diff}\n" + check_approval 1 47554610 22561442 +fi + op_desc_diff=`python ${PADDLE_ROOT}/tools/check_op_desc.py ${PADDLE_ROOT}/paddle/fluid/OP_DESC_DEV.spec ${PADDLE_ROOT}/paddle/fluid/OP_DESC_PR.spec` inference_approve=`echo "$op_desc_diff" | grep "need inference to review" -` slim_approve=`echo "$op_desc_diff" | grep "need slim to review" -` diff --git a/tools/check_op_kernel_same_dtypes.py b/tools/check_op_kernel_same_dtypes.py index 549420475ff..ac557418a9c 100644 --- a/tools/check_op_kernel_same_dtypes.py +++ b/tools/check_op_kernel_same_dtypes.py @@ -65,6 +65,13 @@ def print_diff(op_type, op_kernel_dtype_set, grad_op_kernel_dtype_set): format(op_type + "_grad", " ".join(lack_dtypes))) +def contain_current_op(op_type, op_info_dict): + if not op_type.endswith("_grad"): + return op_type + "_grad" in op_info_dict + else: + return op_type.rstrip("_grad") in op_info_dict + + def check_change_or_add_op_kernel_dtypes_valid(): origin = read_file(sys.argv[1]) new = read_file(sys.argv[2]) @@ -84,14 +91,19 @@ def check_change_or_add_op_kernel_dtypes_valid(): if op_type in origin_all_kernel_dtype_dict: origin_dtype_set = origin_all_kernel_dtype_dict[op_type] # op kernel changed - if origin_dtype_set != dtype_set: + if origin_dtype_set != dtype_set and not contain_current_op( + op_type, added_or_changed_op_info): added_or_changed_op_info[op_type] = dtype_set else: # do nothing continue else: # op kernel added - added_or_changed_op_info[op_type] = dtype_set + if not contain_current_op(op_type, added_or_changed_op_info): + added_or_changed_op_info[op_type] = dtype_set + else: + # do nothing + continue for op_type, dtype_set in added_or_changed_op_info.items(): # if changed forward op -- GitLab