未验证 提交 25a4dac4 编写于 作者: L Leo Chen 提交者: GitHub

Use allow list instead of white list (#25002)

* use allow list instead of white list, test=develop

* reduce include, test=develop
上级 621b6385
...@@ -12,14 +12,16 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ...@@ -12,14 +12,16 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and See the License for the specific language governing permissions and
limitations under the License. */ limitations under the License. */
#include "paddle/fluid/framework/unused_var_check.h"
#include <gflags/gflags.h> #include <gflags/gflags.h>
#include <glog/logging.h> #include <glog/logging.h>
#include <string> #include <string>
#include <unordered_set> #include <unordered_set>
#include <vector> #include <vector>
#include "paddle/fluid/framework/operator.h" #include "paddle/fluid/framework/operator.h"
#include "paddle/fluid/framework/unused_var_check.h"
#include "paddle/fluid/platform/enforce.h" #include "paddle/fluid/platform/enforce.h"
DEFINE_bool(enable_unused_var_check, false, DEFINE_bool(enable_unused_var_check, false,
...@@ -27,7 +29,7 @@ DEFINE_bool(enable_unused_var_check, false, ...@@ -27,7 +29,7 @@ DEFINE_bool(enable_unused_var_check, false,
"especially for grad operator. It should be in unittest."); "especially for grad operator. It should be in unittest.");
// NOTE(zhiqiu): Currently, there are some operators which involves unused // NOTE(zhiqiu): Currently, there are some operators which involves unused
// inputs and cannot be removed from the white_list below. // inputs and cannot be removed from the allow_list below.
// They can be mainly divided into four categories: // They can be mainly divided into four categories:
// 0: the inputs of which are only used in if branch, or used in cuda kernel but // 0: the inputs of which are only used in if branch, or used in cuda kernel but
// not in cpu kernel; // not in cpu kernel;
...@@ -35,7 +37,7 @@ DEFINE_bool(enable_unused_var_check, false, ...@@ -35,7 +37,7 @@ DEFINE_bool(enable_unused_var_check, false,
// 2: the inputs of which are used in fused operators. // 2: the inputs of which are used in fused operators.
// The category number is presented in the comments after each operator. // The category number is presented in the comments after each operator.
const std::unordered_set<std::string> op_has_unsed_vars_white_list = { const std::unordered_set<std::string> op_with_unsed_vars_allow_list = {
"batch_norm", // 0 "batch_norm", // 0
"batch_norm_grad", // 0 "batch_norm_grad", // 0
"sync_batch_norm", // 0 "sync_batch_norm", // 0
...@@ -74,8 +76,8 @@ void LogVarUsageIfUnusedVarCheckEnabled(const std::string &name) { ...@@ -74,8 +76,8 @@ void LogVarUsageIfUnusedVarCheckEnabled(const std::string &name) {
} }
void CheckUnusedVar(const OperatorBase &op, const Scope &scope) { void CheckUnusedVar(const OperatorBase &op, const Scope &scope) {
// skip op in white list and it should be fixed in the future. // skip op in allow list.
if (op_has_unsed_vars_white_list.count(op.Type()) != 0) { if (op_with_unsed_vars_allow_list.count(op.Type()) != 0) {
return; return;
} }
auto *used_set = GetThreadLocalUsedVarNameSet(); auto *used_set = GetThreadLocalUsedVarNameSet();
...@@ -116,7 +118,7 @@ void CheckUnusedVar(const OperatorBase &op, const Scope &scope) { ...@@ -116,7 +118,7 @@ void CheckUnusedVar(const OperatorBase &op, const Scope &scope) {
"from inputs of the operator; if yes, register " "from inputs of the operator; if yes, register "
"NoNeedBufferVarsInference or add " "NoNeedBufferVarsInference or add "
"the operator to " "the operator to "
"white list in unused_var_check.cc. See more details at " "allow list in unused_var_check.cc. See more details at "
"[https://github.com/PaddlePaddle/Paddle/wiki/" "[https://github.com/PaddlePaddle/Paddle/wiki/"
"OP-Should-Not-Have-Unused-Input]"; "OP-Should-Not-Have-Unused-Input]";
PADDLE_ENFORCE_EQ(unsed_input_var_names.size(), 0, PADDLE_ENFORCE_EQ(unsed_input_var_names.size(), 0,
......
...@@ -19,11 +19,13 @@ limitations under the License. */ ...@@ -19,11 +19,13 @@ limitations under the License. */
#include <string> #include <string>
#include <unordered_set> #include <unordered_set>
#include "paddle/fluid/framework/operator.h"
namespace paddle { namespace paddle {
namespace framework { namespace framework {
class OperatorBase;
class Scope;
std::unordered_set<std::string>* GetThreadLocalUsedVarNameSet(); std::unordered_set<std::string>* GetThreadLocalUsedVarNameSet();
void LogVarUsageIfUnusedVarCheckEnabled(const std::string& name); void LogVarUsageIfUnusedVarCheckEnabled(const std::string& name);
......
...@@ -111,7 +111,7 @@ for API_FILE in ${API_FILES[*]}; do ...@@ -111,7 +111,7 @@ for API_FILE in ${API_FILES[*]}; do
echo_line="You must have one RD (gongweibao or seiriosPlus) approval for the paddle/fluid/operators/distributed/send_recv.proto.in, which manages the environment variables.\n" echo_line="You must have one RD (gongweibao or seiriosPlus) approval for the paddle/fluid/operators/distributed/send_recv.proto.in, which manages the environment variables.\n"
check_approval 1 10721757 5442383 check_approval 1 10721757 5442383
elif [ "${API_FILE}" == "paddle/fluid/framework/unused_var_check.cc" ];then elif [ "${API_FILE}" == "paddle/fluid/framework/unused_var_check.cc" ];then
echo_line="You must have one RD (zhiqiu (Recommend) , sneaxiy or luotao1) approval for the changes of paddle/fluid/framework/unused_var_check.cc, which manages the white list of operators that have unused input variables. Before change the white list, please read the specification [https://github.com/PaddlePaddle/Paddle/wiki/OP-Should-Not-Have-Unused-Input] and try to refine code first. \n" echo_line="You must have one RD (zhiqiu (Recommend) , sneaxiy or luotao1) approval for the changes of paddle/fluid/framework/unused_var_check.cc, which manages the allow list of operators that have unused input variables. Before change the allow list, please read the specification [https://github.com/PaddlePaddle/Paddle/wiki/OP-Should-Not-Have-Unused-Input] and try to refine code first. \n"
check_approval 1 6888866 32832641 6836917 check_approval 1 6888866 32832641 6836917
elif [ "${API_FILE}" == "paddle/fluid/pybind/op_function_generator.cc" ];then elif [ "${API_FILE}" == "paddle/fluid/pybind/op_function_generator.cc" ];then
echo_line="You must have one RD (zhiqiu (Recommend) , phlrain) approval for the changes of paddle/fluid/pybind/op_function_generator.cc, which manages the logic of automatic generating op functions for dygraph. \n" echo_line="You must have one RD (zhiqiu (Recommend) , phlrain) approval for the changes of paddle/fluid/pybind/op_function_generator.cc, which manages the logic of automatic generating op functions for dygraph. \n"
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册