From d19bceb68693fbca1bff6c37eb7f0b19148c84d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E6=98=8E=E5=86=AC?= <78149749+winter-wang@users.noreply.github.com> Date: Mon, 7 Jun 2021 14:14:09 +0800 Subject: [PATCH] pack the @op_name@.pbtxt into library. test=develop (#33322) --- paddle/fluid/framework/CMakeLists.txt | 14 +++++++-- .../ir/op_compat_sensible_pass_tester.cc | 12 ++++++++ paddle/fluid/framework/op_def_api.cc | 30 ++++++++++++------- paddle/fluid/framework/op_def_api.h | 23 ++++++++++++++ paddle/fluid/framework/op_def_api.h.in | 12 -------- 5 files changed, 66 insertions(+), 25 deletions(-) create mode 100644 paddle/fluid/framework/op_def_api.h delete mode 100644 paddle/fluid/framework/op_def_api.h.in diff --git a/paddle/fluid/framework/CMakeLists.txt b/paddle/fluid/framework/CMakeLists.txt index f39c16002dd..c06260b72e6 100644 --- a/paddle/fluid/framework/CMakeLists.txt +++ b/paddle/fluid/framework/CMakeLists.txt @@ -29,10 +29,20 @@ add_subdirectory(io) proto_library(framework_proto SRCS framework.proto) proto_library(op_def_proto SRCS op_def.proto) -set(OP_DEF_FOLDER "${PADDLE_SOURCE_DIR}/paddle/fluid/operators/compat/") -configure_file("op_def_api.h.in" "op_def_api.h") cc_library(op_def_api SRCS op_def_api.cc DEPS op_def_proto) +FILE(GLOB OP_DEF_FILES ${PADDLE_SOURCE_DIR}/paddle/fluid/operators/compat/*.pbtxt) +FILE(WRITE ${CMAKE_CURRENT_BINARY_DIR}/op_def.pbtxt + "namespace { \n" + "const std::unordered_map op_def_map = { \n") +foreach(OP_DEF_FILE ${OP_DEF_FILES}) + FILE(READ ${OP_DEF_FILE} OP_DEF_CONTENT) + get_filename_component(OP_NAME ${OP_DEF_FILE} NAME_WE) + FILE(APPEND ${CMAKE_CURRENT_BINARY_DIR}/op_def.pbtxt + "{\"${OP_NAME}\",R\"(${OP_DEF_CONTENT})\"},\n") +endforeach(OP_DEF_FILE) +FILE(APPEND ${CMAKE_CURRENT_BINARY_DIR}/op_def.pbtxt "{\"\",\"\"}};\n}") + proto_library(heter_service_proto SRCS heter_service.proto) proto_library(data_feed_proto SRCS data_feed.proto) proto_library(trainer_desc_proto SRCS trainer_desc.proto DEPS framework_proto diff --git a/paddle/fluid/framework/ir/op_compat_sensible_pass_tester.cc b/paddle/fluid/framework/ir/op_compat_sensible_pass_tester.cc index 598b686c790..87e28ae3a3a 100644 --- a/paddle/fluid/framework/ir/op_compat_sensible_pass_tester.cc +++ b/paddle/fluid/framework/ir/op_compat_sensible_pass_tester.cc @@ -91,6 +91,18 @@ TEST(OpCompatSensiblePass, compatOpAttribute) { delete info.checker_; } +TEST(OpCompatSensiblePass, opDefNotFound) { + OpCompat compat("fc_1"); + + OpDesc fc_op; + + compat.Judge(fc_op); + + OpCompat compat_1(""); + + compat_1.Judge(fc_op); +} + TEST(OpCompatSensiblePass, compatOpAttributeOptional) { OpCompat compat("fc"); compat.AddAttr("activation_type") diff --git a/paddle/fluid/framework/op_def_api.cc b/paddle/fluid/framework/op_def_api.cc index d8aeb23c63e..5e758fe4105 100644 --- a/paddle/fluid/framework/op_def_api.cc +++ b/paddle/fluid/framework/op_def_api.cc @@ -32,6 +32,14 @@ #include "io/fs.h" #include "paddle/fluid/framework/op_def.pb.h" +/* +// op_def.pbtxt +namespace { + const std::unordered_map op_def_map = {...}; +} +*/ +#include "paddle/fluid/framework/op_def.pbtxt" //NOLINT + namespace paddle { namespace framework { @@ -42,20 +50,20 @@ const proto::OpDef& GetOpDef(const std::string& op_name) { std::lock_guard lk(mtx); if (ops_definition.find(op_name) == ops_definition.end()) { proto::OpDef op_def; - std::string op_path = OP_DEF_FOLDER + op_name + ".pbtxt"; - int fd = open(op_path.c_str(), O_RDONLY); - if (fd == -1) { - LOG(WARNING) << op_path << " open failed!"; + if (op_def_map.find(op_name) == op_def_map.end()) { + LOG(WARNING) << op_name << ".pbtxt not exist!"; } else { - ::google::protobuf::io::FileInputStream* input = - new ::google::protobuf::io::FileInputStream(fd); - if (!::google::protobuf::TextFormat::Parse(input, &op_def)) { - LOG(WARNING) << "Failed to parse " << op_path; + if (!::google::protobuf::TextFormat::ParseFromString( + op_def_map.at(op_name), &op_def)) { + LOG(WARNING) << "Failed to parse " << op_name; } - delete input; - close(fd); } - ops_definition.emplace(std::make_pair(op_name, std::move(op_def))); + if (op_def.type() != op_name) { + LOG(WARNING) << op_name << ".pbtxt has error type :" << op_def.type(); + ops_definition.emplace(std::make_pair(op_name, proto::OpDef())); + } else { + ops_definition.emplace(std::make_pair(op_name, std::move(op_def))); + } } } return ops_definition.at(op_name); diff --git a/paddle/fluid/framework/op_def_api.h b/paddle/fluid/framework/op_def_api.h new file mode 100644 index 00000000000..4ec2089f9b1 --- /dev/null +++ b/paddle/fluid/framework/op_def_api.h @@ -0,0 +1,23 @@ +// Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include "paddle/fluid/framework/op_def.pb.h" + +namespace paddle { +namespace framework { +const proto::OpDef& GetOpDef(const std::string& op_name); +} +} diff --git a/paddle/fluid/framework/op_def_api.h.in b/paddle/fluid/framework/op_def_api.h.in deleted file mode 100644 index 7a48c487709..00000000000 --- a/paddle/fluid/framework/op_def_api.h.in +++ /dev/null @@ -1,12 +0,0 @@ -// the folder of pbtxt with op attribute definition -#pragma once - -#include "paddle/fluid/framework/op_def.pb.h" - -#define OP_DEF_FOLDER "@OP_DEF_FOLDER@" - -namespace paddle { -namespace framework { - const proto::OpDef& GetOpDef(const std::string& op_name); -} -} -- GitLab