From 73f4601da8d0fc376240db36bf95fc84880906c7 Mon Sep 17 00:00:00 2001 From: Chen Weihang Date: Mon, 22 Nov 2021 20:07:28 -0600 Subject: [PATCH] [PTen] Adapt to inference api dir for pten (#37415) * adapt to inference api dir for pten * fix conflit with develop * fix test_egr_ds_eager_tensor compile failed --- cmake/inference_lib.cmake | 27 ++++++------ cmake/pten.cmake | 43 +++++++++++++++++++ .../data_structure_tests/eager_tensor_test.cc | 1 + paddle/fluid/framework/custom_operator.cc | 1 + paddle/pten/api/all.h | 9 ---- paddle/pten/api/lib/api_declare.h | 24 +++++++++++ .../registry.h => lib/api_registry.h} | 0 paddle/pten/api/lib/creation.cc | 2 +- paddle/pten/api/lib/linalg.cc | 2 +- paddle/pten/api/lib/manipulation.cc | 2 +- paddle/pten/api/lib/math.cc | 2 +- paddle/pten/api/lib/utils.cc | 9 +++- paddle/pten/tests/api/test_to_api.cc | 6 --- 13 files changed, 95 insertions(+), 33 deletions(-) create mode 100644 cmake/pten.cmake create mode 100644 paddle/pten/api/lib/api_declare.h rename paddle/pten/api/{include/registry.h => lib/api_registry.h} (100%) diff --git a/cmake/inference_lib.cmake b/cmake/inference_lib.cmake index e5a7f0d2bef..1334bf2f964 100644 --- a/cmake/inference_lib.cmake +++ b/cmake/inference_lib.cmake @@ -216,10 +216,7 @@ copy(inference_lib_dist DSTS ${PADDLE_INFERENCE_INSTALL_DIR}/paddle/include/crypto/) include_directories(${CMAKE_BINARY_DIR}/../paddle/fluid/framework/io) -# TODO(chenweihang, before 11.27) Here, the header file of pten is copied to -# the experimental directory, the include path needs to be changed, so the -# header file path needs to be processed here -# copy api headers for custom op +# copy api headers for pten & custom op copy(inference_lib_dist SRCS ${PADDLE_SOURCE_DIR}/paddle/pten/api/ext/* DSTS ${PADDLE_INFERENCE_INSTALL_DIR}/paddle/include/experimental/pten/api/ext/) @@ -231,21 +228,25 @@ copy(inference_lib_dist DSTS ${PADDLE_INFERENCE_INSTALL_DIR}/paddle/include/experimental/pten/api/) copy(inference_lib_dist SRCS ${PADDLE_SOURCE_DIR}/paddle/pten/common/* - DSTS ${PADDLE_INFERENCE_INSTALL_DIR}/paddle/include/experimental/pten/common/) -copy(inference_lib_dist - SRCS ${PADDLE_SOURCE_DIR}/paddle/fluid/platform/complex.h - DSTS ${PADDLE_INFERENCE_INSTALL_DIR}/paddle/include/experimental/pten/common/) -copy(inference_lib_dist - SRCS ${PADDLE_SOURCE_DIR}/paddle/fluid/platform/float16.h - DSTS ${PADDLE_INFERENCE_INSTALL_DIR}/paddle/include/experimental/pten/common/) + ${PADDLE_SOURCE_DIR}/paddle/fluid/platform/bfloat16.h + ${PADDLE_SOURCE_DIR}/paddle/fluid/platform/complex.h + ${PADDLE_SOURCE_DIR}/paddle/fluid/platform/float16.h + DSTS ${PADDLE_INFERENCE_INSTALL_DIR}/paddle/include/experimental/pten/common/ + ${PADDLE_INFERENCE_INSTALL_DIR}/paddle/include/experimental/pten/common/ + ${PADDLE_INFERENCE_INSTALL_DIR}/paddle/include/experimental/pten/common/ + ${PADDLE_INFERENCE_INSTALL_DIR}/paddle/include/experimental/pten/common/) copy(inference_lib_dist SRCS ${PADDLE_SOURCE_DIR}/paddle/utils/any.h DSTS ${PADDLE_INFERENCE_INSTALL_DIR}/paddle/include/experimental/utils/) -# In order to be compatible with the original behavior, the header file name needs to be changed copy(inference_lib_dist SRCS ${PADDLE_SOURCE_DIR}/paddle/extension.h - DSTS ${PADDLE_INFERENCE_INSTALL_DIR}/paddle/include/experimental/ext_all.h) + DSTS ${PADDLE_INFERENCE_INSTALL_DIR}/paddle/include/experimental/) +# the header file of pten is copied to the experimental directory, +# the include path of pten needs to be changed to adapt to inference api path +add_custom_command(TARGET inference_lib_dist POST_BUILD + COMMAND ${CMAKE_COMMAND} -P "${PADDLE_SOURCE_DIR}/cmake/pten.cmake" + COMMENT "Change pten header include path to adapt to inference api path") # CAPI inference library for only inference set(PADDLE_INFERENCE_C_INSTALL_DIR "${CMAKE_BINARY_DIR}/paddle_inference_c_install_dir" CACHE STRING diff --git a/cmake/pten.cmake b/cmake/pten.cmake new file mode 100644 index 00000000000..77d04a2570c --- /dev/null +++ b/cmake/pten.cmake @@ -0,0 +1,43 @@ +# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set(PADDLE_INFERENCE_INSTALL_DIR "${CMAKE_BINARY_DIR}/paddle_inference_install_dir") + +function(pten_header_path_compat TARGET_PATH) +message(STATUS "pten header path compat processing: ${TARGET_PATH}") +string(FIND ${TARGET_PATH} "experimental" pos) +if (pos GREATER 1) + file(GLOB HEADERS "${TARGET_PATH}/*" "*.h") + foreach(header ${HEADERS}) + string(FIND ${header} ".h" hpos) + if (hpos GREATER 1) + file(READ ${header} HEADER_CONTENT) + string(REPLACE "paddle/pten/" "paddle/include/experimental/pten/" HEADER_CONTENT "${HEADER_CONTENT}") + string(REPLACE "paddle/utils/" "paddle/include/experimental/utils/" HEADER_CONTENT "${HEADER_CONTENT}") + file(WRITE ${header} "${HEADER_CONTENT}") + message(STATUS "pten header path compat processing complete: ${header}") + endif() + endforeach() +endif() +endfunction() + +pten_header_path_compat(${PADDLE_INFERENCE_INSTALL_DIR}/paddle/include/experimental) +pten_header_path_compat(${PADDLE_INFERENCE_INSTALL_DIR}/paddle/include/experimental/pten/api) +pten_header_path_compat(${PADDLE_INFERENCE_INSTALL_DIR}/paddle/include/experimental/pten/api/ext) +pten_header_path_compat(${PADDLE_INFERENCE_INSTALL_DIR}/paddle/include/experimental/pten/api/include) +pten_header_path_compat(${PADDLE_INFERENCE_INSTALL_DIR}/paddle/include/experimental/pten/common) + +# In order to be compatible with the original behavior, the header file name needs to be changed +file(RENAME ${PADDLE_INFERENCE_INSTALL_DIR}/paddle/include/experimental/extension.h + ${PADDLE_INFERENCE_INSTALL_DIR}/paddle/include/experimental/ext_all.h) \ No newline at end of file diff --git a/paddle/fluid/eager/tests/data_structure_tests/eager_tensor_test.cc b/paddle/fluid/eager/tests/data_structure_tests/eager_tensor_test.cc index a528867d442..61ad2e1c5fe 100644 --- a/paddle/fluid/eager/tests/data_structure_tests/eager_tensor_test.cc +++ b/paddle/fluid/eager/tests/data_structure_tests/eager_tensor_test.cc @@ -16,6 +16,7 @@ #include "gtest/gtest.h" #include "paddle/fluid/eager/eager_tensor.h" +#include "paddle/pten/api/lib/api_declare.h" #include "paddle/pten/api/lib/utils/allocator.h" // TODO(jiabin): remove nolint here!!! diff --git a/paddle/fluid/framework/custom_operator.cc b/paddle/fluid/framework/custom_operator.cc index 88cfd74cdaf..5f731ea9ac8 100644 --- a/paddle/fluid/framework/custom_operator.cc +++ b/paddle/fluid/framework/custom_operator.cc @@ -34,6 +34,7 @@ limitations under the License. */ #include "paddle/fluid/platform/dynload/dynamic_loader.h" #include "paddle/fluid/string/string_helper.h" #include "paddle/pten/api/all.h" +#include "paddle/pten/api/lib/api_declare.h" #include "paddle/pten/api/lib/ext_compat_utils.h" #include "paddle/pten/api/lib/utils/tensor_utils.h" #include "paddle/pten/core/convert_utils.h" diff --git a/paddle/pten/api/all.h b/paddle/pten/api/all.h index a55d3332bcd..22cbe5fa1fd 100644 --- a/paddle/pten/api/all.h +++ b/paddle/pten/api/all.h @@ -44,12 +44,3 @@ limitations under the License. */ #include "paddle/pten/api/ext/exception.h" #include "paddle/pten/api/ext/op_meta_info.h" #include "paddle/pten/api/ext/place.h" - -// api symbols declare, remove in the future -#include "paddle/pten/api/include/registry.h" - -PT_DECLARE_API(Creation); -PT_DECLARE_API(Linalg); -PT_DECLARE_API(Manipulation); -PT_DECLARE_API(Math); -PT_DECLARE_API(Utils); diff --git a/paddle/pten/api/lib/api_declare.h b/paddle/pten/api/lib/api_declare.h new file mode 100644 index 00000000000..d29050c8ba4 --- /dev/null +++ b/paddle/pten/api/lib/api_declare.h @@ -0,0 +1,24 @@ +/* Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#pragma once + +// api symbols declare, remove in the future +#include "paddle/pten/api/lib/api_registry.h" + +PT_DECLARE_API(Creation); +PT_DECLARE_API(Linalg); +PT_DECLARE_API(Manipulation); +PT_DECLARE_API(Math); +PT_DECLARE_API(Utils); diff --git a/paddle/pten/api/include/registry.h b/paddle/pten/api/lib/api_registry.h similarity index 100% rename from paddle/pten/api/include/registry.h rename to paddle/pten/api/lib/api_registry.h diff --git a/paddle/pten/api/lib/creation.cc b/paddle/pten/api/lib/creation.cc index 5745267156c..523c2f6bd10 100644 --- a/paddle/pten/api/lib/creation.cc +++ b/paddle/pten/api/lib/creation.cc @@ -18,7 +18,7 @@ limitations under the License. */ #include "glog/logging.h" -#include "paddle/pten/api/include/registry.h" +#include "paddle/pten/api/lib/api_registry.h" #include "paddle/pten/api/lib/kernel_dispatch.h" #include "paddle/pten/api/lib/utils/allocator.h" #include "paddle/pten/core/kernel_registry.h" diff --git a/paddle/pten/api/lib/linalg.cc b/paddle/pten/api/lib/linalg.cc index 2ea142be716..3b7d4994ab7 100644 --- a/paddle/pten/api/lib/linalg.cc +++ b/paddle/pten/api/lib/linalg.cc @@ -18,7 +18,7 @@ limitations under the License. */ #include "glog/logging.h" -#include "paddle/pten/api/include/registry.h" +#include "paddle/pten/api/lib/api_registry.h" #include "paddle/pten/api/lib/kernel_dispatch.h" #include "paddle/pten/api/lib/utils/allocator.h" #include "paddle/pten/core/convert_utils.h" diff --git a/paddle/pten/api/lib/manipulation.cc b/paddle/pten/api/lib/manipulation.cc index f3cc647e702..d06602d721d 100644 --- a/paddle/pten/api/lib/manipulation.cc +++ b/paddle/pten/api/lib/manipulation.cc @@ -17,7 +17,7 @@ limitations under the License. */ #include #include "glog/logging.h" -#include "paddle/pten/api/include/registry.h" +#include "paddle/pten/api/lib/api_registry.h" #include "paddle/pten/api/lib/kernel_dispatch.h" #include "paddle/pten/api/lib/utils/allocator.h" #include "paddle/pten/core/kernel_registry.h" diff --git a/paddle/pten/api/lib/math.cc b/paddle/pten/api/lib/math.cc index e5c955b623c..b7fcdd027cf 100644 --- a/paddle/pten/api/lib/math.cc +++ b/paddle/pten/api/lib/math.cc @@ -18,7 +18,7 @@ limitations under the License. */ #include "glog/logging.h" -#include "paddle/pten/api/include/registry.h" +#include "paddle/pten/api/lib/api_registry.h" #include "paddle/pten/api/lib/kernel_dispatch.h" #include "paddle/pten/api/lib/utils/allocator.h" #include "paddle/pten/core/kernel_registry.h" diff --git a/paddle/pten/api/lib/utils.cc b/paddle/pten/api/lib/utils.cc index a09e1b2c9e1..603699b26c1 100644 --- a/paddle/pten/api/lib/utils.cc +++ b/paddle/pten/api/lib/utils.cc @@ -18,12 +18,19 @@ limitations under the License. */ #include "glog/logging.h" -#include "paddle/pten/api/include/registry.h" +#include "paddle/pten/api/lib/api_registry.h" #include "paddle/pten/api/lib/kernel_dispatch.h" #include "paddle/pten/api/lib/utils/allocator.h" +#include "paddle/pten/core/kernel_registry.h" #include "paddle/pten/include/core.h" #include "paddle/pten/include/infershape.h" +PT_DECLARE_MODULE(UtilsCPU); + +#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP) +PT_DECLARE_MODULE(UtilsCUDA); +#endif + namespace paddle { namespace experimental { diff --git a/paddle/pten/tests/api/test_to_api.cc b/paddle/pten/tests/api/test_to_api.cc index 278735033e3..5374903cbb7 100644 --- a/paddle/pten/tests/api/test_to_api.cc +++ b/paddle/pten/tests/api/test_to_api.cc @@ -21,12 +21,6 @@ limitations under the License. */ #include "paddle/pten/core/dense_tensor.h" #include "paddle/pten/core/kernel_registry.h" -PT_DECLARE_MODULE(UtilsCPU); - -#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP) -PT_DECLARE_MODULE(UtilsCUDA); -#endif - namespace pten { namespace tests { -- GitLab