[PTen] Adapt to inference api dir for pten (#37415)

* adapt to inference api dir for pten

* fix conflit with develop

* fix test_egr_ds_eager_tensor compile failed

cmake/inference_lib.cmake

@@ -216,10 +216,7 @@ copy(inference_lib_dist
         DSTS  ${PADDLE_INFERENCE_INSTALL_DIR}/paddle/include/crypto/)
 include_directories(${CMAKE_BINARY_DIR}/../paddle/fluid/framework/io)
 
-# TODO(chenweihang, before 11.27) Here, the header file of pten is copied to
-# the experimental directory, the include path needs to be changed, so the
-# header file path needs to be processed here
-# copy api headers for custom op
+# copy api headers for pten & custom op
 copy(inference_lib_dist
         SRCS  ${PADDLE_SOURCE_DIR}/paddle/pten/api/ext/*
         DSTS  ${PADDLE_INFERENCE_INSTALL_DIR}/paddle/include/experimental/pten/api/ext/)
@@ -231,21 +228,25 @@ copy(inference_lib_dist
         DSTS  ${PADDLE_INFERENCE_INSTALL_DIR}/paddle/include/experimental/pten/api/)
 copy(inference_lib_dist
         SRCS  ${PADDLE_SOURCE_DIR}/paddle/pten/common/*
-        DSTS  ${PADDLE_INFERENCE_INSTALL_DIR}/paddle/include/experimental/pten/common/)
-copy(inference_lib_dist
-        SRCS  ${PADDLE_SOURCE_DIR}/paddle/fluid/platform/complex.h
-        DSTS  ${PADDLE_INFERENCE_INSTALL_DIR}/paddle/include/experimental/pten/common/)
-copy(inference_lib_dist
-        SRCS  ${PADDLE_SOURCE_DIR}/paddle/fluid/platform/float16.h
-        DSTS  ${PADDLE_INFERENCE_INSTALL_DIR}/paddle/include/experimental/pten/common/)
+              ${PADDLE_SOURCE_DIR}/paddle/fluid/platform/bfloat16.h
+              ${PADDLE_SOURCE_DIR}/paddle/fluid/platform/complex.h
+              ${PADDLE_SOURCE_DIR}/paddle/fluid/platform/float16.h
+        DSTS  ${PADDLE_INFERENCE_INSTALL_DIR}/paddle/include/experimental/pten/common/
+              ${PADDLE_INFERENCE_INSTALL_DIR}/paddle/include/experimental/pten/common/
+              ${PADDLE_INFERENCE_INSTALL_DIR}/paddle/include/experimental/pten/common/
+              ${PADDLE_INFERENCE_INSTALL_DIR}/paddle/include/experimental/pten/common/)
 copy(inference_lib_dist
         SRCS  ${PADDLE_SOURCE_DIR}/paddle/utils/any.h
         DSTS  ${PADDLE_INFERENCE_INSTALL_DIR}/paddle/include/experimental/utils/)
-# In order to be compatible with the original behavior, the header file name needs to be changed
 copy(inference_lib_dist
         SRCS  ${PADDLE_SOURCE_DIR}/paddle/extension.h
-        DSTS  ${PADDLE_INFERENCE_INSTALL_DIR}/paddle/include/experimental/ext_all.h)
+        DSTS  ${PADDLE_INFERENCE_INSTALL_DIR}/paddle/include/experimental/)
 
+# the header file of pten is copied to the experimental directory,
+# the include path of pten needs to be changed to adapt to inference api path
+add_custom_command(TARGET inference_lib_dist POST_BUILD
+        COMMAND ${CMAKE_COMMAND} -P "${PADDLE_SOURCE_DIR}/cmake/pten.cmake"
+        COMMENT "Change pten header include path to adapt to inference api path")
 
 # CAPI inference library for only inference
 set(PADDLE_INFERENCE_C_INSTALL_DIR "${CMAKE_BINARY_DIR}/paddle_inference_c_install_dir" CACHE STRING

cmake/pten.cmake

+# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set(PADDLE_INFERENCE_INSTALL_DIR "${CMAKE_BINARY_DIR}/paddle_inference_install_dir")
+
+function(pten_header_path_compat TARGET_PATH)
+message(STATUS "pten header path compat processing: ${TARGET_PATH}")
+string(FIND ${TARGET_PATH} "experimental" pos)
+if (pos GREATER 1)
+    file(GLOB HEADERS "${TARGET_PATH}/*" "*.h")
+    foreach(header ${HEADERS})
+        string(FIND ${header} ".h" hpos)
+        if (hpos GREATER 1)
+            file(READ ${header} HEADER_CONTENT)
+            string(REPLACE "paddle/pten/" "paddle/include/experimental/pten/" HEADER_CONTENT "${HEADER_CONTENT}")
+            string(REPLACE "paddle/utils/" "paddle/include/experimental/utils/" HEADER_CONTENT "${HEADER_CONTENT}")
+            file(WRITE ${header} "${HEADER_CONTENT}")
+            message(STATUS "pten header path compat processing complete: ${header}")
+        endif()
+    endforeach()
+endif()
+endfunction()
+
+pten_header_path_compat(${PADDLE_INFERENCE_INSTALL_DIR}/paddle/include/experimental)
+pten_header_path_compat(${PADDLE_INFERENCE_INSTALL_DIR}/paddle/include/experimental/pten/api)
+pten_header_path_compat(${PADDLE_INFERENCE_INSTALL_DIR}/paddle/include/experimental/pten/api/ext)
+pten_header_path_compat(${PADDLE_INFERENCE_INSTALL_DIR}/paddle/include/experimental/pten/api/include)
+pten_header_path_compat(${PADDLE_INFERENCE_INSTALL_DIR}/paddle/include/experimental/pten/common)
+
+# In order to be compatible with the original behavior, the header file name needs to be changed
+file(RENAME ${PADDLE_INFERENCE_INSTALL_DIR}/paddle/include/experimental/extension.h
+            ${PADDLE_INFERENCE_INSTALL_DIR}/paddle/include/experimental/ext_all.h)
\ No newline at end of file

paddle/fluid/eager/tests/data_structure_tests/eager_tensor_test.cc

@@ -16,6 +16,7 @@
 #include "gtest/gtest.h"
 
 #include "paddle/fluid/eager/eager_tensor.h"
+#include "paddle/pten/api/lib/api_declare.h"
 #include "paddle/pten/api/lib/utils/allocator.h"
 
 // TODO(jiabin): remove nolint here!!!

paddle/fluid/framework/custom_operator.cc

@@ -34,6 +34,7 @@ limitations under the License. */
 #include "paddle/fluid/platform/dynload/dynamic_loader.h"
 #include "paddle/fluid/string/string_helper.h"
 #include "paddle/pten/api/all.h"
+#include "paddle/pten/api/lib/api_declare.h"
 #include "paddle/pten/api/lib/ext_compat_utils.h"
 #include "paddle/pten/api/lib/utils/tensor_utils.h"
 #include "paddle/pten/core/convert_utils.h"

paddle/pten/api/all.h

@@ -44,12 +44,3 @@ limitations under the License. */
 #include "paddle/pten/api/ext/exception.h"
 #include "paddle/pten/api/ext/op_meta_info.h"
 #include "paddle/pten/api/ext/place.h"
-
-// api symbols declare, remove in the future
-#include "paddle/pten/api/include/registry.h"
-
-PT_DECLARE_API(Creation);
-PT_DECLARE_API(Linalg);
-PT_DECLARE_API(Manipulation);
-PT_DECLARE_API(Math);
-PT_DECLARE_API(Utils);

paddle/pten/api/lib/api_declare.h

+/* Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#pragma once
+
+// api symbols declare, remove in the future
+#include "paddle/pten/api/lib/api_registry.h"
+
+PT_DECLARE_API(Creation);
+PT_DECLARE_API(Linalg);
+PT_DECLARE_API(Manipulation);
+PT_DECLARE_API(Math);
+PT_DECLARE_API(Utils);

paddle/pten/api/lib/creation.cc

@@ -18,7 +18,7 @@ limitations under the License. */
 
 #include "glog/logging.h"
 
-#include "paddle/pten/api/include/registry.h"
+#include "paddle/pten/api/lib/api_registry.h"
 #include "paddle/pten/api/lib/kernel_dispatch.h"
 #include "paddle/pten/api/lib/utils/allocator.h"
 #include "paddle/pten/core/kernel_registry.h"

paddle/pten/api/lib/linalg.cc

@@ -18,7 +18,7 @@ limitations under the License. */
 
 #include "glog/logging.h"
 
-#include "paddle/pten/api/include/registry.h"
+#include "paddle/pten/api/lib/api_registry.h"
 #include "paddle/pten/api/lib/kernel_dispatch.h"
 #include "paddle/pten/api/lib/utils/allocator.h"
 #include "paddle/pten/core/convert_utils.h"

paddle/pten/api/lib/manipulation.cc

@@ -17,7 +17,7 @@ limitations under the License. */
 #include <memory>
 
 #include "glog/logging.h"
-#include "paddle/pten/api/include/registry.h"
+#include "paddle/pten/api/lib/api_registry.h"
 #include "paddle/pten/api/lib/kernel_dispatch.h"
 #include "paddle/pten/api/lib/utils/allocator.h"
 #include "paddle/pten/core/kernel_registry.h"

paddle/pten/api/lib/math.cc

@@ -18,7 +18,7 @@ limitations under the License. */
 
 #include "glog/logging.h"
 
-#include "paddle/pten/api/include/registry.h"
+#include "paddle/pten/api/lib/api_registry.h"
 #include "paddle/pten/api/lib/kernel_dispatch.h"
 #include "paddle/pten/api/lib/utils/allocator.h"
 #include "paddle/pten/core/kernel_registry.h"

paddle/pten/api/lib/utils.cc

@@ -18,12 +18,19 @@ limitations under the License. */
 
 #include "glog/logging.h"
 
-#include "paddle/pten/api/include/registry.h"
+#include "paddle/pten/api/lib/api_registry.h"
 #include "paddle/pten/api/lib/kernel_dispatch.h"
 #include "paddle/pten/api/lib/utils/allocator.h"
+#include "paddle/pten/core/kernel_registry.h"
 #include "paddle/pten/include/core.h"
 #include "paddle/pten/include/infershape.h"
 
+PT_DECLARE_MODULE(UtilsCPU);
+
+#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
+PT_DECLARE_MODULE(UtilsCUDA);
+#endif
+
 namespace paddle {
 namespace experimental {
 

paddle/pten/tests/api/test_to_api.cc

@@ -21,12 +21,6 @@ limitations under the License. */
 #include "paddle/pten/core/dense_tensor.h"
 #include "paddle/pten/core/kernel_registry.h"
 
-PT_DECLARE_MODULE(UtilsCPU);
-
-#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
-PT_DECLARE_MODULE(UtilsCUDA);
-#endif
-
 namespace pten {
 namespace tests {