From 9af72957520e4dffa6356bc637e0532bd799ab75 Mon Sep 17 00:00:00 2001 From: wanghuancoder Date: Wed, 2 Mar 2022 11:14:25 +0800 Subject: [PATCH] [Eager] open eager when WITH_PYTHON (#39979) * open eager when WITH_PYTHON, test=develop * refine, test=develop * refine, test=develop * add DWITH_PYTHON for gen_fluid_lib, test=develop --- paddle/fluid/eager/CMakeLists.txt | 2 +- paddle/fluid/eager/api/generated/CMakeLists.txt | 2 +- .../eager_generated/backwards/CMakeLists.txt | 2 +- .../eager_generated/forwards/CMakeLists.txt | 2 +- paddle/fluid/eager/tests/CMakeLists.txt | 2 +- paddle/fluid/eager/tests/task_tests/CMakeLists.txt | 2 +- paddle/fluid/pybind/CMakeLists.txt | 8 ++++---- paddle/fluid/pybind/pybind.cc | 8 ++------ paddle/scripts/paddle_build.sh | 14 ++++++++++---- python/paddle/fluid/tests/unittests/CMakeLists.txt | 2 +- 10 files changed, 23 insertions(+), 21 deletions(-) diff --git a/paddle/fluid/eager/CMakeLists.txt b/paddle/fluid/eager/CMakeLists.txt index 5e16ab2b391..8cb69caf663 100644 --- a/paddle/fluid/eager/CMakeLists.txt +++ b/paddle/fluid/eager/CMakeLists.txt @@ -2,7 +2,7 @@ set(eager_deps phi phi_api hook_utils tensor_utils utils global_utils backward p set(fluid_deps tracer layer proto_desc operator op_registry variable_helper memcpy) set(generated_deps dygraph_function dygraph_node) -if(NOT ON_INFER) +if(NOT ((NOT WITH_PYTHON) AND ON_INFER)) message("Performing Eager Dygraph Auto Code Generation") add_subdirectory(auto_code_generator) endif() diff --git a/paddle/fluid/eager/api/generated/CMakeLists.txt b/paddle/fluid/eager/api/generated/CMakeLists.txt index ebbef286f79..4f634c6884b 100644 --- a/paddle/fluid/eager/api/generated/CMakeLists.txt +++ b/paddle/fluid/eager/api/generated/CMakeLists.txt @@ -1,5 +1,5 @@ add_subdirectory(eager_generated) -if(NOT ON_INFER) +if(NOT ((NOT WITH_PYTHON) AND ON_INFER)) add_subdirectory(fluid_generated) endif() diff --git a/paddle/fluid/eager/api/generated/eager_generated/backwards/CMakeLists.txt b/paddle/fluid/eager/api/generated/eager_generated/backwards/CMakeLists.txt index 77d8ec57efc..81ff07b8963 100644 --- a/paddle/fluid/eager/api/generated/eager_generated/backwards/CMakeLists.txt +++ b/paddle/fluid/eager/api/generated/eager_generated/backwards/CMakeLists.txt @@ -1,6 +1,6 @@ cc_library(scale_node SRCS scale_node.cc DEPS global_utils phi phi_api grad_node_info) -if(NOT ON_INFER) +if(NOT (NOT WITH_PYTHON AND ON_INFER)) cc_library(final_dygraph_node SRCS nodes.cc DEPS ${eager_deps}) add_dependencies(final_dygraph_node eager_final_state_codegen) endif() diff --git a/paddle/fluid/eager/api/generated/eager_generated/forwards/CMakeLists.txt b/paddle/fluid/eager/api/generated/eager_generated/forwards/CMakeLists.txt index 60b35340eab..c70bb80c35c 100644 --- a/paddle/fluid/eager/api/generated/eager_generated/forwards/CMakeLists.txt +++ b/paddle/fluid/eager/api/generated/eager_generated/forwards/CMakeLists.txt @@ -1,6 +1,6 @@ cc_library(eager_scale SRCS scale.cc DEPS phi_api phi autograd_meta scale_node) -if(NOT ON_INFER) +if(NOT (NOT WITH_PYTHON AND ON_INFER)) cc_library(final_dygraph_function SRCS dygraph_functions.cc DEPS ${eager_deps}) add_dependencies(final_dygraph_function eager_final_state_codegen) endif() diff --git a/paddle/fluid/eager/tests/CMakeLists.txt b/paddle/fluid/eager/tests/CMakeLists.txt index c1506d8139b..2bfb9937c8c 100644 --- a/paddle/fluid/eager/tests/CMakeLists.txt +++ b/paddle/fluid/eager/tests/CMakeLists.txt @@ -1,6 +1,6 @@ add_subdirectory(data_structure_tests) add_subdirectory(task_tests) -if(NOT ON_INFER) +if(NOT ((NOT WITH_PYTHON) AND ON_INFER)) add_subdirectory(performance_tests) endif() diff --git a/paddle/fluid/eager/tests/task_tests/CMakeLists.txt b/paddle/fluid/eager/tests/task_tests/CMakeLists.txt index dbdb52eb536..c65ad4641cf 100644 --- a/paddle/fluid/eager/tests/task_tests/CMakeLists.txt +++ b/paddle/fluid/eager/tests/task_tests/CMakeLists.txt @@ -6,7 +6,7 @@ cc_test(test_egr_task_hook SRCS hook_test.cc DEPS ${eager_deps} ${fluid_deps} ea cc_test(test_egr_task_cross_batch SRCS cross_batch_accumulation_test.cc DEPS ${eager_deps} ${fluid_deps} eager_scale scale_node) cc_test(test_egr_task_fwd_bwd_joint SRCS fwd_bwd_joint_test.cc DEPS ${eager_deps} ${fluid_deps} eager_scale scale_node) -if(NOT ON_INFER) +if(NOT ((NOT WITH_PYTHON) AND ON_INFER)) cc_test(test_egr_task_hook_intermidiate SRCS hook_test_intermidiate.cc DEPS ${eager_deps} ${fluid_deps} ${generated_deps} dygraph_node) cc_test(test_egr_task_autocodegen SRCS generated_test.cc DEPS ${eager_deps} ${fluid_deps} ${generated_deps}) endif() diff --git a/paddle/fluid/pybind/CMakeLists.txt b/paddle/fluid/pybind/CMakeLists.txt index c61e8212b02..48d42f803a8 100644 --- a/paddle/fluid/pybind/CMakeLists.txt +++ b/paddle/fluid/pybind/CMakeLists.txt @@ -242,7 +242,7 @@ if(WITH_PYTHON) COMMAND ${CMAKE_COMMAND} -E copy_if_different ${tmp_impl_file} ${impl_file} COMMENT "copy_if_different ${tmp_impl_file} to ${impl_file}" DEPENDS ${OP_IMPL_DEPS}) - if(NOT ON_INFER) + if(NOT ((NOT WITH_PYTHON) AND ON_INFER)) add_custom_command(OUTPUT ${eager_impl_file} COMMAND ${CMAKE_BINARY_DIR}/paddle/fluid/pybind/eager_op_function_generator_retry.bat COMMAND ${CMAKE_COMMAND} -E copy_if_different ${tmp_eager_impl_file} ${eager_impl_file} @@ -276,7 +276,7 @@ if(WITH_PYTHON) COMMENT "copy_if_different ${tmp_impl_file} to ${impl_file}" DEPENDS ${OP_IMPL_DEPS} VERBATIM) - if(NOT ON_INFER) + if(NOT ((NOT WITH_PYTHON) AND ON_INFER)) add_custom_command(OUTPUT ${eager_impl_file} COMMAND ${CMAKE_COMMAND} -E env "LD_LIBRARY_PATH=$ENV{LD_LIBRARY_PATH}:." "${CMAKE_CURRENT_BINARY_DIR}/eager_op_function_generator" @@ -288,7 +288,7 @@ if(WITH_PYTHON) endif() endif(WIN32) add_custom_target(op_function_generator_cmd ALL DEPENDS ${impl_file}) - if(NOT ON_INFER) + if(NOT ((NOT WITH_PYTHON) AND ON_INFER)) add_custom_target(eager_op_function_generator_cmd ALL DEPENDS ${eager_impl_file}) endif() @@ -296,7 +296,7 @@ if(WITH_PYTHON) cc_library(op_function_common SRCS op_function_common.cc DEPS ${PYBIND_DEPS}) list(APPEND PYBIND_DEPS op_function_common) - if(NOT ON_INFER) + if(NOT ((NOT WITH_PYTHON) AND ON_INFER)) cc_library(paddle_eager SRCS eager.cc eager_functions.cc eager_method.cc eager_properties.cc eager_utils.cc DEPS eager_api autograd_meta backward grad_node_info phi op_function_common final_dygraph_function final_dygraph_node dygraph_function dygraph_node accumulation_node global_utils utils python) diff --git a/paddle/fluid/pybind/pybind.cc b/paddle/fluid/pybind/pybind.cc index 3d8815e2eb6..2d9272dd0ed 100644 --- a/paddle/fluid/pybind/pybind.cc +++ b/paddle/fluid/pybind/pybind.cc @@ -79,12 +79,10 @@ limitations under the License. */ #include "paddle/fluid/platform/profiler.h" #include "paddle/fluid/pybind/cuda_streams_py.h" #include "paddle/fluid/pybind/distributed_py.h" -#include "paddle/phi/core/compat/convert_utils.h" -#include "paddle/phi/core/lod_utils.h" -#ifndef PADDLE_ON_INFERENCE #include "paddle/fluid/pybind/eager.h" -#endif #include "paddle/fluid/pybind/io.h" +#include "paddle/phi/core/compat/convert_utils.h" +#include "paddle/phi/core/lod_utils.h" #include "paddle/utils/none.h" #ifdef PADDLE_WITH_ASCEND #include "paddle/fluid/pybind/ascend_wrapper_py.h" @@ -529,9 +527,7 @@ PYBIND11_MODULE(core_avx, m) { PYBIND11_MODULE(core_noavx, m) { #endif -#ifndef PADDLE_ON_INFERENCE BindEager(&m); -#endif BindCudaStream(&m); // Not used, just make sure cpu_info.cc is linked. diff --git a/paddle/scripts/paddle_build.sh b/paddle/scripts/paddle_build.sh index 8528ba34e21..9bef7e12851 100755 --- a/paddle/scripts/paddle_build.sh +++ b/paddle/scripts/paddle_build.sh @@ -2374,7 +2374,7 @@ EOF fi startTime_s=`date +%s` set +e - cmake .. -DWITH_DISTRIBUTE=OFF -DON_INFER=ON -DWITH_TENSORRT=ON -DCUDA_ARCH_NAME=${CUDA_ARCH_NAME:-Auto};build_error=$? + cmake .. -DWITH_DISTRIBUTE=OFF -DON_INFER=ON -DWITH_TENSORRT=ON -DCUDA_ARCH_NAME=${CUDA_ARCH_NAME:-Auto} -DWITH_PYTHON=${WITH_PYTHON:-ON};build_error=$? # reset ccache zero stats for collect PR's actual hit rate ccache -z @@ -2739,7 +2739,9 @@ function main() { test_fluid_lib ;; build_inference_lib) - python ${PADDLE_ROOT}/tools/remove_grad_op_and_kernel.py + if [ "${WITH_PYTHON}" == "OFF" ] ; then + python ${PADDLE_ROOT}/tools/remove_grad_op_and_kernel.py + fi cmake_gen ${PYTHON_ABI:-""} gen_fluid_lib ${parallel_number} ;; @@ -2790,7 +2792,9 @@ function main() { ;; test_inference) PADDLE_ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}")/../../" && pwd )" - python ${PADDLE_ROOT}/tools/remove_grad_op_and_kernel.py + if [ "${WITH_PYTHON}" == "OFF" ] ; then + python ${PADDLE_ROOT}/tools/remove_grad_op_and_kernel.py + fi gen_fluid_lib ${parallel_number} test_fluid_lib #test_fluid_lib_train @@ -2800,7 +2804,9 @@ function main() { ;; build_inference) PADDLE_ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}")/../../" && pwd )" - python ${PADDLE_ROOT}/tools/remove_grad_op_and_kernel.py + if [ "${WITH_PYTHON}" == "OFF" ] ; then + python ${PADDLE_ROOT}/tools/remove_grad_op_and_kernel.py + fi gen_fluid_lib ${parallel_number} ;; gpu_inference) diff --git a/python/paddle/fluid/tests/unittests/CMakeLists.txt b/python/paddle/fluid/tests/unittests/CMakeLists.txt index 7d64cf7bd89..2f6df075478 100644 --- a/python/paddle/fluid/tests/unittests/CMakeLists.txt +++ b/python/paddle/fluid/tests/unittests/CMakeLists.txt @@ -116,7 +116,7 @@ foreach(TEST_OP ${MIXED_DIST_TEST_OPS}) list(REMOVE_ITEM TEST_OPS ${TEST_OP}) endforeach() -if(ON_INFER) +if(NOT WITH_PYTHON AND ON_INFER) LIST(REMOVE_ITEM TEST_OPS test_eager_trace_op) endif() -- GitLab