CMakeLists.txt 8.3 KB
Newer Older
1 2 3 4
# Adapt to custom op mechanism: Include the header files related to the data type
# to avoid exposing the path of the underlying file
include_directories(${PADDLE_SOURCE_DIR}/paddle/fluid/platform)

5
set(PYBIND_DEPS pybind python proto_desc memory executor fleet_wrapper box_wrapper prune
J
Jiabin Yang 已提交
6
  feed_fetch_method pass_builder parallel_executor profiler layer tracer engine scope_pool
7
  analysis_predictor imperative_profiler imperative_flag save_load_util dlpack_tensor device_context
8
  gloo_wrapper infer_io_utils heter_wrapper generator op_version_registry ps_gpu_wrapper custom_operator)
9

S
seemingwang 已提交
10 11 12 13
if (WITH_PSCORE)
  set(PYBIND_DEPS ${PYBIND_DEPS} ps_service)
  set(PYBIND_DEPS ${PYBIND_DEPS} graph_py_service)
endif()
14
if (WITH_GPU OR WITH_ROCM)
15
  set(PYBIND_DEPS ${PYBIND_DEPS} dynload_cuda)
16
  set(PYBIND_DEPS ${PYBIND_DEPS} cuda_device_guard)
17 18
endif()

19
if (WITH_NCCL OR WITH_RCCL)
20
  set(PYBIND_DEPS ${PYBIND_DEPS} nccl_wrapper)
21
  set(PYBIND_DEPS ${PYBIND_DEPS} reducer)
22 23
endif()

24 25 26 27 28
if (WITH_XPU_BKCL)
  set(PYBIND_DEPS ${PYBIND_DEPS} reducer)
  set(PYBIND_DEPS ${PYBIND_DEPS} bkcl_context)
endif()

29
if(NOT WIN32)
30
  set(PYBIND_DEPS ${PYBIND_DEPS} data_loader)
31
  set(PYBIND_DEPS ${PYBIND_DEPS} mmap_allocator)
32
  if (WITH_NCCL OR WITH_RCCL)
33 34
    set(PYBIND_DEPS ${PYBIND_DEPS} nccl_context)
  endif()
35
endif(NOT WIN32)
F
flame 已提交
36

S
sneaxiy 已提交
37 38
if(WITH_PYTHON)
  list(APPEND PYBIND_DEPS py_func_op)
39
  list(APPEND PYBIND_DEPS py_layer_op)
S
sneaxiy 已提交
40
endif()
41 42 43 44 45 46

set(PYBIND_SRCS
  pybind.cc
  exception.cc
  protobuf.cc
  const_value.cc
47
  global_value_getter_setter.cc
48 49
  reader_py.cc
  fleet_wrapper_py.cc
T
Thunderbrook 已提交
50
  heter_wrapper_py.cc
T
Thunderbrook 已提交
51
  ps_gpu_wrapper_py.cc
52
  gloo_wrapper_py.cc
H
hutuxian 已提交
53
  box_helper_py.cc
54 55 56
  data_set_py.cc
  imperative.cc
  ir.cc
Y
yaoxuefeng 已提交
57
  inference_api.cc
58
  compatible.cc
59
  io.cc
Y
yaoxuefeng 已提交
60
  generator_py.cc)
61

62
if(WITH_ASCEND)
63 64
  set(PYBIND_DEPS ${PYBIND_DEPS} ascend_wrapper)
  set(PYBIND_SRCS ${PYBIND_SRCS} ascend_wrapper_py.cc)
65
endif()
66

67 68 69 70 71
if(WITH_GLOO)
  set(PYBIND_DEPS ${PYBIND_DEPS} gloo_context)
  set(PYBIND_SRCS ${PYBIND_SRCS} gloo_context_py.cc)
endif(WITH_GLOO)

Y
Yanghello 已提交
72 73 74 75 76
if (WITH_CRYPTO)
  set(PYBIND_DEPS ${PYBIND_DEPS} paddle_crypto)
  set(PYBIND_SRCS ${PYBIND_SRCS} crypto.cc)
endif (WITH_CRYPTO)

T
Thunderbrook 已提交
77 78 79 80 81 82 83 84
if (WITH_PSLIB)
  set(DISTRIBUTE_COMPILE_FLAGS "-Wno-non-virtual-dtor -Wno-error=non-virtual-dtor -Wno-error=delete-non-virtual-dtor -Wno-error=sign-compare -Wno-error=unused-variable -Wno-error=return-type -Wno-error=unused-but-set-variable -Wno-error=type-limits -Wno-error=unknown-pragmas -Wno-error=parentheses -Wno-error=unused-result")
  if (CMAKE_CXX_COMPILER_VERSION VERSION_GREATER 7.0)
      set(DISTRIBUTE_COMPILE_FLAGS
              "${DISTRIBUTE_COMPILE_FLAGS} -faligned-new")
  endif()
  set_source_files_properties(heter_wrapper_py.cc PROPERTIES COMPILE_FLAGS ${DISTRIBUTE_COMPILE_FLAGS})
endif(WITH_PSLIB)
T
tangwei12 已提交
85
if (WITH_PSCORE)
T
tangwei12 已提交
86 87
  set(DISTRIBUTE_COMPILE_FLAGS "-Wno-non-virtual-dtor -Wno-error=non-virtual-dtor -Wno-error=delete-non-virtual-dtor -Wno-error=sign-compare -Wno-error=unused-variable -Wno-error=return-type -Wno-error=unused-but-set-variable -Wno-error=type-limits -Wno-error=unknown-pragmas -Wno-error=parentheses -Wno-error=unused-result")
  set_source_files_properties(fleet_py.cc PROPERTIES COMPILE_FLAGS ${DISTRIBUTE_COMPILE_FLAGS})
1
123malin 已提交
88
  list(APPEND PYBIND_DEPS fleet communicator index_wrapper index_sampler)
T
tangwei12 已提交
89
  list(APPEND PYBIND_SRCS fleet_py.cc)
90
endif()
91

92
if (WITH_NCCL OR WITH_RCCL)
93 94 95
  list(APPEND PYBIND_SRCS nccl_wrapper_py.cc)
endif()

L
Leo Chen 已提交
96 97
if(WITH_PYTHON)
  # generate op pybind functions automatically for dygraph.
98 99 100 101 102
  if (WITH_ASCEND_CL)
    set(OP_FUNCTION_GENERETOR_DEPS pybind proto_desc executor layer tracer engine imperative_profiler imperative_flag ascend_wrapper)
  else()
    set(OP_FUNCTION_GENERETOR_DEPS pybind proto_desc executor layer tracer engine imperative_profiler imperative_flag)
  endif()
L
Leo Chen 已提交
103 104 105
  list(APPEND OP_FUNCTION_GENERETOR_DEPS ${GLOB_OP_LIB})
  list(APPEND OP_FUNCTION_GENERETOR_DEPS ${GLOB_OPERATOR_DEPS})

106
  if (WITH_NCCL OR WITH_RCCL)
L
Leo Chen 已提交
107
    list(APPEND OP_FUNCTION_GENERETOR_DEPS nccl_context)
108
  endif()
L
Leo Chen 已提交
109

110 111 112 113
  if(WITH_XPU_BKCL)
    list(APPEND OP_FUNCTION_GENERETOR_DEPS bkcl_context)
  endif(WITH_XPU_BKCL)

L
Leo Chen 已提交
114
  add_executable(op_function_generator op_function_generator.cc)
115
  target_link_libraries(op_function_generator ${OP_FUNCTION_GENERETOR_DEPS})
116

L
Leo Chen 已提交
117 118
  get_property (os_dependency_modules GLOBAL PROPERTY OS_DEPENDENCY_MODULES)
  target_link_libraries(op_function_generator ${os_dependency_modules})
119 120 121
  if(WITH_ROCM)
    target_link_libraries(op_function_generator ${ROCM_HIPRTC_LIB})
  endif()
L
Leo Chen 已提交
122

123 124 125
  set(impl_file ${CMAKE_SOURCE_DIR}/paddle/fluid/pybind/op_function_impl.h)
  set(tmp_impl_file ${impl_file}.tmp)

126
  if(WIN32)
127
    if("${CMAKE_GENERATOR}" STREQUAL "Ninja")
128 129 130 131
      set(op_function_generator_path "${CMAKE_CURRENT_BINARY_DIR}")
    else()
      set(op_function_generator_path "${CMAKE_CURRENT_BINARY_DIR}/${CMAKE_BUILD_TYPE}")
    endif()
132 133 134 135
    file(TO_NATIVE_PATH ${op_function_generator_path} op_function_generator_path)
    file(TO_NATIVE_PATH ${impl_file} impl_file)
    file(TO_NATIVE_PATH ${tmp_impl_file} tmp_impl_file)

136 137 138 139
    file(WRITE ${CMAKE_BINARY_DIR}/paddle/fluid/pybind/op_function_generator_retry.bat ""
    "set build_times=1\n"
    ":retry\n"
    "ECHO op_function_generator run %build_times% time\n"
140 141 142
    "if exist ${tmp_impl_file} del ${tmp_impl_file}\n"
    "taskkill /f /im op_function_generator.exe 2>NUL\n"
    "${op_function_generator_path}\\op_function_generator.exe ${tmp_impl_file}\n"
143 144
    "if %ERRORLEVEL% NEQ 0 (\n"
    "    set /a build_times=%build_times%+1\n"
Z
Zhou Wei 已提交
145
    "    if %build_times% GEQ 10 (\n"
146 147 148 149 150 151 152
    "        exit /b 1\n"
    "    ) else (\n"
    "        goto :retry\n"
    "    )\n"
    ")\n"
    "exit /b 0")

Z
Zhou Wei 已提交
153
    add_custom_command(TARGET op_function_generator POST_BUILD
154
          COMMAND ${CMAKE_BINARY_DIR}/paddle/fluid/pybind/op_function_generator_retry.bat
155 156
          COMMAND ${CMAKE_COMMAND} -E copy_if_different ${tmp_impl_file} ${impl_file}
          COMMENT "copy_if_different ${tmp_impl_file} to ${impl_file}"
157 158
    )

159
    if(${CBLAS_PROVIDER} STREQUAL MKLML)
160
      add_custom_command(TARGET op_function_generator
161
            PRE_LINK
162 163
            COMMAND ${CMAKE_COMMAND} -E copy ${MKLML_SHARED_LIB} ${op_function_generator_path}
            COMMAND ${CMAKE_COMMAND} -E copy ${MKLML_SHARED_IOMP_LIB} ${op_function_generator_path}
L
Leo Chen 已提交
164
          )
165
    else(${CBLAS_PROVIDER} STREQUAL EXTERN_OPENBLAS)
166
      add_custom_command(TARGET op_function_generator
167
            PRE_LINK
168
            COMMAND ${CMAKE_COMMAND} -E copy ${OPENBLAS_SHARED_LIB} ${op_function_generator_path}
169 170
          )
    endif()
L
Leo Chen 已提交
171
    if(WITH_MKLDNN)
172
      add_custom_command(TARGET op_function_generator
173
          PRE_LINK
174
          COMMAND ${CMAKE_COMMAND} -E copy ${MKLDNN_SHARED_LIB} ${op_function_generator_path}
L
Leo Chen 已提交
175
          )
176
    endif()
L
Leo Chen 已提交
177
  else(WIN32)
178
    # If there are no *.so in /usr/lib or LD_LIBRARY_PATH,
L
Leo Chen 已提交
179
    # copy these *.so to current directory and append current directory to
180
    # LD_LIBRARY_PATH. This is different with Windows platformm, which search
L
Leo Chen 已提交
181
    # *.dll in current directory automatically.
182 183 184
    add_custom_command(TARGET op_function_generator
          POST_BUILD
          COMMAND ${CMAKE_COMMAND} -E env "LD_LIBRARY_PATH=$ENV{LD_LIBRARY_PATH}:."
185
              "${CMAKE_CURRENT_BINARY_DIR}/op_function_generator"
186 187
              "${tmp_impl_file}"
          COMMAND ${CMAKE_COMMAND} -E copy_if_different ${tmp_impl_file} ${impl_file}
188
          COMMENT "copy_if_different ${tmp_impl_file} to ${impl_file}"
189 190
          VERBATIM
    )
L
Leo Chen 已提交
191
    if(WITH_MKL)
192
      add_custom_command(TARGET op_function_generator
193
            PRE_LINK
194 195 196
            COMMAND ${CMAKE_COMMAND} -E copy ${MKLML_SHARED_LIB} ${CMAKE_CURRENT_BINARY_DIR}
            COMMAND ${CMAKE_COMMAND} -E copy ${MKLML_SHARED_IOMP_LIB} ${CMAKE_CURRENT_BINARY_DIR}
            )
L
Leo Chen 已提交
197 198
    endif(WITH_MKL)
    if(WITH_MKLDNN)
199
      add_custom_command(TARGET op_function_generator
200
          PRE_LINK
L
Leo Chen 已提交
201 202 203 204 205
          COMMAND ${CMAKE_COMMAND} -E copy ${MKLDNN_SHARED_LIB} ${CMAKE_CURRENT_BINARY_DIR}
          )
    endif(WITH_MKLDNN)
  endif(WIN32)

206 207 208 209 210 211 212 213 214 215 216
  cc_library(paddle_pybind SHARED
  SRCS ${PYBIND_SRCS}
  DEPS ${PYBIND_DEPS}
  ${GLOB_OP_LIB} ${GLOB_OPERATOR_DEPS})
  if(NOT APPLE AND NOT WIN32)
    target_link_libraries(paddle_pybind rt)
  endif(NOT APPLE AND NOT WIN32)

  if(WITH_ROCM)
    target_link_libraries(paddle_pybind ${ROCM_HIPRTC_LIB})
  endif()
217

P
peizhilin 已提交
218 219
  get_property (os_dependency_modules GLOBAL PROPERTY OS_DEPENDENCY_MODULES)
  target_link_libraries(paddle_pybind ${os_dependency_modules})
220
  add_dependencies(paddle_pybind op_function_generator)
221
endif(WITH_PYTHON)