CMakeLists.txt 12.0 KB
Newer Older
1
set(PYBIND_DEPS pybind python proto_desc memory executor fleet_wrapper box_wrapper prune
2
  feed_fetch_method pass generate_pass pass_builder parallel_executor profiler layer tracer engine scope_pool
3
  analysis_predictor imperative_profiler imperative_flag save_load_util dlpack_tensor device_context
H
Huihuang Zheng 已提交
4
  gloo_wrapper infer_io_utils heter_wrapper generator op_version_registry ps_gpu_wrapper custom_operator
L
LiYuRio 已提交
5
  cost_model cuda_graph_with_memory_pool fleet_executor)
6

S
seemingwang 已提交
7 8 9 10
if (WITH_PSCORE)
  set(PYBIND_DEPS ${PYBIND_DEPS} ps_service)
  set(PYBIND_DEPS ${PYBIND_DEPS} graph_py_service)
endif()
11
if (WITH_GPU OR WITH_ROCM)
12
  set(PYBIND_DEPS ${PYBIND_DEPS} dynload_cuda)
13
  set(PYBIND_DEPS ${PYBIND_DEPS} cuda_device_guard)
14 15
endif()

16 17 18
if (WITH_GPU)
  set(PYBIND_DEPS ${PYBIND_DEPS} cuda_profiler)
endif()
J
jianghaicheng 已提交
19 20 21
if (WITH_IPU)
  set(PYBIND_DEPS ${PYBIND_DEPS} ipu_info)
endif()
22

23
if (WITH_NCCL OR WITH_RCCL)
24
  set(PYBIND_DEPS ${PYBIND_DEPS} nccl_wrapper)
25
  set(PYBIND_DEPS ${PYBIND_DEPS} reducer)
26 27
endif()

28 29 30
if (WITH_XPU_BKCL)
  set(PYBIND_DEPS ${PYBIND_DEPS} reducer)
  set(PYBIND_DEPS ${PYBIND_DEPS} bkcl_context)
K
kuizhiqing 已提交
31 32 33 34 35 36 37
  set(PYBIND_DEPS ${PYBIND_DEPS} heter_ccl_context)
endif()

if (WITH_ASCEND_CL)
  set(PYBIND_DEPS ${PYBIND_DEPS} reducer)
  set(PYBIND_DEPS ${PYBIND_DEPS} hccl_context)
  set(PYBIND_DEPS ${PYBIND_DEPS} heter_ccl_context)
38 39
endif()

40
if(NOT WIN32)
41
  set(PYBIND_DEPS ${PYBIND_DEPS} data_loader)
42
  set(PYBIND_DEPS ${PYBIND_DEPS} mmap_allocator)
43
  if (WITH_NCCL OR WITH_RCCL)
44
    set(PYBIND_DEPS ${PYBIND_DEPS} nccl_context)
K
kuizhiqing 已提交
45
    set(PYBIND_DEPS ${PYBIND_DEPS} heter_ccl_context)
46
  endif()
47
endif(NOT WIN32)
F
flame 已提交
48

S
sneaxiy 已提交
49 50
if(WITH_PYTHON)
  list(APPEND PYBIND_DEPS py_func_op)
51
  list(APPEND PYBIND_DEPS py_layer_op)
S
sneaxiy 已提交
52
endif()
53 54 55 56 57 58

set(PYBIND_SRCS
  pybind.cc
  exception.cc
  protobuf.cc
  const_value.cc
59
  global_value_getter_setter.cc
60 61
  reader_py.cc
  fleet_wrapper_py.cc
T
Thunderbrook 已提交
62
  heter_wrapper_py.cc
T
Thunderbrook 已提交
63
  ps_gpu_wrapper_py.cc
64
  gloo_wrapper_py.cc
H
hutuxian 已提交
65
  box_helper_py.cc
66 67 68
  data_set_py.cc
  imperative.cc
  ir.cc
H
Huihuang Zheng 已提交
69
  bind_cost_model.cc
L
LiYuRio 已提交
70
  bind_fleet_executor.cc
Y
yaoxuefeng 已提交
71
  inference_api.cc
72
  compatible.cc
73
  io.cc
74 75
  generator_py.cc
  cuda_streams_py.cc)
76

77
if(WITH_ASCEND)
78 79
  set(PYBIND_DEPS ${PYBIND_DEPS} ascend_wrapper)
  set(PYBIND_SRCS ${PYBIND_SRCS} ascend_wrapper_py.cc)
80
endif()
81

82 83 84
if(WITH_GLOO)
  set(PYBIND_DEPS ${PYBIND_DEPS} gloo_context)
  set(PYBIND_SRCS ${PYBIND_SRCS} gloo_context_py.cc)
85 86
  set(PYBIND_DEPS ${PYBIND_DEPS} imperative_gloo_context)
  set(PYBIND_DEPS ${PYBIND_DEPS} reducer)
87 88
endif(WITH_GLOO)

Y
Yanghello 已提交
89 90 91 92 93
if (WITH_CRYPTO)
  set(PYBIND_DEPS ${PYBIND_DEPS} paddle_crypto)
  set(PYBIND_SRCS ${PYBIND_SRCS} crypto.cc)
endif (WITH_CRYPTO)

T
Thunderbrook 已提交
94 95 96 97 98 99 100 101
if (WITH_PSLIB)
  set(DISTRIBUTE_COMPILE_FLAGS "-Wno-non-virtual-dtor -Wno-error=non-virtual-dtor -Wno-error=delete-non-virtual-dtor -Wno-error=sign-compare -Wno-error=unused-variable -Wno-error=return-type -Wno-error=unused-but-set-variable -Wno-error=type-limits -Wno-error=unknown-pragmas -Wno-error=parentheses -Wno-error=unused-result")
  if (CMAKE_CXX_COMPILER_VERSION VERSION_GREATER 7.0)
      set(DISTRIBUTE_COMPILE_FLAGS
              "${DISTRIBUTE_COMPILE_FLAGS} -faligned-new")
  endif()
  set_source_files_properties(heter_wrapper_py.cc PROPERTIES COMPILE_FLAGS ${DISTRIBUTE_COMPILE_FLAGS})
endif(WITH_PSLIB)
T
tangwei12 已提交
102
if (WITH_PSCORE)
T
tangwei12 已提交
103 104
  set(DISTRIBUTE_COMPILE_FLAGS "-Wno-non-virtual-dtor -Wno-error=non-virtual-dtor -Wno-error=delete-non-virtual-dtor -Wno-error=sign-compare -Wno-error=unused-variable -Wno-error=return-type -Wno-error=unused-but-set-variable -Wno-error=type-limits -Wno-error=unknown-pragmas -Wno-error=parentheses -Wno-error=unused-result")
  set_source_files_properties(fleet_py.cc PROPERTIES COMPILE_FLAGS ${DISTRIBUTE_COMPILE_FLAGS})
1
123malin 已提交
105
  list(APPEND PYBIND_DEPS fleet communicator index_wrapper index_sampler)
T
tangwei12 已提交
106
  list(APPEND PYBIND_SRCS fleet_py.cc)
107
endif()
108

109
if (WITH_NCCL OR WITH_RCCL)
110 111 112
  list(APPEND PYBIND_SRCS nccl_wrapper_py.cc)
endif()

L
Leo Chen 已提交
113 114
if(WITH_PYTHON)
  # generate op pybind functions automatically for dygraph.
115 116 117 118 119
  if (WITH_ASCEND_CL)
    set(OP_FUNCTION_GENERETOR_DEPS pybind proto_desc executor layer tracer engine imperative_profiler imperative_flag ascend_wrapper)
  else()
    set(OP_FUNCTION_GENERETOR_DEPS pybind proto_desc executor layer tracer engine imperative_profiler imperative_flag)
  endif()
L
Leo Chen 已提交
120 121 122
  list(APPEND OP_FUNCTION_GENERETOR_DEPS ${GLOB_OP_LIB})
  list(APPEND OP_FUNCTION_GENERETOR_DEPS ${GLOB_OPERATOR_DEPS})

123
  if (WITH_NCCL OR WITH_RCCL)
L
Leo Chen 已提交
124
    list(APPEND OP_FUNCTION_GENERETOR_DEPS nccl_context)
125
  endif()
L
Leo Chen 已提交
126

127 128 129 130
  if(WITH_XPU_BKCL)
    list(APPEND OP_FUNCTION_GENERETOR_DEPS bkcl_context)
  endif(WITH_XPU_BKCL)

131 132 133 134
  if(WITH_ASCEND_CL)
    list(APPEND OP_FUNCTION_GENERETOR_DEPS hccl_context)
  endif(WITH_ASCEND_CL)

L
Leo Chen 已提交
135
  add_executable(op_function_generator op_function_generator.cc)
136
  target_link_libraries(op_function_generator ${OP_FUNCTION_GENERETOR_DEPS})
137 138
  add_executable(eager_op_function_generator eager_op_function_generator.cc)
  target_link_libraries(eager_op_function_generator ${OP_FUNCTION_GENERETOR_DEPS})
139

L
Leo Chen 已提交
140 141
  get_property (os_dependency_modules GLOBAL PROPERTY OS_DEPENDENCY_MODULES)
  target_link_libraries(op_function_generator ${os_dependency_modules})
142
  target_link_libraries(eager_op_function_generator ${os_dependency_modules})
143 144
  if(WITH_ROCM)
    target_link_libraries(op_function_generator ${ROCM_HIPRTC_LIB})
145
    target_link_libraries(eager_op_function_generator ${ROCM_HIPRTC_LIB})
146
  endif()
L
Leo Chen 已提交
147

148 149
  set(impl_file ${CMAKE_SOURCE_DIR}/paddle/fluid/pybind/op_function_impl.h)
  set(tmp_impl_file ${impl_file}.tmp)
150 151
  set(eager_impl_file ${CMAKE_SOURCE_DIR}/paddle/fluid/pybind/eager_op_function_impl.h)
  set(tmp_eager_impl_file ${eager_impl_file}.tmp)
152

153
  set(OP_IMPL_DEPS op_function_generator)
154 155
  set(EAGER_OP_IMPL_DEPS eager_op_function_generator)

156
  if(WIN32)
157
    if("${CMAKE_GENERATOR}" STREQUAL "Ninja")
158
      set(op_impl_path "${CMAKE_CURRENT_BINARY_DIR}")
159
    else()
160
      set(op_impl_path "${CMAKE_CURRENT_BINARY_DIR}/${CMAKE_BUILD_TYPE}")
161
    endif()
162

163 164 165 166
    file(WRITE ${CMAKE_BINARY_DIR}/paddle/fluid/pybind/op_function_generator_retry.bat ""
    "set build_times=1\n"
    ":retry\n"
    "ECHO op_function_generator run %build_times% time\n"
167
    "taskkill /f /im op_function_generator.exe 2>NUL\n"
168
    "${op_impl_path}/op_function_generator.exe ${tmp_impl_file}\n"
169 170
    "if %ERRORLEVEL% NEQ 0 (\n"
    "    set /a build_times=%build_times%+1\n"
Z
Zhou Wei 已提交
171
    "    if %build_times% GEQ 10 (\n"
172 173 174 175 176 177 178
    "        exit /b 1\n"
    "    ) else (\n"
    "        goto :retry\n"
    "    )\n"
    ")\n"
    "exit /b 0")

179 180 181 182 183
    file(WRITE ${CMAKE_BINARY_DIR}/paddle/fluid/pybind/eager_op_function_generator_retry.bat ""
    "set build_times=1\n"
    ":retry\n"
    "ECHO eager_op_function_generator run %build_times% time\n"
    "taskkill /f /im eager_op_function_generator.exe 2>NUL\n"
184
    "${op_impl_path}/eager_op_function_generator.exe ${tmp_eager_impl_file} ${PADDLE_SOURCE_DIR}/paddle/fluid/eager/auto_code_generator/op_list.txt\n"
185 186 187 188 189 190 191 192 193 194
    "if %ERRORLEVEL% NEQ 0 (\n"
    "    set /a build_times=%build_times%+1\n"
    "    if %build_times% GEQ 10 (\n"
    "        exit /b 1\n"
    "    ) else (\n"
    "        goto :retry\n"
    "    )\n"
    ")\n"
    "exit /b 0")

195
    if(${CBLAS_PROVIDER} STREQUAL MKLML)
196 197 198 199
      ADD_CUSTOM_COMMAND(OUTPUT ${op_impl_path}/libiomp5md.dll
        COMMAND ${CMAKE_COMMAND} -E copy ${MKLML_SHARED_IOMP_LIB} ${op_impl_path}
        DEPENDS mklml)
      list(APPEND OP_IMPL_DEPS ${op_impl_path}/libiomp5md.dll)
200
      list(APPEND EAGER_OP_IMPL_DEPS ${op_impl_path}/libiomp5md.dll)
201
    else(${CBLAS_PROVIDER} STREQUAL EXTERN_OPENBLAS)
202 203 204 205
      ADD_CUSTOM_COMMAND(OUTPUT ${op_impl_path}/openblas.dll
        COMMAND ${CMAKE_COMMAND} -E copy ${OPENBLAS_SHARED_LIB} ${op_impl_path}
        DEPENDS extern_openblas)
      list(APPEND OP_IMPL_DEPS ${op_impl_path}/openblas.dll)
206
      list(APPEND EAGER_OP_IMPL_DEPS ${op_impl_path}/openblas.dll)
207
    endif()
L
Leo Chen 已提交
208
    if(WITH_MKLDNN)
209 210 211 212
      ADD_CUSTOM_COMMAND(OUTPUT ${op_impl_path}/mkldnn.dll
        COMMAND ${CMAKE_COMMAND} -E copy ${MKLDNN_SHARED_LIB} ${op_impl_path}
        DEPENDS mkldnn)
        list(APPEND OP_IMPL_DEPS ${op_impl_path}/mkldnn.dll)
213
        list(APPEND EAGER_OP_IMPL_DEPS ${op_impl_path}/mkldnn.dll)
214
    endif()
215 216 217 218 219 220

    add_custom_command(OUTPUT ${impl_file}
      COMMAND ${CMAKE_BINARY_DIR}/paddle/fluid/pybind/op_function_generator_retry.bat
      COMMAND ${CMAKE_COMMAND} -E copy_if_different ${tmp_impl_file} ${impl_file}
      COMMENT "copy_if_different ${tmp_impl_file} to ${impl_file}"
      DEPENDS ${OP_IMPL_DEPS})
221 222 223 224 225 226 227
    if(NOT ON_INFER)
      add_custom_command(OUTPUT ${eager_impl_file}
        COMMAND ${CMAKE_BINARY_DIR}/paddle/fluid/pybind/eager_op_function_generator_retry.bat
        COMMAND ${CMAKE_COMMAND} -E copy_if_different ${tmp_eager_impl_file} ${eager_impl_file}
        COMMENT "copy_if_different ${tmp_eager_impl_file} to ${eager_impl_file}"
        DEPENDS ${EAGER_OP_IMPL_DEPS})
    endif()
L
Leo Chen 已提交
228
  else(WIN32)
229
    # If there are no *.so in /usr/lib or LD_LIBRARY_PATH,
L
Leo Chen 已提交
230
    # copy these *.so to current directory and append current directory to
231
    # LD_LIBRARY_PATH. This is different with Windows platformm, which search
L
Leo Chen 已提交
232
    # *.dll in current directory automatically.
233 234 235 236 237
    if(WITH_MKLML)
      ADD_CUSTOM_COMMAND(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/libiomp5.so
        COMMAND ${CMAKE_COMMAND} -E copy ${MKLML_SHARED_IOMP_LIB} ${CMAKE_CURRENT_BINARY_DIR}
        DEPENDS mklml)
      list(APPEND OP_IMPL_DEPS ${CMAKE_CURRENT_BINARY_DIR}/libiomp5.so)
238
      list(APPEND EAGER_OP_IMPL_DEPS ${CMAKE_CURRENT_BINARY_DIR}/libiomp5.so)
239 240 241 242 243 244
    endif()
    if(WITH_MKLDNN)
      ADD_CUSTOM_COMMAND(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/libdnnl.so.0
        COMMAND ${CMAKE_COMMAND} -E copy ${MKLDNN_SHARED_LIB} ${CMAKE_CURRENT_BINARY_DIR}
        DEPENDS mkldnn)
      list(APPEND OP_IMPL_DEPS ${CMAKE_CURRENT_BINARY_DIR}/libdnnl.so.0)
245
      list(APPEND EAGER_OP_IMPL_DEPS ${CMAKE_CURRENT_BINARY_DIR}/libdnnl.so.0)
246 247
    endif()
    add_custom_command(OUTPUT ${impl_file}
248
          COMMAND ${CMAKE_COMMAND} -E env "LD_LIBRARY_PATH=$ENV{LD_LIBRARY_PATH}:."
249
              "${CMAKE_CURRENT_BINARY_DIR}/op_function_generator"
250 251
              "${tmp_impl_file}"
          COMMAND ${CMAKE_COMMAND} -E copy_if_different ${tmp_impl_file} ${impl_file}
252
          COMMENT "copy_if_different ${tmp_impl_file} to ${impl_file}"
253 254
          DEPENDS ${OP_IMPL_DEPS}
          VERBATIM)
255 256 257 258
    if(NOT ON_INFER)
      add_custom_command(OUTPUT ${eager_impl_file}
            COMMAND ${CMAKE_COMMAND} -E env "LD_LIBRARY_PATH=$ENV{LD_LIBRARY_PATH}:."
                "${CMAKE_CURRENT_BINARY_DIR}/eager_op_function_generator"
259
                "${tmp_eager_impl_file}" "${PADDLE_SOURCE_DIR}/paddle/fluid/eager/auto_code_generator/op_list.txt"
260 261 262 263 264
            COMMAND ${CMAKE_COMMAND} -E copy_if_different ${tmp_eager_impl_file} ${eager_impl_file}
            COMMENT "copy_if_different ${tmp_eager_impl_file} to ${eager_impl_file}"
            DEPENDS ${EAGER_OP_IMPL_DEPS}
            VERBATIM)
      endif()
L
Leo Chen 已提交
265
  endif(WIN32)
266
  add_custom_target(op_function_generator_cmd ALL DEPENDS ${impl_file})
267 268 269
  if(NOT ON_INFER)
    add_custom_target(eager_op_function_generator_cmd ALL DEPENDS ${eager_impl_file})
  endif()
L
Leo Chen 已提交
270

271
  list(APPEND PYBIND_DEPS interpretercore standalone_executor)
272 273
  cc_library(op_function_common SRCS op_function_common.cc DEPS ${PYBIND_DEPS})
  list(APPEND PYBIND_DEPS op_function_common)
274

275 276 277 278 279 280 281 282
  if(NOT ON_INFER)
    cc_library(paddle_eager
    SRCS eager.cc eager_functions.cc eager_method.cc eager_properties.cc eager_utils.cc
    DEPS eager_api autograd_meta backward grad_node_info pten op_function_common dygraph_function dygraph_node math_cpu linalg_cpu creation_cpu utils_cpu manipulation_cpu accumulation_node global_utils utils python)
    add_dependencies(paddle_eager eager_codegen)
    add_dependencies(paddle_eager eager_op_function_generator_cmd)
    list(APPEND PYBIND_DEPS paddle_eager)
  endif()
283

284
  cc_library(paddle_pybind SHARED
285 286 287
    SRCS ${PYBIND_SRCS}
    DEPS ${PYBIND_DEPS} ${GLOB_OP_LIB} ${GLOB_OPERATOR_DEPS})

288 289 290 291 292 293 294
  if(NOT APPLE AND NOT WIN32)
    target_link_libraries(paddle_pybind rt)
  endif(NOT APPLE AND NOT WIN32)

  if(WITH_ROCM)
    target_link_libraries(paddle_pybind ${ROCM_HIPRTC_LIB})
  endif()
295

P
peizhilin 已提交
296 297
  get_property (os_dependency_modules GLOBAL PROPERTY OS_DEPENDENCY_MODULES)
  target_link_libraries(paddle_pybind ${os_dependency_modules})
298
  add_dependencies(paddle_pybind op_function_generator_cmd)
299
endif(WITH_PYTHON)