operators.cmake 25.7 KB
Newer Older
1 2
# CMake file `unity_build` is used to handle Unity Build compilation.
include(unity_build)
W
Wu Yi 已提交
3
set(PART_CUDA_KERNEL_FILES)
4 5

function(find_register FILENAME PATTERN OUTPUT)
6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26
  # find the op_name of REGISTER_OPERATOR(op_name, ...), REGISTER_OP_CPU_KERNEL(op_name, ...) , etc.
  # set op_name to OUTPUT
  set(options "")
  set(oneValueArgs "")
  set(multiValueArgs "")
  file(READ ${FILENAME} CONTENT)
  # message ("number of arguments sent to function: ${ARGC}")
  # message ("all function arguments:               ${ARGV}")
  # message("PATTERN ${PATTERN}")
  string(REGEX MATCH "${PATTERN}\\([ \t\r\n]*[a-z0-9_]*," register "${CONTENT}")
  if(NOT register STREQUAL "")
    string(REPLACE "${PATTERN}(" "" register "${register}")
    string(REPLACE "," "" register "${register}")
    # [ \t\r\n]+ is used for blank characters.
    # Here we use '+' instead of '*' since it is a REPLACE operation.
    string(REGEX REPLACE "[ \t\r\n]+" "" register "${register}")
  endif()

  set(${OUTPUT}
      ${register}
      PARENT_SCOPE)
27 28
endfunction()

29
function(find_phi_register FILENAME ADD_PATH PATTERN)
30 31 32 33 34
  # set op_name to OUTPUT
  file(READ ${FILENAME} CONTENT)
  string(
    REGEX
      MATCH
35
      "${PATTERN}\\([ \t\r\n]*[a-z0-9_]*,[[ \\\t\r\n\/]*[a-z0-9_]*]?[ \\\t\r\n]*[a-zA-Z]*,[ \\\t\r\n]*[A-Z_]*"
36 37 38
      register
      "${CONTENT}")
  if(NOT register STREQUAL "")
39
    string(REPLACE "${PATTERN}(" "" register "${register}")
40 41 42 43 44 45 46 47 48 49 50 51 52 53
    string(REPLACE "," ";" register "${register}")
    string(REGEX REPLACE "[ \\\t\r\n]+" "" register "${register}")
    string(REGEX REPLACE "//cuda_only" "" register "${register}")
    list(GET register 0 kernel_name)
    list(GET register 1 kernel_backend)
    list(GET register 2 kernel_layout)

    file(
      APPEND ${ADD_PATH}
      "PD_DECLARE_KERNEL(${kernel_name}, ${kernel_backend}, ${kernel_layout});\n"
    )
  endif()
endfunction()

W
Wu Yi 已提交
54
function(op_library TARGET)
55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82
  # op_library is a function to create op library. The interface is same as
  # cc_library. But it handle split GPU/CPU code and link some common library
  # for ops.
  set(cc_srcs)
  set(cu_srcs)
  set(hip_srcs)
  set(cu_cc_srcs)
  set(hip_cc_srcs)
  set(xpu_cc_srcs)
  set(xpu_kp_cc_srcs)
  set(npu_cc_srcs)
  set(mlu_cc_srcs)
  set(cudnn_cu_cc_srcs)
  set(miopen_cu_cc_srcs)
  set(cudnn_cu_srcs)
  set(miopen_cu_srcs)
  set(CUDNN_FILE)
  set(MIOPEN_FILE)
  set(mkldnn_cc_srcs)
  set(MKLDNN_FILE)
  set(op_common_deps operator op_registry math_function layer
                     common_infer_shape_functions)
  if(WITH_ASCEND_CL)
    set(op_common_deps ${op_common_deps} npu_op_runner)
  endif()
  if(WITH_MLU)
    set(op_common_deps ${op_common_deps} mlu_baseop)
  endif()
F
fwenguang 已提交
83

84 85 86 87 88 89 90
  # Option `UNITY` is used to specify that operator `TARGET` will compiles with Unity Build.
  set(options UNITY)
  set(oneValueArgs "")
  set(multiValueArgs SRCS DEPS)
  set(pybind_flag 0)
  cmake_parse_arguments(op_library "${options}" "${oneValueArgs}"
                        "${multiValueArgs}" ${ARGN})
W
Wu Yi 已提交
91

92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182
  list(LENGTH op_library_SRCS op_library_SRCS_len)
  if(${op_library_SRCS_len} EQUAL 0)
    if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${TARGET}.cc)
      list(APPEND cc_srcs ${TARGET}.cc)
    endif()
    if(WITH_GPU)
      if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${TARGET}.cu.cc)
        list(APPEND cu_cc_srcs ${TARGET}.cu.cc)
      endif()
      if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${TARGET}.cu)
        list(APPEND cu_srcs ${TARGET}.cu)
      endif()
      # rename in KP: .kps -> .cu
      if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${TARGET}.kps)
        file(COPY ${TARGET}.kps DESTINATION ${CMAKE_CURRENT_BINARY_DIR})
        file(RENAME ${CMAKE_CURRENT_BINARY_DIR}/${TARGET}.kps
             ${CMAKE_CURRENT_BINARY_DIR}/${TARGET}.cu)
        list(APPEND cu_srcs ${CMAKE_CURRENT_BINARY_DIR}/${TARGET}.cu)
      endif()
      if(WITH_NV_JETSON)
        list(REMOVE_ITEM cu_srcs "decode_jpeg_op.cu")
      endif()
      if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${TARGET}.part.cu)
        set(PART_CUDA_KERNEL_FILES
            ${CMAKE_CURRENT_SOURCE_DIR}/${TARGET}.part.cu
            ${PART_CUDA_KERNEL_FILES}
            PARENT_SCOPE)
        list(APPEND cu_srcs ${CMAKE_CURRENT_SOURCE_DIR}/${TARGET}.part.cu)
      endif()
      string(REPLACE "_op" "_cudnn_op" CUDNN_FILE "${TARGET}")
      if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${CUDNN_FILE}.cu.cc)
        list(APPEND cudnn_cu_cc_srcs ${CUDNN_FILE}.cu.cc)
      endif()
      if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${CUDNN_FILE}.cu)
        list(APPEND cudnn_cu_srcs ${CUDNN_FILE}.cu)
      endif()
    endif()
    if(WITH_ROCM)
      if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${TARGET}.cu.cc)
        list(APPEND hip_cc_srcs ${TARGET}.cu.cc)
      endif()
      if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${TARGET}.cu)
        list(APPEND hip_srcs ${TARGET}.cu)
      endif()
      # rename in KP: .kps -> .cu
      if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${TARGET}.kps)
        file(COPY ${TARGET}.kps DESTINATION ${CMAKE_CURRENT_BINARY_DIR})
        file(RENAME ${CMAKE_CURRENT_BINARY_DIR}/${TARGET}.kps
             ${CMAKE_CURRENT_BINARY_DIR}/${TARGET}.cu)
        list(APPEND hip_srcs ${CMAKE_CURRENT_BINARY_DIR}/${TARGET}.cu)
      endif()
      if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${TARGET}.part.cu)
        set(PART_CUDA_KERNEL_FILES
            ${CMAKE_CURRENT_SOURCE_DIR}/${TARGET}.part.cu
            ${PART_CUDA_KERNEL_FILES}
            PARENT_SCOPE)
        list(APPEND hip_srcs ${CMAKE_CURRENT_SOURCE_DIR}/${TARGET}.part.cu)
      endif()
      string(REPLACE "_op" "_cudnn_op" MIOPEN_FILE "${TARGET}")
      if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${MIOPEN_FILE}.cu.cc)
        list(APPEND miopen_cu_cc_srcs ${MIOPEN_FILE}.cu.cc)
      endif()
      if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${MIOPEN_FILE}.cu)
        list(APPEND miopen_cu_srcs ${MIOPEN_FILE}.cu)
      endif()
    endif()
    if(WITH_MKLDNN)
      string(REPLACE "_op" "_mkldnn_op" MKLDNN_FILE "${TARGET}")
      if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/mkldnn/${MKLDNN_FILE}.cc)
        list(APPEND mkldnn_cc_srcs mkldnn/${MKLDNN_FILE}.cc)
      endif()
    endif()
    if(WITH_XPU)
      string(REPLACE "_op" "_op_xpu" XPU_FILE "${TARGET}")
      if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${XPU_FILE}.cc)
        list(APPEND xpu_cc_srcs ${XPU_FILE}.cc)
      endif()
    endif()
    if(WITH_XPU_KP)
      if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${TARGET}.xpu)
        list(APPEND xpu_kp_cc_srcs ${TARGET}.xpu)
      endif()
      if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${TARGET}.kps)
        list(APPEND xpu_kp_cc_srcs ${TARGET}.kps)
      endif()
    endif()
    if(WITH_ASCEND_CL)
      string(REPLACE "_op" "_op_npu" NPU_FILE "${TARGET}")
      if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${NPU_FILE}.cc)
        list(APPEND npu_cc_srcs ${NPU_FILE}.cc)
      endif()
W
Wu Yi 已提交
183
    endif()
184 185 186 187 188
    if(WITH_MLU)
      string(REPLACE "_op" "_op_mlu" MLU_FILE "${TARGET}")
      if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${MLU_FILE}.cc)
        list(APPEND mlu_cc_srcs ${MLU_FILE}.cc)
      endif()
W
Wu Yi 已提交
189
    endif()
190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221
  else()
    foreach(src ${op_library_SRCS})
      if(WITH_ROCM AND ${src} MATCHES ".*_cudnn_op.cu$")
        list(APPEND miopen_cu_srcs ${src})
      elseif(WITH_ROCM AND ${src} MATCHES ".*\\.cu$")
        list(APPEND hip_srcs ${src})
      elseif(WITH_ROCM AND ${src} MATCHES ".*_cudnn_op.cu.cc$")
        list(APPEND miopen_cu_cc_srcs ${src})
      elseif(WITH_ROCM AND ${src} MATCHES ".*\\.cu.cc$")
        list(APPEND hip_cc_srcs ${src})
      elseif(WITH_GPU AND ${src} MATCHES ".*_cudnn_op.cu$")
        list(APPEND cudnn_cu_srcs ${src})
      elseif(WITH_GPU AND ${src} MATCHES ".*\\.cu$")
        list(APPEND cu_srcs ${src})
      elseif(WITH_GPU AND ${src} MATCHES ".*_cudnn_op.cu.cc$")
        list(APPEND cudnn_cu_cc_srcs ${src})
      elseif(WITH_GPU AND ${src} MATCHES ".*\\.cu.cc$")
        list(APPEND cu_cc_srcs ${src})
      elseif(WITH_MKLDNN AND ${src} MATCHES ".*_mkldnn_op.cc$")
        list(APPEND mkldnn_cc_srcs ${src})
      elseif(WITH_XPU AND ${src} MATCHES ".*_op_xpu.cc$")
        list(APPEND xpu_cc_srcs ${src})
      elseif(WITH_XPU_KP AND ${src} MATCHES ".*\\.xpu$")
        list(APPEND xpu_kp_cc_srcs ${src})
      elseif(WITH_XPU_KP AND ${src} MATCHES ".*\\.kps$")
        list(APPEND xpu_kp_cc_srcs ${src})
      elseif(WITH_ASCEND_CL AND ${src} MATCHES ".*_op_npu.cc$")
        list(APPEND npu_cc_srcs ${src})
      elseif(WITH_MLU AND ${src} MATCHES ".*_op_mlu.cc$")
        list(APPEND mlu_cc_srcs ${src})
      elseif(${src} MATCHES ".*\\.cc$")
        list(APPEND cc_srcs ${src})
H
huangjiyi 已提交
222 223 224 225 226 227 228 229 230 231
      elseif((WITH_ROCM OR WITH_GPU) AND ${src} MATCHES ".*\\.kps$")
        string(REPLACE ".kps" ".cu" src_cu ${src})
        file(COPY ${src} DESTINATION ${CMAKE_CURRENT_BINARY_DIR})
        file(RENAME ${CMAKE_CURRENT_BINARY_DIR}/${src}
             ${CMAKE_CURRENT_BINARY_DIR}/${src_cu})
        if(WITH_ROCM)
          list(APPEND hip_srcs ${CMAKE_CURRENT_BINARY_DIR}/${src_cu})
        else()
          list(APPEND cu_srcs ${CMAKE_CURRENT_BINARY_DIR}/${src_cu})
        endif()
232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248
      else()
        message(
          FATAL_ERROR
            "${TARGET} Source file ${src} should only be .cc or .cu or .xpu")
      endif()
    endforeach()
  endif()

  list(LENGTH xpu_cc_srcs xpu_cc_srcs_len)
  list(LENGTH xpu_kp_cc_srcs xpu_kp_cc_srcs_len)
  list(LENGTH cc_srcs cc_srcs_len)
  if(${cc_srcs_len} EQUAL 0)
    message(
      FATAL_ERROR
        "The op library ${TARGET} should contains at least one .cc file")
  endif()
  if(WIN32)
W
Wu Yi 已提交
249
    # remove windows unsupported op, because windows has no nccl, no warpctc such ops.
P
peizhilin 已提交
250
    foreach(windows_unsupport_op "nccl_op" "gen_nccl_id_op")
251 252 253
      if("${TARGET}" STREQUAL "${windows_unsupport_op}")
        return()
      endif()
W
Wu Yi 已提交
254
    endforeach()
W
Wilber 已提交
255
  endif()
256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273

  # Unity Build relies on global option `WITH_UNITY_BUILD` and local option `UNITY`.
  if(WITH_UNITY_BUILD AND op_library_UNITY)
    # Generate the unity target name by the directory where source files located.
    string(REPLACE "${PADDLE_SOURCE_DIR}/paddle/fluid/" "" UNITY_TARGET
                   ${CMAKE_CURRENT_SOURCE_DIR})
    string(REPLACE "/" "_" UNITY_TARGET ${UNITY_TARGET})
    set(UNITY_TARGET "paddle_${UNITY_TARGET}_unity")
    if(NOT ${UNITY_TARGET} IN_LIST OP_LIBRARY)
      set(OP_LIBRARY
          ${UNITY_TARGET} ${OP_LIBRARY}
          CACHE INTERNAL "op libs")
    endif()
  else()
    set(OP_LIBRARY
        ${TARGET} ${OP_LIBRARY}
        CACHE INTERNAL "op libs")
  endif()
274

275 276 277 278 279 280 281
  list(LENGTH op_library_DEPS op_library_DEPS_len)
  if(${op_library_DEPS_len} GREATER 0)
    set(DEPS_OPS
        ${TARGET} ${DEPS_OPS}
        PARENT_SCOPE)
  endif()
  if(WITH_GPU)
282 283
    # Unity Build relies on global option `WITH_UNITY_BUILD` and local option `UNITY`.
    if(WITH_UNITY_BUILD AND op_library_UNITY)
284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301
      # Combine the cc and cu source files.
      compose_unity_target_sources(${UNITY_TARGET} cc ${cc_srcs} ${cu_cc_srcs}
                                   ${cudnn_cu_cc_srcs} ${mkldnn_cc_srcs})
      compose_unity_target_sources(${UNITY_TARGET} cu ${cudnn_cu_srcs}
                                   ${cu_srcs})
      if(TARGET ${UNITY_TARGET})
        # If `UNITY_TARGET` exists, add source files to `UNITY_TARGET`.
        target_sources(${UNITY_TARGET} PRIVATE ${unity_target_cc_sources}
                                               ${unity_target_cu_sources})
      else()
        # If `UNITY_TARGET` does not exist, create `UNITY_TARGET` with source files.
        nv_library(
          ${UNITY_TARGET}
          SRCS ${unity_target_cc_sources} ${unity_target_cu_sources}
          DEPS ${op_library_DEPS} ${op_common_deps})
      endif()
      # Add alias library to handle dependencies.
      add_library(${TARGET} ALIAS ${UNITY_TARGET})
302
    else()
303 304 305 306 307
      nv_library(
        ${TARGET}
        SRCS ${cc_srcs} ${cu_cc_srcs} ${cudnn_cu_cc_srcs} ${cudnn_cu_srcs}
             ${mkldnn_cc_srcs} ${cu_srcs}
        DEPS ${op_library_DEPS} ${op_common_deps})
308
    endif()
309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339
  elseif(WITH_ROCM)
    list(REMOVE_ITEM miopen_cu_cc_srcs "affine_grid_cudnn_op.cu.cc")
    list(REMOVE_ITEM miopen_cu_cc_srcs "grid_sampler_cudnn_op.cu.cc")
    list(REMOVE_ITEM hip_srcs "cholesky_op.cu")
    list(REMOVE_ITEM hip_srcs "cholesky_solve_op.cu")
    list(REMOVE_ITEM hip_srcs "lu_op.cu")
    list(REMOVE_ITEM hip_srcs "matrix_rank_op.cu")
    list(REMOVE_ITEM hip_srcs "svd_op.cu")
    list(REMOVE_ITEM hip_srcs "eigvalsh_op.cu")
    list(REMOVE_ITEM hip_srcs "qr_op.cu")
    list(REMOVE_ITEM hip_srcs "eigh_op.cu")
    list(REMOVE_ITEM hip_srcs "lstsq_op.cu")
    list(REMOVE_ITEM hip_srcs "multinomial_op.cu")
    list(REMOVE_ITEM hip_srcs "decode_jpeg_op.cu")
    hip_library(
      ${TARGET}
      SRCS ${cc_srcs} ${hip_cc_srcs} ${miopen_cu_cc_srcs} ${miopen_cu_srcs}
           ${mkldnn_cc_srcs} ${hip_srcs}
      DEPS ${op_library_DEPS} ${op_common_deps})
  elseif(WITH_XPU_KP AND ${xpu_kp_cc_srcs_len} GREATER 0)
    xpu_library(
      ${TARGET}
      SRCS ${cc_srcs} ${mkldnn_cc_srcs} ${xpu_cc_srcs} ${xpu_kp_cc_srcs}
      DEPS ${op_library_DEPS} ${op_common_deps})
  else()
    # deal with CANN version control while registering NPU operators before build
    if(WITH_ASCEND_CL)
      if(CANN_VERSION LESS 504000)
        list(REMOVE_ITEM npu_cc_srcs "multinomial_op_npu.cc")
        list(REMOVE_ITEM npu_cc_srcs "take_along_axis_op_npu.cc")
      endif()
W
Wu Yi 已提交
340
    endif()
341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363
    # Unity Build relies on global option `WITH_UNITY_BUILD` and local option `UNITY`.
    if(WITH_UNITY_BUILD AND op_library_UNITY)
      # Combine the cc source files.
      compose_unity_target_sources(
        ${UNITY_TARGET}
        cc
        ${cc_srcs}
        ${mkldnn_cc_srcs}
        ${xpu_cc_srcs}
        ${npu_cc_srcs}
        ${mlu_cc_srcs})
      if(TARGET ${UNITY_TARGET})
        # If `UNITY_TARGET` exists, add source files to `UNITY_TARGET`.
        target_sources(${UNITY_TARGET} PRIVATE ${unity_target_cc_sources})
      else()
        # If `UNITY_TARGET` does not exist, create `UNITY_TARGET` with source files.
        cc_library(
          ${UNITY_TARGET}
          SRCS ${unity_target_cc_sources}
          DEPS ${op_library_DEPS} ${op_common_deps})
      endif()
      # Add alias library to handle dependencies.
      add_library(${TARGET} ALIAS ${UNITY_TARGET})
W
Wu Yi 已提交
364
    else()
365 366 367 368 369
      cc_library(
        ${TARGET}
        SRCS ${cc_srcs} ${mkldnn_cc_srcs} ${xpu_cc_srcs} ${npu_cc_srcs}
             ${mlu_cc_srcs}
        DEPS ${op_library_DEPS} ${op_common_deps})
W
Wu Yi 已提交
370
    endif()
371
  endif()
W
Wu Yi 已提交
372

373 374 375 376 377 378 379 380 381
  list(LENGTH cu_srcs cu_srcs_len)
  list(LENGTH hip_srcs hip_srcs_len)
  list(LENGTH cu_cc_srcs cu_cc_srcs_len)
  list(LENGTH hip_cc_srcs hip_cc_srcs_len)
  list(LENGTH mkldnn_cc_srcs mkldnn_cc_srcs_len)
  list(LENGTH xpu_cc_srcs xpu_cc_srcs_len)
  list(LENGTH miopen_cu_cc_srcs miopen_cu_cc_srcs_len)
  list(LENGTH npu_cc_srcs npu_cc_srcs_len)
  list(LENGTH mlu_cc_srcs mlu_cc_srcs_len)
382

383 384 385 386 387 388 389 390 391 392 393
  # Define operators that don't need pybind here.
  foreach(
    manual_pybind_op
    "compare_all_op"
    "compare_op"
    "logical_op"
    "bitwise_op"
    "nccl_op"
    "tensor_array_read_write_op"
    "tensorrt_engine_op"
    "conv_fusion_op")
394

395 396 397 398
    if("${TARGET}" STREQUAL "${manual_pybind_op}")
      set(pybind_flag 1)
    endif()
  endforeach()
W
Wu Yi 已提交
399

400 401 402 403 404
  # The registration of USE_OP, please refer to paddle/fluid/framework/op_registry.h.
  # Note that it's enough to just adding one operator to pybind in a *_op.cc file.
  # And for detail pybind information, please see generated paddle/pybind/pybind.h.
  set(ORIGINAL_TARGET ${TARGET})
  string(REGEX REPLACE "_op" "" TARGET "${TARGET}")
405

406 407 408
  foreach(cc_src ${cc_srcs})
    # pybind USE_OP_ITSELF
    set(op_name "")
409
    # Add PHI Kernel Registry Message
410
    find_phi_register(${cc_src} ${pybind_file} "PD_REGISTER_KERNEL")
411
    find_phi_register(${cc_src} ${pybind_file} "PD_REGISTER_STRUCT_KERNEL")
412
    find_phi_register(${cc_src} ${pybind_file} "PD_REGISTER_GENERAL_KERNEL")
413 414 415 416 417 418 419
    find_register(${cc_src} "REGISTER_OPERATOR" op_name)
    if(NOT ${op_name} EQUAL "")
      file(APPEND ${pybind_file} "USE_OP_ITSELF(${op_name});\n")
      # hack: for example, the target in conv_transpose_op.cc is conv2d_transpose, used in mkldnn
      set(TARGET ${op_name})
      set(pybind_flag 1)
    endif()
W
Wu Yi 已提交
420

421 422 423 424 425 426 427 428 429 430 431
    # pybind USE_OP_ITSELF
    set(op_name "")
    # Add PHI Kernel Registry Message
    find_register(${cc_src} "REGISTER_ACTIVATION_OP" op_name)
    if(NOT ${op_name} EQUAL "")
      file(APPEND ${pybind_file} "USE_OP_ITSELF(${op_name});\n")
      # hack: for example, the target in conv_transpose_op.cc is conv2d_transpose, used in mkldnn
      set(TARGET ${op_name})
      set(pybind_flag 1)
    endif()

432 433 434 435 436 437 438 439
    set(op_name "")
    find_register(${cc_src} "REGISTER_OP_WITHOUT_GRADIENT" op_name)
    if(NOT ${op_name} EQUAL "")
      file(APPEND ${pybind_file} "USE_OP_ITSELF(${op_name});\n")
      # hack: for example, the target in conv_transpose_op.cc is conv2d_transpose, used in mkldnn
      set(TARGET ${op_name})
      set(pybind_flag 1)
    endif()
W
Wu Yi 已提交
440

441 442 443 444 445 446 447 448 449 450 451 452 453 454
    # pybind USE_OP_DEVICE_KERNEL for CPU
    set(op_name "")
    find_register(${cc_src} "REGISTER_OP_CPU_KERNEL" op_name)
    if(NOT ${op_name} EQUAL "")
      file(APPEND ${pybind_file} "USE_OP_DEVICE_KERNEL(${op_name}, CPU);\n")
      # why change TARGET here?
      # when building padle with on_infer, the REGISTER_OPERATOR(*_grad) will be removed before compiling (see details in remove_grad_op_and_kernel.py)
      # in elementwise_op.cc, it will find REGISTER_OPERATOR(grad_add) and set TARGET to grad_add
      # and, in the following "mkldnn" part, it will add USE_OP_DEVICE_KERNEL(grad_add, MKLDNN) to pybind.h
      # however, grad_add has no mkldnn kernel.
      set(TARGET ${op_name})
      set(pybind_flag 1)
    endif()
  endforeach()
W
Wu Yi 已提交
455

456 457 458 459 460
  # pybind USE_OP_DEVICE_KERNEL for CUDA
  list(APPEND cu_srcs ${cu_cc_srcs})
  # message("cu_srcs ${cu_srcs}")
  foreach(cu_src ${cu_srcs})
    set(op_name "")
461
    # Add PHI Kernel Registry Message
462
    find_phi_register(${cu_src} ${pybind_file} "PD_REGISTER_KERNEL")
463
    find_phi_register(${cu_src} ${pybind_file} "PD_REGISTER_STRUCT_KERNEL")
464
    find_phi_register(${cu_src} ${pybind_file} "PD_REGISTER_GENERAL_KERNEL")
465 466 467 468 469 470
    find_register(${cu_src} "REGISTER_OP_CUDA_KERNEL" op_name)
    if(NOT ${op_name} EQUAL "")
      file(APPEND ${pybind_file} "USE_OP_DEVICE_KERNEL(${op_name}, CUDA);\n")
      set(pybind_flag 1)
    endif()
  endforeach()
W
Wu Yi 已提交
471

472 473 474 475 476 477 478 479 480
  # pybind USE_OP_DEVICE_KERNEL for ROCm
  list(APPEND hip_srcs ${hip_cc_srcs})
  # message("hip_srcs ${hip_srcs}")
  foreach(hip_src ${hip_srcs})
    set(op_name "")
    find_register(${hip_src} "REGISTER_OP_CUDA_KERNEL" op_name)
    if(NOT ${op_name} EQUAL "")
      file(APPEND ${pybind_file} "USE_OP_DEVICE_KERNEL(${op_name}, CUDA);\n")
      set(pybind_flag 1)
Y
Y_Xuan 已提交
481
    endif()
482
  endforeach()
Y
Y_Xuan 已提交
483

484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502
  # pybind USE_OP_DEVICE_KERNEL for CUDNN/MIOPEN
  list(APPEND cudnn_cu_srcs ${cudnn_cu_cc_srcs})
  list(APPEND cudnn_cu_srcs ${miopen_cu_cc_srcs})
  list(APPEND cudnn_cu_srcs ${miopen_cu_srcs})
  list(LENGTH cudnn_cu_srcs cudnn_cu_srcs_len)
  #message("cudnn_cu_srcs ${cudnn_cu_srcs}")
  if(${cudnn_cu_srcs_len} GREATER 0 AND ${ORIGINAL_TARGET} STREQUAL
                                        "activation_op")
    file(APPEND ${pybind_file} "USE_OP_DEVICE_KERNEL(relu, CUDNN);\n")
  else()
    foreach(cudnn_src ${cudnn_cu_srcs})
      set(op_name "")
      find_register(${cudnn_src} "REGISTER_OP_KERNEL" op_name)
      if(NOT ${op_name} EQUAL "")
        file(APPEND ${pybind_file} "USE_OP_DEVICE_KERNEL(${op_name}, CUDNN);\n")
        set(pybind_flag 1)
      endif()
    endforeach()
  endif()
503

504
  if(WITH_XPU AND ${xpu_cc_srcs_len} GREATER 0)
505
    if(${ORIGINAL_TARGET} STREQUAL "activation_op")
506
      file(APPEND ${pybind_file} "USE_OP_DEVICE_KERNEL(relu, XPU);\n")
507
    else()
508
      foreach(xpu_src ${xpu_cc_srcs})
509 510 511
        set(op_name "")
        find_register(${xpu_src} "REGISTER_OP_XPU_KERNEL" op_name)
        if(NOT ${op_name} EQUAL "")
512 513
          file(APPEND ${pybind_file} "USE_OP_DEVICE_KERNEL(${op_name}, XPU);\n")
          set(pybind_flag 1)
514
        else()
515 516 517 518
          find_register(${xpu_src} "REGISTER_OP_XPU_KERNEL_FUNCTOR" op_name)
          if(NOT ${op_name} EQUAL "")
            file(APPEND ${pybind_file}
                 "USE_OP_DEVICE_KERNEL(${op_name}, XPU);\n")
519
            set(pybind_flag 1)
520
          endif()
521
        endif()
522
      endforeach()
L
Liu-xiandong 已提交
523
    endif()
524
  endif()
L
Liu-xiandong 已提交
525

526 527 528 529 530
  # pybind USE_OP_DEVICE_KERNEL for XPU KP
  if(WITH_XPU_KP AND ${xpu_kp_cc_srcs_len} GREATER 0)
    foreach(xpu_kp_src ${xpu_kp_cc_srcs})
      set(op_name "")
      find_register(${xpu_kp_src} "REGISTER_OP_KERNEL" op_name)
531 532
      find_phi_register(${xpu_kp_src} ${pybind_file}
                        "PD_REGISTER_STRUCT_KERNEL")
533 534 535 536 537 538 539
      if(NOT ${op_name} EQUAL "")
        file(APPEND ${pybind_file} "USE_OP_DEVICE_KERNEL(${op_name}, KP);\n")
        message(STATUS "Building KP Target: ${op_name}")
        set(pybind_flag 1)
      endif()
    endforeach()
  endif()
F
fwenguang 已提交
540

541 542 543 544 545 546 547 548 549 550 551
  # pybind USE_OP_DEVICE_KERNEL for NPU
  if(WITH_ASCEND_CL AND ${npu_cc_srcs_len} GREATER 0)
    foreach(npu_src ${npu_cc_srcs})
      set(op_name "")
      find_register(${npu_src} "REGISTER_OP_NPU_KERNEL" op_name)
      if(NOT ${op_name} EQUAL "")
        file(APPEND ${pybind_file} "USE_OP_DEVICE_KERNEL(${op_name}, NPU);\n")
        set(pybind_flag 1)
      endif()
    endforeach()
  endif()
552

553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568
  # pybind USE_OP_DEVICE_KERNEL for MLU
  if(WITH_MLU AND ${mlu_cc_srcs_len} GREATER 0)
    foreach(mlu_src ${mlu_cc_srcs})
      set(op_name "")
      find_register(${mlu_src} "REGISTER_OP_MLU_KERNEL" op_name)
      if(NOT ${op_name} EQUAL "")
        file(APPEND ${pybind_file} "USE_OP_DEVICE_KERNEL(${op_name}, MLU);\n")
        set(pybind_flag 1)
      endif()
    endforeach()
  endif()

  # pybind USE_OP_DEVICE_KERNEL for MKLDNN
  if(WITH_MKLDNN AND ${mkldnn_cc_srcs_len} GREATER 0)
    # Append first implemented MKLDNN activation operator
    if(${MKLDNN_FILE} STREQUAL "activation_mkldnn_op")
569
      file(APPEND ${pybind_file} "USE_OP_DEVICE_KERNEL(softplus, MKLDNN);\n")
570 571
    else()
      foreach(mkldnn_src ${mkldnn_cc_srcs})
572 573 574
        set(op_name "")
        find_register(${mkldnn_src} "REGISTER_OP_KERNEL" op_name)
        if(NOT ${op_name} EQUAL "")
575 576 577
          file(APPEND ${pybind_file}
               "USE_OP_DEVICE_KERNEL(${op_name}, MKLDNN);\n")
          set(pybind_flag 1)
578
        endif()
579
      endforeach()
W
Wu Yi 已提交
580
    endif()
581
  endif()
W
Wu Yi 已提交
582

583 584 585 586 587 588 589 590
  # pybind USE_NO_KERNEL_OP
  # HACK: if REGISTER_OP_CPU_KERNEL presents the operator must have kernel
  string(REGEX MATCH "REGISTER_OP_CPU_KERNEL" regex_result "${TARGET_CONTENT}")
  string(REPLACE "_op" "" TARGET "${TARGET}")
  if(${pybind_flag} EQUAL 0 AND regex_result STREQUAL "")
    file(APPEND ${pybind_file} "USE_NO_KERNEL_OP(${TARGET});\n")
    set(pybind_flag 1)
  endif()
591

592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607
  # pybind USE_OP
  if(${pybind_flag} EQUAL 0)
    # NOTE(*): activation use macro to regist the kernels, set use_op manually.
    if(${TARGET} STREQUAL "activation")
      file(APPEND ${pybind_file} "USE_OP_ITSELF(relu);\n")
    elseif(${TARGET} STREQUAL "fake_dequantize")
      file(APPEND ${pybind_file} "USE_OP(fake_dequantize_max_abs);\n")
    elseif(${TARGET} STREQUAL "fake_quantize")
      file(APPEND ${pybind_file} "USE_OP(fake_quantize_abs_max);\n")
    elseif(${TARGET} STREQUAL "tensorrt_engine_op")
      message(
        STATUS
          "Pybind skips [tensorrt_engine_op], for this OP is only used in inference"
      )
    else()
      file(APPEND ${pybind_file} "USE_OP(${TARGET});\n")
W
Wu Yi 已提交
608
    endif()
609
  endif()
W
Wu Yi 已提交
610 611 612
endfunction()

function(register_operators)
613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628
  set(options "")
  set(oneValueArgs "")
  set(multiValueArgs EXCLUDES DEPS)
  cmake_parse_arguments(register_operators "${options}" "${oneValueArgs}"
                        "${multiValueArgs}" ${ARGN})
  file(
    GLOB OPS
    RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}"
    "*_op.cc")
  string(REPLACE "_mkldnn" "" OPS "${OPS}")
  string(REPLACE "_xpu" "" OPS "${OPS}")
  string(REPLACE "_npu" "" OPS "${OPS}")
  string(REPLACE "_mlu" "" OPS "${OPS}")
  string(REPLACE ".cc" "" OPS "${OPS}")
  list(REMOVE_DUPLICATES OPS)
  list(LENGTH register_operators_DEPS register_operators_DEPS_len)
W
Wu Yi 已提交
629

630 631 632 633 634 635 636 637 638 639
  foreach(src ${OPS})
    list(FIND register_operators_EXCLUDES ${src} _index)
    if(${_index} EQUAL -1)
      if(${register_operators_DEPS_len} GREATER 0)
        op_library(${src} UNITY DEPS ${register_operators_DEPS})
      else()
        op_library(${src} UNITY)
      endif()
    endif()
  endforeach()
640

641 642 643 644 645
  # Complete the processing of `UNITY_TARGET`.
  if(WITH_UNITY_BUILD)
    finish_unity_target(cc)
    if(WITH_GPU)
      finish_unity_target(cu)
646
    endif()
647
  endif()
W
Wu Yi 已提交
648
endfunction()
649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703

function(prune_pybind_h)
  set(op_list ${OP_LIST})

  list(APPEND op_list "load_combine")
  list(APPEND op_list "tensorrt_engine")

  # add fused_op in op_list
  list(APPEND op_list "fc")
  list(APPEND op_list "conv2d_fusion")
  list(APPEND op_list "fusion_seqconv_eltadd_relu")
  list(APPEND op_list "fusion_seqpool_cvm_concat")
  list(APPEND op_list "fusion_gru")
  list(APPEND op_list "fusion_seqexpand_concat_fc")
  list(APPEND op_list "fusion_repeated_fc_relu")
  list(APPEND op_list "fusion_squared_mat_sub")

  # add plugin_op in op_list
  list(APPEND op_list "anchor_generator")

  file(STRINGS ${pybind_file} op_registry_list)

  file(WRITE ${pybind_file_prune} "")
  file(
    APPEND ${pybind_file_prune}
    "// Generated by the paddle/fluid/operators/CMakeLists.txt.  DO NOT EDIT!\n"
  )

  # add USE_OP_ITSELF for all op in op_list
  foreach(op_name IN LISTS op_list)
    file(APPEND ${pybind_file_prune} "USE_OP_ITSELF(${op_name});\n")
  endforeach()

  foreach(op_registry IN LISTS op_registry_list)
    if(NOT ${op_registry} EQUAL "")
      foreach(op_name IN LISTS op_list)
        string(FIND ${op_registry} "(${op_name})" index1)
        string(FIND ${op_registry} "(${op_name}," index2)
        string(FIND ${op_registry} "USE_OP_ITSELF" index3)
        if(((NOT ${index1} EQUAL "-1") OR (NOT ${index2} EQUAL "-1"))
           AND (${index3} EQUAL "-1"))
          file(APPEND ${pybind_file_prune} "${op_registry}\n")
        endif()
      endforeach()
    endif()
  endforeach()

  file(WRITE ${pybind_file} "")
  file(STRINGS ${pybind_file_prune} op_registry_list_tmp)
  foreach(op_name IN LISTS op_registry_list_tmp)
    if(NOT ${op_name} EQUAL "")
      file(APPEND ${pybind_file} "${op_name}\n")
    endif()
  endforeach()
endfunction()