refactor lite cmake (#1823)

move cmake functions to lite.cmake

CMakeLists.txt

@@ -44,7 +44,6 @@ find_package(Threads REQUIRED)
 include(simd)
 
 ################################ Exposed Configurations #######################################
-option(WITH_GPU         "Compile PaddlePaddle with NVIDIA GPU"          ${CUDA_FOUND})
 option(WITH_DSO         "Compile PaddlePaddle with dynamic linked CUDA" ON)
 option(WITH_AVX         "Compile PaddlePaddle with AVX intrinsics"      ${AVX_FOUND})
 option(WITH_PYTHON      "Compile PaddlePaddle with python interpreter"  ON)
@@ -52,14 +51,6 @@ option(WITH_TESTING     "Compile PaddlePaddle with unit testing"        OFF)
 option(WITH_MKL         "Compile PaddlePaddle with MKL support."        ${AVX_FOUND})
 option(WITH_ARM_DOTPROD "Compile PaddlePaddle with ARM dot production"  ON)
 option(WITH_SYSTEM_BLAS   "Use system blas library"           OFF)
-option(WITH_DISTRIBUTE  "Compile with distributed support"              OFF)
-option(WITH_BRPC_RDMA     "Use brpc rdma as the rpc protocal"           OFF)
-################################ Internal Configurations #######################################
-option(WITH_PROFILER    "Compile PaddlePaddle with GPU profiler and gperftools"        OFF)
-option(WITH_JEMALLOC    "Compile PaddlePaddle with jemalloc"            OFF)
-option(WITH_COVERAGE    "Compile PaddlePaddle with code coverage"       OFF)
-option(COVERALLS_UPLOAD "Package code coverage data to coveralls"       OFF)
-option(REPLACE_ENFORCE_GLOG "Replace PADDLE_ENFORCE with glog/CHECK for better debug." OFF)
 # TODO(Superjomn) Remove WITH_ANAKIN option if not needed latter.
 if(ANDROID OR IOS OR ARMLINUX)
     set(WITH_GPU OFF CACHE STRING

cmake/configure.cmake

@@ -61,7 +61,7 @@ if(WITH_PSLIB)
     add_definitions(-DPADDLE_WITH_PSLIB)
 endif()
 
-if(WITH_GPU)
+if(LITE_WITH_GPU)
     add_definitions(-DPADDLE_WITH_CUDA)
     add_definitions(-DEIGEN_USE_GPU)
 

cmake/lite.cmake

+set(LITE_URL "http://paddle-inference-dist.bj.bcebos.com" CACHE STRING "inference download url")
+
+function(lite_download_and_uncompress INSTALL_DIR URL FILENAME)
+    message(STATUS "Download inference test stuff from ${URL}/${FILENAME}")
+    string(REGEX REPLACE "[-%.]" "_" FILENAME_EX ${FILENAME})
+    set(EXTERNAL_PROJECT_NAME "extern_lite_download_${FILENAME_EX}")
+    set(UNPACK_DIR "${INSTALL_DIR}/src/${EXTERNAL_PROJECT_NAME}")
+    ExternalProject_Add(
+            ${EXTERNAL_PROJECT_NAME}
+            ${EXTERNAL_PROJECT_LOG_ARGS}
+            PREFIX                ${INSTALL_DIR}
+            DOWNLOAD_COMMAND      wget --no-check-certificate -q -O ${INSTALL_DIR}/${FILENAME} ${URL}/${FILENAME} && ${CMAKE_COMMAND} -E tar xzf ${INSTALL_DIR}/${FILENAME}
+            DOWNLOAD_DIR          ${INSTALL_DIR}
+            DOWNLOAD_NO_PROGRESS  1
+            CONFIGURE_COMMAND     ""
+            BUILD_COMMAND         ""
+            UPDATE_COMMAND        ""
+            INSTALL_COMMAND       ""
+    )
+endfunction()
+
+function (lite_deps TARGET)
+  set(options "")
+  set(oneValueArgs "")
+  set(multiValueArgs DEPS X86_DEPS CUDA_DEPS ARM_DEPS PROFILE_DEPS LIGHT_DEPS HVY_DEPS CL_DEPS FPGA_DEPS NPU_DEPS ARGS)
+  cmake_parse_arguments(lite_deps "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN})
+
+  set(deps ${lite_deps_DEPS})
+
+  if(LITE_WITH_X86)
+    foreach(var ${lite_deps_X86_DEPS})
+      set(deps ${deps} ${var})
+    endforeach(var)
+  endif()
+
+  if(LITE_WITH_CUDA)
+    foreach(var ${lite_deps_CUDA_DEPS})
+      set(deps ${deps} ${var})
+    endforeach(var)
+  endif()
+
+  if(LITE_WITH_ARM)
+    foreach(var ${lite_deps_ARM_DEPS})
+      set(deps ${deps} ${var})
+    endforeach(var)
+  endif()
+
+  if(LITE_WITH_PROFILE)
+    foreach(var ${lite_deps_PROFILE_DEPS})
+      set(deps ${deps} ${var})
+    endforeach(var)
+  endif()
+
+  if(LITE_WITH_LIGHT_WEIGHT_FRAMEWORK)
+    foreach(var ${lite_deps_LIGHT_DEPS})
+      set(deps ${deps} ${var})
+    endforeach(var)
+  endif()
+
+  if (NOT LITE_WITH_LIGHT_WEIGHT_FRAMEWORK)
+    foreach(var ${lite_deps_HVY_DEPS})
+      set(deps ${deps} ${var})
+    endforeach(var)
+  endif()
+
+  if (LITE_WITH_OPENCL)
+    foreach(var ${lite_deps_CL_DEPS})
+      set(deps ${deps} ${var})
+    endforeach(var)
+  endif()
+
+  if (LITE_WITH_FPGA)
+    foreach(var ${lite_deps_FPGA_DEPS})
+      set(deps ${deps} ${var})
+    endforeach(var)
+  endif()
+
+  if (LITE_WITH_NPU)
+    foreach(var ${lite_deps_NPU_DEPS})
+      set(deps ${deps} ${var})
+    endforeach(var)
+  endif()
+
+  set(${TARGET} ${deps} PARENT_SCOPE)
+endfunction()
+
+
+# A fake target to include all the libraries and tests the lite module depends.
+add_custom_target(lite_compile_deps COMMAND echo 1)
+
+# Add names for lite libraries for latter compile. We use this name list to avoid compiling
+# the whole fluid project to accelerate the compile speed.
+set(offline_lib_registry_file "${CMAKE_BINARY_DIR}/lite_libs.txt")
+file(WRITE ${offline_lib_registry_file} "") # clean
+
+# cc_library with branch support.
+# The branches:
+#  X86_DEPS: works only when LITE_WITH_X86 is ON.
+#  CUDA_DEPS:     LITE_WITH_CUDA
+#  ARM_DEPS:      LITE_WITH_ARM
+#  PROFILE_DEPS:  LITE_WITH_PROFILE
+#  LIGHT_DEPS:    LITE_WITH_LIGHT_WEIGHT_FRAMEWORK
+#  HVY_DEPS:      NOT LITE_WITH_LIGHT_WEIGHT_FRAMEWORK
+#  EXCLUDE_COMPILE_DEPS: TARGET will not be included in lite_compile_deps if this is not None
+function(lite_cc_library TARGET)
+    set(options SHARED shared STATIC static MODULE module)
+    set(oneValueArgs "")
+    set(multiValueArgs SRCS DEPS X86_DEPS CUDA_DEPS CL_DEPS NPU_DEPS ARM_DEPS FPGA_DEPS PROFILE_DEPS LIGHT_DEPS
+      HVY_DEPS EXCLUDE_COMPILE_DEPS ARGS)
+    cmake_parse_arguments(args "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN})
+
+    set(deps "")
+    lite_deps(deps
+            DEPS ${args_DEPS}
+            X86_DEPS ${args_X86_DEPS}
+            CUDA_DEPS ${args_CUDA_DEPS}
+            CL_DEPS ${args_CL_DEPS}
+            NPU_DEPS ${args_NPU_DEPS}
+            ARM_DEPS ${args_ARM_DEPS}
+            FPGA_DEPS ${args_FPGA_DEPS}
+            PROFILE_DEPS ${args_PROFILE_DEPS}
+            LIGHT_DEPS ${args_LIGHT_DEPS}
+            HVY_DEPS ${args_HVY_DEPS}
+            )
+
+    if (args_SHARED OR ARGS_shared)
+        cc_library(${TARGET} SRCS ${args_SRCS} DEPS ${deps} ${args_DEPS} SHARED)
+    elseif (args_MODULE OR ARGS_module)
+        add_library(${TARGET} MODULE ${args_SRCS})
+        add_dependencies(${TARGET} ${deps} ${args_DEPS})
+    else()
+        cc_library(${TARGET} SRCS ${args_SRCS} DEPS ${deps} ${args_DEPS})
+    endif()
+    target_compile_options(${TARGET} BEFORE PRIVATE -Wno-ignored-qualifiers)
+
+    # collect targets need to compile for lite
+    if (args_SRCS AND NOT args_EXCLUDE_COMPILE_DEPS)
+        add_dependencies(lite_compile_deps ${TARGET})
+    endif()
+
+    # register a library name.
+    file(APPEND ${offline_lib_registry_file} "${TARGET}\n")
+endfunction()
+
+function(lite_cc_binary TARGET)
+    set(options "")
+    set(oneValueArgs "")
+    set(multiValueArgs SRCS DEPS X86_DEPS CUDA_DEPS CL_DEPS ARM_DEPS FPGA_DEPS PROFILE_DEPS
+      LIGHT_DEPS HVY_DEPS EXCLUDE_COMPILE_DEPS ARGS)
+    cmake_parse_arguments(args "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN})
+
+    set(deps "")
+    lite_deps(deps
+            DEPS ${args_DEPS}
+            X86_DEPS ${args_X86_DEPS}
+            CUDA_DEPS ${args_CUDA_DEPS}
+            CL_DEPS ${args_CL_DEPS}
+            ARM_DEPS ${args_ARM_DEPS}
+            FPGA_DEPS ${args_FPGA_DEPS}
+            PROFILE_DEPS ${args_PROFILE_DEPS}
+            LIGHT_DEPS ${args_LIGHT_DEPS}
+            HVY_DEPS ${args_HVY_DEPS}
+            )
+    cc_binary(${TARGET} SRCS ${args_SRCS} DEPS ${deps} ${args_DEPS})
+    target_compile_options(${TARGET} BEFORE PRIVATE -Wno-ignored-qualifiers)
+    # collect targets need to compile for lite
+    if (NOT args_EXCLUDE_COMPILE_DEPS)
+        add_dependencies(lite_compile_deps ${TARGET})
+    endif()
+endfunction()
+
+# Add a unit-test name to file for latter offline manual test.
+set(offline_test_registry_file "${CMAKE_BINARY_DIR}/lite_tests.txt")
+file(WRITE ${offline_test_registry_file} "") # clean
+# Test lite modules.
+
+function(lite_cc_test TARGET)
+    if(NOT WITH_TESTING)
+        return()
+    endif()
+    set(options "")
+    set(oneValueArgs "")
+    set(multiValueArgs SRCS DEPS X86_DEPS CUDA_DEPS CL_DEPS ARM_DEPS FPGA_DEPS PROFILE_DEPS
+        LIGHT_DEPS HVY_DEPS EXCLUDE_COMPILE_DEPS
+        ARGS)
+    cmake_parse_arguments(args "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN})
+
+    set(deps "")
+    lite_deps(deps
+              DEPS ${args_DEPS}
+              X86_DEPS ${args_X86_DEPS}
+              CUDA_DEPS ${args_CUDA_DEPS}
+              CL_DEPS ${args_CL_DEPS}
+              ARM_DEPS ${args_ARM_DEPS}
+              FPGA_DEPS ${args_FPGA_DEPS}
+              PROFILE_DEPS ${args_PROFILE_DEPS}
+              LIGHT_DEPS ${args_LIGHT_DEPS}
+              HVY_DEPS ${args_HVY_DEPS}
+              )
+    _lite_cc_test(${TARGET} SRCS ${args_SRCS} DEPS ${deps} ARGS ${args_ARGS})
+    target_compile_options(${TARGET} BEFORE PRIVATE -Wno-ignored-qualifiers)
+    file(APPEND ${offline_test_registry_file} "${TARGET}\n")
+
+    # collect targets need to compile for lite
+    if (NOT args_EXCLUDE_COMPILE_DEPS)
+        add_dependencies(lite_compile_deps ${TARGET})
+    endif()
+endfunction()
+
+
 # Bundle several static libraries into one.
 function(bundle_static_library tgt_name bundled_tgt_name fake_target)
   list(APPEND static_libs ${tgt_name})

lite/CMakeLists.txt

@@ -13,214 +13,6 @@ set(LITE_MODEL_DIR "${THIRD_PARTY_PATH}/install")
 
 set(LITE_ON_MOBILE ${LITE_WITH_LIGHT_WEIGHT_FRAMEWORK})
 
-set(LITE_URL "http://paddle-inference-dist.bj.bcebos.com" CACHE STRING "inference download url")
-
-function(lite_download_and_uncompress INSTALL_DIR URL FILENAME)
-    message(STATUS "Download inference test stuff from ${URL}/${FILENAME}")
-    string(REGEX REPLACE "[-%.]" "_" FILENAME_EX ${FILENAME})
-    set(EXTERNAL_PROJECT_NAME "extern_lite_download_${FILENAME_EX}")
-    set(UNPACK_DIR "${INSTALL_DIR}/src/${EXTERNAL_PROJECT_NAME}")
-    ExternalProject_Add(
-            ${EXTERNAL_PROJECT_NAME}
-            ${EXTERNAL_PROJECT_LOG_ARGS}
-            PREFIX                ${INSTALL_DIR}
-            DOWNLOAD_COMMAND      wget --no-check-certificate -q -O ${INSTALL_DIR}/${FILENAME} ${URL}/${FILENAME} && ${CMAKE_COMMAND} -E tar xzf ${INSTALL_DIR}/${FILENAME}
-            DOWNLOAD_DIR          ${INSTALL_DIR}
-            DOWNLOAD_NO_PROGRESS  1
-            CONFIGURE_COMMAND     ""
-            BUILD_COMMAND         ""
-            UPDATE_COMMAND        ""
-            INSTALL_COMMAND       ""
-    )
-endfunction()
-
-function (lite_deps TARGET)
-  set(options "")
-  set(oneValueArgs "")
-  set(multiValueArgs DEPS X86_DEPS CUDA_DEPS ARM_DEPS PROFILE_DEPS LIGHT_DEPS HVY_DEPS CL_DEPS FPGA_DEPS NPU_DEPS ARGS)
-  cmake_parse_arguments(lite_deps "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN})
-
-  set(deps ${lite_deps_DEPS})
-
-  if(LITE_WITH_X86)
-    foreach(var ${lite_deps_X86_DEPS})
-      set(deps ${deps} ${var})
-    endforeach(var)
-  endif()
-
-  if(LITE_WITH_CUDA)
-    foreach(var ${lite_deps_CUDA_DEPS})
-      set(deps ${deps} ${var})
-    endforeach(var)
-  endif()
-
-  if(LITE_WITH_ARM)
-    foreach(var ${lite_deps_ARM_DEPS})
-      set(deps ${deps} ${var})
-    endforeach(var)
-  endif()
-
-  if(LITE_WITH_PROFILE)
-    foreach(var ${lite_deps_PROFILE_DEPS})
-      set(deps ${deps} ${var})
-    endforeach(var)
-  endif()
-
-  if(LITE_WITH_LIGHT_WEIGHT_FRAMEWORK)
-    foreach(var ${lite_deps_LIGHT_DEPS})
-      set(deps ${deps} ${var})
-    endforeach(var)
-  endif()
-
-  if (NOT LITE_WITH_LIGHT_WEIGHT_FRAMEWORK)
-    foreach(var ${lite_deps_HVY_DEPS})
-      set(deps ${deps} ${var})
-    endforeach(var)
-  endif()
-
-  if (LITE_WITH_OPENCL)
-    foreach(var ${lite_deps_CL_DEPS})
-      set(deps ${deps} ${var})
-    endforeach(var)
-  endif()
-
-  if (LITE_WITH_FPGA)
-    foreach(var ${lite_deps_FPGA_DEPS})
-      set(deps ${deps} ${var})
-    endforeach(var)
-  endif()
-
-  if (LITE_WITH_NPU)
-    foreach(var ${lite_deps_NPU_DEPS})
-      set(deps ${deps} ${var})
-    endforeach(var)
-  endif()
-
-  set(${TARGET} ${deps} PARENT_SCOPE)
-endfunction()
-
-
-# A fake target to include all the libraries and tests the lite module depends.
-add_custom_target(lite_compile_deps COMMAND echo 1)
-
-# Add names for lite libraries for latter compile. We use this name list to avoid compiling
-# the whole fluid project to accelerate the compile speed.
-set(offline_lib_registry_file "${CMAKE_BINARY_DIR}/lite_libs.txt")
-file(WRITE ${offline_lib_registry_file} "") # clean
-
-# cc_library with branch support.
-# The branches:
-#  X86_DEPS: works only when LITE_WITH_X86 is ON.
-#  CUDA_DEPS:     LITE_WITH_CUDA
-#  ARM_DEPS:      LITE_WITH_ARM
-#  PROFILE_DEPS:  LITE_WITH_PROFILE
-#  LIGHT_DEPS:    LITE_WITH_LIGHT_WEIGHT_FRAMEWORK
-#  HVY_DEPS:      NOT LITE_WITH_LIGHT_WEIGHT_FRAMEWORK
-#  EXCLUDE_COMPILE_DEPS: TARGET will not be included in lite_compile_deps if this is not None
-function(lite_cc_library TARGET)
-    set(options SHARED shared STATIC static MODULE module)
-    set(oneValueArgs "")
-    set(multiValueArgs SRCS DEPS X86_DEPS CUDA_DEPS CL_DEPS NPU_DEPS ARM_DEPS FPGA_DEPS PROFILE_DEPS LIGHT_DEPS
-      HVY_DEPS EXCLUDE_COMPILE_DEPS ARGS)
-    cmake_parse_arguments(args "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN})
-
-    set(deps "")
-    lite_deps(deps
-            DEPS ${args_DEPS}
-            X86_DEPS ${args_X86_DEPS}
-            CUDA_DEPS ${args_CUDA_DEPS}
-            CL_DEPS ${args_CL_DEPS}
-            NPU_DEPS ${args_NPU_DEPS}
-            ARM_DEPS ${args_ARM_DEPS}
-            FPGA_DEPS ${args_FPGA_DEPS}
-            PROFILE_DEPS ${args_PROFILE_DEPS}
-            LIGHT_DEPS ${args_LIGHT_DEPS}
-            HVY_DEPS ${args_HVY_DEPS}
-            )
-
-    if (args_SHARED OR ARGS_shared)
-        cc_library(${TARGET} SRCS ${args_SRCS} DEPS ${deps} ${args_DEPS} SHARED)
-    elseif (args_MODULE OR ARGS_module)
-        add_library(${TARGET} MODULE ${args_SRCS})
-        add_dependencies(${TARGET} ${deps} ${args_DEPS})
-    else()
-        cc_library(${TARGET} SRCS ${args_SRCS} DEPS ${deps} ${args_DEPS})
-    endif()
-    target_compile_options(${TARGET} BEFORE PRIVATE -Wno-ignored-qualifiers)
-
-    # collect targets need to compile for lite
-    if (args_SRCS AND NOT args_EXCLUDE_COMPILE_DEPS)
-        add_dependencies(lite_compile_deps ${TARGET})
-    endif()
-
-    # register a library name.
-    file(APPEND ${offline_lib_registry_file} "${TARGET}\n")
-endfunction()
-
-function(lite_cc_binary TARGET)
-    set(options "")
-    set(oneValueArgs "")
-    set(multiValueArgs SRCS DEPS X86_DEPS CUDA_DEPS CL_DEPS ARM_DEPS FPGA_DEPS PROFILE_DEPS
-      LIGHT_DEPS HVY_DEPS EXCLUDE_COMPILE_DEPS ARGS)
-    cmake_parse_arguments(args "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN})
-
-    set(deps "")
-    lite_deps(deps
-            DEPS ${args_DEPS}
-            X86_DEPS ${args_X86_DEPS}
-            CUDA_DEPS ${args_CUDA_DEPS}
-            CL_DEPS ${args_CL_DEPS}
-            ARM_DEPS ${args_ARM_DEPS}
-            FPGA_DEPS ${args_FPGA_DEPS}
-            PROFILE_DEPS ${args_PROFILE_DEPS}
-            LIGHT_DEPS ${args_LIGHT_DEPS}
-            HVY_DEPS ${args_HVY_DEPS}
-            )
-    cc_binary(${TARGET} SRCS ${args_SRCS} DEPS ${deps} ${args_DEPS})
-    target_compile_options(${TARGET} BEFORE PRIVATE -Wno-ignored-qualifiers)
-    # collect targets need to compile for lite
-    if (NOT args_EXCLUDE_COMPILE_DEPS)
-        add_dependencies(lite_compile_deps ${TARGET})
-    endif()
-endfunction()
-
-# Add a unit-test name to file for latter offline manual test.
-set(offline_test_registry_file "${CMAKE_BINARY_DIR}/lite_tests.txt")
-file(WRITE ${offline_test_registry_file} "") # clean
-# Test lite modules.
-
-function(lite_cc_test TARGET)
-    if(NOT WITH_TESTING)
-        return()
-    endif()
-    set(options "")
-    set(oneValueArgs "")
-    set(multiValueArgs SRCS DEPS X86_DEPS CUDA_DEPS CL_DEPS ARM_DEPS FPGA_DEPS PROFILE_DEPS
-        LIGHT_DEPS HVY_DEPS EXCLUDE_COMPILE_DEPS
-        ARGS)
-    cmake_parse_arguments(args "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN})
-
-    set(deps "")
-    lite_deps(deps
-              DEPS ${args_DEPS}
-              X86_DEPS ${args_X86_DEPS}
-              CUDA_DEPS ${args_CUDA_DEPS}
-              CL_DEPS ${args_CL_DEPS}
-              ARM_DEPS ${args_ARM_DEPS}
-              FPGA_DEPS ${args_FPGA_DEPS}
-              PROFILE_DEPS ${args_PROFILE_DEPS}
-              LIGHT_DEPS ${args_LIGHT_DEPS}
-              HVY_DEPS ${args_HVY_DEPS}
-              )
-    _lite_cc_test(${TARGET} SRCS ${args_SRCS} DEPS ${deps} ARGS ${args_ARGS})
-    target_compile_options(${TARGET} BEFORE PRIVATE -Wno-ignored-qualifiers)
-    file(APPEND ${offline_test_registry_file} "${TARGET}\n")
-
-    # collect targets need to compile for lite
-    if (NOT args_EXCLUDE_COMPILE_DEPS)
-        add_dependencies(lite_compile_deps ${TARGET})
-    endif()
-endfunction()
 
 add_subdirectory(utils)
 add_subdirectory(operators)
@@ -236,6 +28,7 @@ add_subdirectory(fpga)
 add_subdirectory(model_parser)
 add_subdirectory(api)
 add_subdirectory(fluid)
+
 if (NOT LITE_ON_TINY_PUBLISH)
     add_subdirectory(tests)
     add_subdirectory(tools)

lite/tools/build.sh

@@ -105,11 +105,28 @@ function print_usage {
     echo -e "compile all arm tests:"
     echo -e "   ./build.sh --arm_os=<os> --arm_abi=<abi> --arm_lang=<lang> test"
     echo
+    echo -e "argument choices:"
+    echo
+    echo -e "--arm_os:\t android"
+    echo -e "--arm_abi:\t armv8|armv7"
+    echo -e "--arm_lang:\t gcc|clang"
+    echo -e "--android_stl:\t c++_static|c++_shared"
+    echo
+    echo -e "tasks:"
+    echo
+    echo -e "tiny_publish: a small library for deployment."
+    echo -e "full_publish: a full library for debug and test."
+    echo -e "test: produce all the unittests."
     echo "----------------------------------------"
     echo
 }
 
 function main {
+    if [ -z "$1" ]; then
+        print_usage
+        exit -1
+    fi
+
     # Parse command line.
     for i in "$@"; do
         case $i in