add InferEngine for arm and xpu

CMakeLists.txt

@@ -51,6 +51,7 @@ option(WITH_MKL	    "Compile Paddle Serving with MKL support."      OFF)
 option(WITH_GPU	    "Compile Paddle Serving with NVIDIA GPU"        OFF)
 option(WITH_LITE    "Compile Paddle Serving with NVIDIA GPU"        OFF)
 option(WITH_XPU	    "Compile Paddle Serving with NVIDIA GPU"        OFF)
+option(WITH_PYTHON  "Compile Paddle Serving with Python"		    OFF)
 option(CLIENT  	    "Compile Paddle Serving Client"		    OFF)
 option(SERVER	    "Compile Paddle Serving Server"		    OFF)
 option(APP          "Compile Paddle Serving App package"	    OFF)
@@ -82,8 +83,10 @@ include(external/protobuf)
 include(external/brpc)
 include(external/gflags)
 include(external/glog)
+if (WITH_PYTHON)
 include(external/pybind11)
 include(external/python)
+endif()
 include(generic)
 include(flags)
 endif()
@@ -149,4 +152,6 @@ if(SERVER)
 add_subdirectory(paddle_inference)
 endif()
 
+if (WITH_PYTHON)
 add_subdirectory(python)
+endif()

cmake/external/boost.cmake

@@ -22,9 +22,15 @@ set(BOOST_PROJECT       "extern_boost")
 # version of boost, say, 1.66.0, doesn't build on CentOS 6.  We
 # checked that the devtools package of CentOS 6 installs boost 1.41.0.
 # So we use 1.41.0 here.
-set(BOOST_VER           "1.41.0")
-set(BOOST_TAR "boost_1_41_0" CACHE STRING "" FORCE)
-set(BOOST_URL "http://paddlepaddledeps.cdn.bcebos.com/${BOOST_TAR}.tar.gz" CACHE STRING "" FORCE)
+if(WITH_LITE)
+    set(BOOST_VER           "1.66.0")
+    set(BOOST_TAR "boost_1_66_0" CACHE STRING "" FORCE)
+    set(BOOST_URL "https://dl.bintray.com/boostorg/release/${BOOST_VER}/source/${BOOST_TAR}.tar.gz" CACHE STRING "" FORCE)
+else()
+    set(BOOST_VER           "1.41.0")
+    set(BOOST_TAR "boost_1_41_0" CACHE STRING "" FORCE)
+    set(BOOST_URL "http://paddlepaddledeps.cdn.bcebos.com/${BOOST_TAR}.tar.gz" CACHE STRING "" FORCE)
+endif()
 
 MESSAGE(STATUS "BOOST_TAR: ${BOOST_TAR}, BOOST_URL: ${BOOST_URL}")
 

cmake/external/brpc.cmake

@@ -35,13 +35,21 @@ INCLUDE_DIRECTORIES(${BRPC_INCLUDE_DIR})
 # Reference https://stackoverflow.com/questions/45414507/pass-a-list-of-prefix-paths-to-externalproject-add-in-cmake-args
 set(prefix_path "${THIRD_PARTY_PATH}/install/gflags|${THIRD_PARTY_PATH}/install/leveldb|${THIRD_PARTY_PATH}/install/snappy|${THIRD_PARTY_PATH}/install/gtest|${THIRD_PARTY_PATH}/install/protobuf|${THIRD_PARTY_PATH}/install/zlib|${THIRD_PARTY_PATH}/install/glog")
 
+if(WITH_LITE)
+    set(BRPC_REPO "https://github.com/apache/incubator-brpc.git")
+    set(BRPC_TAG "master")
+else()
+    set(BRPC_REPO "https://github.com/wangjiawei04/brpc")
+    set(BRPC_TAG "6d79e0b17f25107c35b705ea58d888083f59ff47")
+endif()
+
 # If minimal .a is need, you can set  WITH_DEBUG_SYMBOLS=OFF
 ExternalProject_Add(
     extern_brpc
     ${EXTERNAL_PROJECT_LOG_ARGS}
     # TODO(gongwb): change to de newst repo when they changed.
-    GIT_REPOSITORY  "https://github.com/wangjiawei04/brpc"
-    GIT_TAG         "6d79e0b17f25107c35b705ea58d888083f59ff47"
+    GIT_REPOSITORY  ${BRPC_REPO}
+    GIT_TAG         ${BRPC_TAG}
     PREFIX          ${BRPC_SOURCES_DIR}
     UPDATE_COMMAND  ""
     CMAKE_ARGS      -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}

cmake/generic.cmake

@@ -93,7 +93,11 @@ include_directories(${CMAKE_CURRENT_BINARY_DIR})
 if(NOT APPLE)
   find_package(Threads REQUIRED)
   link_libraries(${CMAKE_THREAD_LIBS_INIT})
-  set(CMAKE_CXX_LINK_EXECUTABLE "${CMAKE_CXX_LINK_EXECUTABLE} -pthread -ldl -lrt")
+  if(WITH_LITE OR WITH_XPU)
+    set(CMAKE_CXX_LINK_EXECUTABLE "${CMAKE_CXX_LINK_EXECUTABLE} -fopenmp -pthread -ldl -lrt")
+  else()
+    set(CMAKE_CXX_LINK_EXECUTABLE "${CMAKE_CXX_LINK_EXECUTABLE} -pthread -ldl -lrt")
+  endif()
 endif(NOT APPLE)
 
 set_property(GLOBAL PROPERTY FLUID_MODULES "")

cmake/paddlepaddle.cmake

@@ -51,7 +51,12 @@ else()
     endif()
 endif()
 
-SET(PADDLE_LIB_PATH "http://paddle-inference-lib.bj.bcebos.com/${PADDLE_LIB_VERSION}/fluid_inference.tgz")
+if(WITH_LITE)
+    SET(PADDLE_LIB_PATH "/home/phytium/houjue/serving_tmp/build_paddle/fluid_inference.tgz")
+else()
+    SET(PADDLE_LIB_PATH "http://paddle-inference-lib.bj.bcebos.com/${PADDLE_LIB_VERSION}/fluid_inference.tgz")
+endif()
+
 MESSAGE(STATUS "PADDLE_LIB_PATH=${PADDLE_LIB_PATH}")
 if (WITH_GPU OR WITH_MKLML)
     if (WITH_TRT)
@@ -124,6 +129,19 @@ ADD_LIBRARY(nvinfer_plugin SHARED IMPORTED GLOBAL)
 SET_PROPERTY(TARGET nvinfer_plugin PROPERTY IMPORTED_LOCATION ${TENSORRT_ROOT}/lib/libnvinfer_plugin.so)
 endif()
 
+if (WITH_LITE)
+ADD_LIBRARY(paddle_api_full_bundled STATIC IMPORTED GLOBAL)
+SET_PROPERTY(TARGET paddle_api_full_bundled PROPERTY IMPORTED_LOCATION ${PADDLE_INSTALL_DIR}/third_party/install/lite/cxx/lib/libpaddle_api_full_bundled.a)
+
+if (WITH_XPU)
+    ADD_LIBRARY(xpuapi SHARED IMPORTED GLOBAL)
+    SET_PROPERTY(TARGET xpuapi PROPERTY IMPORTED_LOCATION ${PADDLE_INSTALL_DIR}/third_party/install/xpu/lib/libxpuapi.so)
+
+    ADD_LIBRARY(xpurt SHARED IMPORTED GLOBAL)
+    SET_PROPERTY(TARGET xpurt PROPERTY IMPORTED_LOCATION ${PADDLE_INSTALL_DIR}/third_party/install/xpu/lib/libxpurt.so)
+endif()
+endif()
+
 ADD_LIBRARY(xxhash STATIC IMPORTED GLOBAL)
 SET_PROPERTY(TARGET xxhash PROPERTY IMPORTED_LOCATION ${PADDLE_INSTALL_DIR}/third_party/install/xxhash/lib/libxxhash.a)
 
@@ -132,6 +150,13 @@ LIST(APPEND external_project_dependencies paddle)
 LIST(APPEND paddle_depend_libs
     xxhash)
 
+if(WITH_LITE)
+LIST(APPEND paddle_depend_libs paddle_api_full_bundled)
+if(WITH_XPU)
+    LIST(APPEND paddle_depend_libs xpuapi xpurt)
+endif()
+endif()
+
 if(WITH_TRT)
 LIST(APPEND paddle_depend_libs
     nvinfer nvinfer_plugin)

core/configure/CMakeLists.txt

@@ -31,6 +31,8 @@ install(FILES ${inc}
         DESTINATION ${PADDLE_SERVING_INSTALL_DIR}/include/configure)
 endif()
 
+if (WITH_PYTHON)
+
 py_proto_compile(general_model_config_py_proto SRCS proto/general_model_config.proto)
 add_custom_target(general_model_config_py_proto_init ALL COMMAND ${CMAKE_COMMAND} -E touch __init__.py)
 add_dependencies(general_model_config_py_proto general_model_config_py_proto_init)
@@ -118,3 +120,5 @@ add_custom_command(TARGET multi_lang_general_model_service_py_proto POST_BUILD
                 WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR})
 endif()
 endif()
+
+endif()

core/configure/proto/server_configure.proto

@@ -48,8 +48,6 @@ message EngineDesc {
   optional bool use_lite = 18;
   optional bool use_xpu = 19;
 };
-};
-};
 
 // model_toolkit conf
 message ModelToolkitConf { repeated EngineDesc engines = 1; };

core/general-server/CMakeLists.txt

@@ -6,6 +6,11 @@ add_dependencies(serving pdcodegen fluid_cpu_engine pdserving paddle_fluid cube-
 if (WITH_GPU)
     add_dependencies(serving fluid_gpu_engine)
 endif()
+
+if (WITH_LITE)
+    add_dependencies(serving fluid_arm_engine)
+endif()
+
 target_include_directories(serving PUBLIC
         ${CMAKE_CURRENT_BINARY_DIR}/../../core/predictor
         )
@@ -15,6 +20,11 @@ if(WITH_GPU)
             -Wl,--no-whole-archive)
 endif()
 
+if(WITH_LITE)
+    target_link_libraries(serving -Wl,--whole-archive fluid_arm_engine
+            -Wl,--no-whole-archive)
+endif()
+
 target_link_libraries(serving -Wl,--whole-archive fluid_cpu_engine
         -Wl,--no-whole-archive)
 

core/predictor/framework/infer.h

@@ -42,8 +42,6 @@ class InferEngineCreationParams {
     _use_lite = false;
     _use_xpu = false;
   }
-  }
-  }
 
   void set_path(const std::string& path) { _path = path; }
 
@@ -92,9 +90,9 @@ class InferEngineCreationParams {
               << "model_path = " << _path << ", "
               << "enable_memory_optimization = " << _enable_memory_optimization
               << ", "
-              << "enable_tensorrt = " << _enable_tensorrt << ", "
-              << "enable_lite = " << _enable_lite << ", "
-              << "enable_xpu = " << _enable_xpu << ", "
+              << "enable_tensorrt = " << _use_trt << ", "
+              << "enable_lite = " << _use_lite << ", "
+              << "enable_xpu = " << _use_xpu << ", "
               << "enable_ir_optimization = " << _enable_ir_optimization << ", "
               << "static_optimization = " << _static_optimization << ", "
               << "force_update_static_cache = " << _force_update_static_cache;
@@ -203,7 +201,7 @@ class ReloadableInferEngine : public InferEngine {
       _infer_engine_params.set_use_lite(conf.use_lite());
     }
 
-    if (conf.has_use_trt()) {
+    if (conf.has_use_xpu()) {
       _infer_engine_params.set_use_xpu(conf.use_xpu());
     }
 

paddle_inference/inferencer-fluid-arm/include/fluid_arm_engine.h

@@ -129,11 +129,11 @@ class FluidArmAnalysisCore : public FluidFamilyCore {
       analysis_config.EnableMemoryOptim();
     }
 
-    if (params.enable_lite()) {
+    if (params.use_lite()) {
       analysis_config.EnableLiteEngine(paddle::AnalysisConfig::Precision::kFloat32, true);
     }
 
-    if (params.enable_xpu()) {
+    if (params.use_xpu()) {
       analysis_config.EnableXpu(100);
     }
 
@@ -146,7 +146,8 @@ class FluidArmAnalysisCore : public FluidFamilyCore {
       return -1;
     }
 
-    VLOG(2) << "create paddle predictor sucess, path: " << data_path;
+    VLOG(2) << "[FluidArmAnalysisCore] create paddle predictor sucess, path: " << data_path;
+    params.dump();
     return 0;
   }
 };
@@ -177,7 +178,8 @@ class FluidArmNativeCore : public FluidFamilyCore {
       return -1;
     }
 
-    VLOG(2) << "create paddle predictor sucess, path: " << data_path;
+    VLOG(2) << "[FluidArmNativeCore] create paddle predictor sucess, path: " << data_path;
+    params.dump();
     return 0;
   }
 };
@@ -208,6 +210,14 @@ class FluidArmAnalysisDirCore : public FluidFamilyCore {
       analysis_config.SwitchIrOptim(false);
     }
 
+    if (params.use_lite()) {
+      analysis_config.EnableLiteEngine(paddle::AnalysisConfig::Precision::kFloat32, true);
+    }
+
+    if (params.use_xpu()) {
+      analysis_config.EnableXpu(100);
+    }
+
     AutoLock lock(GlobalPaddleCreateMutex::instance());
     _core =
         paddle::CreatePaddlePredictor<paddle::AnalysisConfig>(analysis_config);
@@ -216,7 +226,8 @@ class FluidArmAnalysisDirCore : public FluidFamilyCore {
       return -1;
     }
 
-    VLOG(2) << "create paddle predictor sucess, path: " << data_path;
+    VLOG(2) << "[FluidArmAnalysisDirCore] create paddle predictor sucess, path: " << data_path;
+    params.dump();
     return 0;
   }
 };
@@ -235,7 +246,7 @@ class FluidArmNativeDirCore : public FluidFamilyCore {
     native_config.model_dir = data_path;
     native_config.use_gpu = false;
     native_config.device = 0;
-    native_config.fraction_of_arm_memory = 0;
+    native_config.fraction_of_gpu_memory = 0;
     AutoLock lock(GlobalPaddleCreateMutex::instance());
     _core = paddle::CreatePaddlePredictor<paddle::NativeConfig,
                                           paddle::PaddleEngineKind::kNative>(
@@ -490,7 +501,7 @@ class FluidArmNativeDirWithSigmoidCore : public FluidArmWithSigmoidCore {
     native_config.model_dir = data_path;
     native_config.use_gpu = false;
     native_config.device = 0;
-    native_config.fraction_of_arm_memory = 0;
+    native_config.fraction_of_gpu_memory = 0;
     AutoLock lock(GlobalPaddleCreateMutex::instance());
     _core->_fluid_core =
         paddle::CreatePaddlePredictor<paddle::NativeConfig,

paddle_inference/inferencer-fluid-arm/src/fluid_arm_engine.cpp

@@ -20,35 +20,35 @@ namespace paddle_serving {
 namespace fluid_arm {
 
 REGIST_FACTORY_OBJECT_IMPL_WITH_NAME(
-    ::baidu::paddle_serving::predictor::FluidInferEngine<FluidCpuAnalysisCore>,
+    ::baidu::paddle_serving::predictor::FluidInferEngine<FluidArmAnalysisCore>,
     ::baidu::paddle_serving::predictor::InferEngine,
     "FLUID_ARM_ANALYSIS");
 
 REGIST_FACTORY_OBJECT_IMPL_WITH_NAME(
     ::baidu::paddle_serving::predictor::FluidInferEngine<
-        FluidCpuAnalysisDirCore>,
+        FluidArmAnalysisDirCore>,
     ::baidu::paddle_serving::predictor::InferEngine,
     "FLUID_ARM_ANALYSIS_DIR");
 
 REGIST_FACTORY_OBJECT_IMPL_WITH_NAME(
     ::baidu::paddle_serving::predictor::FluidInferEngine<
-        FluidCpuAnalysisDirWithSigmoidCore>,
+        FluidArmAnalysisDirWithSigmoidCore>,
     ::baidu::paddle_serving::predictor::InferEngine,
     "FLUID_ARM_ANALYSIS_DIR_SIGMOID");
 
 REGIST_FACTORY_OBJECT_IMPL_WITH_NAME(
-    ::baidu::paddle_serving::predictor::FluidInferEngine<FluidCpuNativeCore>,
+    ::baidu::paddle_serving::predictor::FluidInferEngine<FluidArmNativeCore>,
     ::baidu::paddle_serving::predictor::InferEngine,
     "FLUID_ARM_NATIVE");
 
 REGIST_FACTORY_OBJECT_IMPL_WITH_NAME(
-    ::baidu::paddle_serving::predictor::FluidInferEngine<FluidCpuNativeDirCore>,
+    ::baidu::paddle_serving::predictor::FluidInferEngine<FluidArmNativeDirCore>,
     ::baidu::paddle_serving::predictor::InferEngine,
     "FLUID_ARM_NATIVE_DIR");
 
 REGIST_FACTORY_OBJECT_IMPL_WITH_NAME(
     ::baidu::paddle_serving::predictor::FluidInferEngine<
-        FluidCpuNativeDirWithSigmoidCore>,
+        FluidArmNativeDirWithSigmoidCore>,
     ::baidu::paddle_serving::predictor::InferEngine,
     "FLUID_ARM_NATIVE_DIR_SIGMOID");