diff --git a/CMakeLists.txt b/CMakeLists.txt index 33465d55a70ca00ac08f251d316fa45be69ca202..d01be6d74f8c0b12b8b50e5ec3e63795a9ef4a09 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -28,6 +28,7 @@ message(STATUS "C compiler: ${CMAKE_C_COMPILER}, version: " find_package(Git REQUIRED) find_package(Threads REQUIRED) +find_package(CUDA QUIET) include(simd) @@ -43,10 +44,10 @@ set(THIRD_PARTY_PATH "${CMAKE_BINARY_DIR}/third_party" CACHE STRING set(THIRD_PARTY_BUILD_TYPE Release) -option(WITH_AVX "Compile Paddle Serving with AVX intrinsics" ${AVX_FOUND}) -option(WITH_MKL "Compile Paddle Serving with MKL support." ${AVX_FOUND}) -option(CLIENT_ONLY "Compile client libraries and demos only" - FALSE) +option(WITH_AVX "Compile Paddle Serving with AVX intrinsics" ${AVX_FOUND}) +option(WITH_MKL "Compile Paddle Serving with MKL support." ${AVX_FOUND}) +option(WITH_GPU "Compile Paddle Serving with NVIDIA GPU" ${CUDA_FOUND}) +option(CLIENT_ONLY "Compile client libraries and demos only" FALSE) set(WITH_MKLML ${WITH_MKL}) if (NOT DEFINED WITH_MKLDNN) @@ -108,5 +109,8 @@ add_subdirectory(demo-client) if (NOT CLIENT_ONLY) add_subdirectory(predictor) add_subdirectory(inferencer-fluid-cpu) +if (WITH_GPU) +add_subdirectory(inferencer-fluid-gpu) +endif() add_subdirectory(demo-serving) endif() diff --git a/cmake/cuda.cmake b/cmake/cuda.cmake new file mode 100644 index 0000000000000000000000000000000000000000..ef4192ecc98ea6de0c81c1f33320528d547b818a --- /dev/null +++ b/cmake/cuda.cmake @@ -0,0 +1,231 @@ +if(NOT WITH_GPU) + return() +endif() + +set(paddle_known_gpu_archs "30 35 50 52 60 61 70") +set(paddle_known_gpu_archs7 "30 35 50 52") +set(paddle_known_gpu_archs8 "30 35 50 52 60 61") +set(paddle_known_gpu_archs9 "30 35 50 52 60 61 70") +set(paddle_known_gpu_archs10 "30 35 50 52 60 61 70 75") + +###################################################################################### +# A function for automatic detection of GPUs installed (if autodetection is enabled) +# Usage: +# detect_installed_gpus(out_variable) +function(detect_installed_gpus out_variable) + if(NOT CUDA_gpu_detect_output) + set(cufile ${PROJECT_BINARY_DIR}/detect_cuda_archs.cu) + + file(WRITE ${cufile} "" + "#include \n" + "int main() {\n" + " int count = 0;\n" + " if (cudaSuccess != cudaGetDeviceCount(&count)) return -1;\n" + " if (count == 0) return -1;\n" + " for (int device = 0; device < count; ++device) {\n" + " cudaDeviceProp prop;\n" + " if (cudaSuccess == cudaGetDeviceProperties(&prop, device))\n" + " std::printf(\"%d.%d \", prop.major, prop.minor);\n" + " }\n" + " return 0;\n" + "}\n") + + execute_process(COMMAND "${CUDA_NVCC_EXECUTABLE}" "-ccbin=${CUDA_HOST_COMPILER}" + "--run" "${cufile}" + WORKING_DIRECTORY "${PROJECT_BINARY_DIR}/CMakeFiles/" + RESULT_VARIABLE nvcc_res OUTPUT_VARIABLE nvcc_out + ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE) + + if(nvcc_res EQUAL 0) + # only keep the last line of nvcc_out + STRING(REGEX REPLACE ";" "\\\\;" nvcc_out "${nvcc_out}") + STRING(REGEX REPLACE "\n" ";" nvcc_out "${nvcc_out}") + list(GET nvcc_out -1 nvcc_out) + string(REPLACE "2.1" "2.1(2.0)" nvcc_out "${nvcc_out}") + set(CUDA_gpu_detect_output ${nvcc_out} CACHE INTERNAL "Returned GPU architetures from detect_installed_gpus tool" FORCE) + endif() + endif() + + if(NOT CUDA_gpu_detect_output) + message(STATUS "Automatic GPU detection failed. Building for all known architectures.") + set(${out_variable} ${paddle_known_gpu_archs} PARENT_SCOPE) + else() + set(${out_variable} ${CUDA_gpu_detect_output} PARENT_SCOPE) + endif() +endfunction() + + +######################################################################## +# Function for selecting GPU arch flags for nvcc based on CUDA_ARCH_NAME +# Usage: +# select_nvcc_arch_flags(out_variable) +function(select_nvcc_arch_flags out_variable) + # List of arch names + set(archs_names "Kepler" "Maxwell" "Pascal" "Volta" "Turing" "All" "Manual") + set(archs_name_default "All") + list(APPEND archs_names "Auto") + + # set CUDA_ARCH_NAME strings (so it will be seen as dropbox in CMake-Gui) + set(CUDA_ARCH_NAME ${archs_name_default} CACHE STRING "Select target NVIDIA GPU achitecture.") + set_property( CACHE CUDA_ARCH_NAME PROPERTY STRINGS "" ${archs_names} ) + mark_as_advanced(CUDA_ARCH_NAME) + + # verify CUDA_ARCH_NAME value + if(NOT ";${archs_names};" MATCHES ";${CUDA_ARCH_NAME};") + string(REPLACE ";" ", " archs_names "${archs_names}") + message(FATAL_ERROR "Only ${archs_names} architeture names are supported.") + endif() + + if(${CUDA_ARCH_NAME} STREQUAL "Manual") + set(CUDA_ARCH_BIN ${paddle_known_gpu_archs} CACHE STRING "Specify 'real' GPU architectures to build binaries for, BIN(PTX) format is supported") + set(CUDA_ARCH_PTX "50" CACHE STRING "Specify 'virtual' PTX architectures to build PTX intermediate code for") + mark_as_advanced(CUDA_ARCH_BIN CUDA_ARCH_PTX) + else() + unset(CUDA_ARCH_BIN CACHE) + unset(CUDA_ARCH_PTX CACHE) + endif() + + if(${CUDA_ARCH_NAME} STREQUAL "Kepler") + set(cuda_arch_bin "30 35") + elseif(${CUDA_ARCH_NAME} STREQUAL "Maxwell") + set(cuda_arch_bin "50") + elseif(${CUDA_ARCH_NAME} STREQUAL "Pascal") + set(cuda_arch_bin "60 61") + elseif(${CUDA_ARCH_NAME} STREQUAL "Volta") + set(cuda_arch_bin "70") + elseif(${CUDA_ARCH_NAME} STREQUAL "Turing") + set(cuda_arch_bin "75") + elseif(${CUDA_ARCH_NAME} STREQUAL "All") + set(cuda_arch_bin ${paddle_known_gpu_archs}) + elseif(${CUDA_ARCH_NAME} STREQUAL "Auto") + detect_installed_gpus(cuda_arch_bin) + else() # (${CUDA_ARCH_NAME} STREQUAL "Manual") + set(cuda_arch_bin ${CUDA_ARCH_BIN}) + endif() + + # remove dots and convert to lists + string(REGEX REPLACE "\\." "" cuda_arch_bin "${cuda_arch_bin}") + string(REGEX REPLACE "\\." "" cuda_arch_ptx "${CUDA_ARCH_PTX}") + string(REGEX MATCHALL "[0-9()]+" cuda_arch_bin "${cuda_arch_bin}") + string(REGEX MATCHALL "[0-9]+" cuda_arch_ptx "${cuda_arch_ptx}") + list(REMOVE_DUPLICATES cuda_arch_bin) + list(REMOVE_DUPLICATES cuda_arch_ptx) + + set(nvcc_flags "") + set(nvcc_archs_readable "") + + # Tell NVCC to add binaries for the specified GPUs + foreach(arch ${cuda_arch_bin}) + if(arch MATCHES "([0-9]+)\\(([0-9]+)\\)") + # User explicitly specified PTX for the concrete BIN + list(APPEND nvcc_flags -gencode arch=compute_${CMAKE_MATCH_2},code=sm_${CMAKE_MATCH_1}) + list(APPEND nvcc_archs_readable sm_${CMAKE_MATCH_1}) + else() + # User didn't explicitly specify PTX for the concrete BIN, we assume PTX=BIN + list(APPEND nvcc_flags -gencode arch=compute_${arch},code=sm_${arch}) + list(APPEND nvcc_archs_readable sm_${arch}) + endif() + endforeach() + + # Tell NVCC to add PTX intermediate code for the specified architectures + foreach(arch ${cuda_arch_ptx}) + list(APPEND nvcc_flags -gencode arch=compute_${arch},code=compute_${arch}) + list(APPEND nvcc_archs_readable compute_${arch}) + endforeach() + + string(REPLACE ";" " " nvcc_archs_readable "${nvcc_archs_readable}") + set(${out_variable} ${nvcc_flags} PARENT_SCOPE) + set(${out_variable}_readable ${nvcc_archs_readable} PARENT_SCOPE) +endfunction() + +message(STATUS "CUDA detected: " ${CUDA_VERSION}) +if (${CUDA_VERSION} LESS 7.0) + set(paddle_known_gpu_archs ${paddle_known_gpu_archs}) + add_definitions("-DPADDLE_CUDA_BINVER=\"60\"") +elseif (${CUDA_VERSION} LESS 8.0) # CUDA 7.x + set(paddle_known_gpu_archs ${paddle_known_gpu_archs7}) + list(APPEND CUDA_NVCC_FLAGS "-D_MWAITXINTRIN_H_INCLUDED") + list(APPEND CUDA_NVCC_FLAGS "-D__STRICT_ANSI__") + add_definitions("-DPADDLE_CUDA_BINVER=\"70\"") +elseif (${CUDA_VERSION} LESS 9.0) # CUDA 8.x + set(paddle_known_gpu_archs ${paddle_known_gpu_archs8}) + list(APPEND CUDA_NVCC_FLAGS "-D_MWAITXINTRIN_H_INCLUDED") + list(APPEND CUDA_NVCC_FLAGS "-D__STRICT_ANSI__") + # CUDA 8 may complain that sm_20 is no longer supported. Suppress the + # warning for now. + list(APPEND CUDA_NVCC_FLAGS "-Wno-deprecated-gpu-targets") + add_definitions("-DPADDLE_CUDA_BINVER=\"80\"") +elseif (${CUDA_VERSION} LESS 10.0) # CUDA 9.x + set(paddle_known_gpu_archs ${paddle_known_gpu_archs9}) + list(APPEND CUDA_NVCC_FLAGS "-D_MWAITXINTRIN_H_INCLUDED") + list(APPEND CUDA_NVCC_FLAGS "-D__STRICT_ANSI__") + add_definitions("-DPADDLE_CUDA_BINVER=\"90\"") +elseif (${CUDA_VERSION} LESS 11.0) # CUDA 10.x + set(paddle_known_gpu_archs ${paddle_known_gpu_archs10}) + list(APPEND CUDA_NVCC_FLAGS "-D_MWAITXINTRIN_H_INCLUDED") + list(APPEND CUDA_NVCC_FLAGS "-D__STRICT_ANSI__") + add_definitions("-DPADDLE_CUDA_BINVER=\"100\"") +endif() + +include_directories(${CUDA_INCLUDE_DIRS}) +list(APPEND EXTERNAL_LIBS ${CUDA_LIBRARIES} ${CUDA_rt_LIBRARY}) +if(NOT WITH_DSO) + # TODO(panyx0718): CUPTI only allows DSO? + list(APPEND EXTERNAL_LIBS ${CUDNN_LIBRARY} ${CUPTI_LIBRARY} ${CUDA_CUBLAS_LIBRARIES} ${CUDA_curand_LIBRARY} ${NCCL_LIBRARY}) + if(WIN32) + set_property(GLOBAL PROPERTY CUDA_MODULES ${CUDNN_LIBRARY} ${CUDA_CUBLAS_LIBRARIES} ${CUDA_curand_LIBRARY}) + endif(WIN32) +endif(NOT WITH_DSO) + +# setting nvcc arch flags +select_nvcc_arch_flags(NVCC_FLAGS_EXTRA) +list(APPEND CUDA_NVCC_FLAGS ${NVCC_FLAGS_EXTRA}) +message(STATUS "Added CUDA NVCC flags for: ${NVCC_FLAGS_EXTRA_readable}") + +# Set C++11 support +set(CUDA_PROPAGATE_HOST_FLAGS OFF) + +# Release/Debug flags set by cmake. Such as -O3 -g -DNDEBUG etc. +# So, don't set these flags here. +if (NOT WIN32) # windows msvc2015 support c++11 natively. +# -std=c++11 -fPIC not recoginize by msvc, -Xcompiler will be added by cmake. +list(APPEND CUDA_NVCC_FLAGS "-std=c++11") +list(APPEND CUDA_NVCC_FLAGS "-Xcompiler -fPIC") +endif(NOT WIN32) + +if(WITH_FAST_MATH) + # Make use of fast math library. https://docs.nvidia.com/cuda/cuda-compiler-driver-nvcc/index.html + list(APPEND CUDA_NVCC_FLAGS "--use_fast_math") +endif() +# in cuda9, suppress cuda warning on eigen +list(APPEND CUDA_NVCC_FLAGS "-w") +# Set :expt-relaxed-constexpr to suppress Eigen warnings +list(APPEND CUDA_NVCC_FLAGS "--expt-relaxed-constexpr") + +if (NOT WIN32) + if(CMAKE_BUILD_TYPE STREQUAL "Debug") + list(APPEND CUDA_NVCC_FLAGS ${CMAKE_CXX_FLAGS_DEBUG}) + elseif(CMAKE_BUILD_TYPE STREQUAL "Release") + list(APPEND CUDA_NVCC_FLAGS ${CMAKE_CXX_FLAGS_RELEASE}) + elseif(CMAKE_BUILD_TYPE STREQUAL "RelWithDebInfo") + list(APPEND CUDA_NVCC_FLAGS ${CMAKE_CXX_FLAGS_RELWITHDEBINFO}) + elseif(CMAKE_BUILD_TYPE STREQUAL "MinSizeRel") + # nvcc 9 does not support -Os. Use Release flags instead + list(APPEND CUDA_NVCC_FLAGS ${CMAKE_CXX_FLAGS_RELEASE}) + endif() +else(NOT WIN32) + list(APPEND CUDA_NVCC_FLAGS "-Xcompiler \"/wd 4244 /wd 4267 /wd 4819\"") + list(APPEND CUDA_NVCC_FLAGS "--compiler-options;/bigobj") + if(CMAKE_BUILD_TYPE STREQUAL "Debug") + list(APPEND CUDA_NVCC_FLAGS "-g -G") + # match the cl's _ITERATOR_DEBUG_LEVEL + list(APPEND CUDA_NVCC_FLAGS "-D_DEBUG") + elseif(CMAKE_BUILD_TYPE STREQUAL "Release") + list(APPEND CUDA_NVCC_FLAGS "-O3 -DNDEBUG") + else() + message(FATAL "Windows only support Release or Debug build now. Please set visual studio build type to Release/Debug, x64 build.") +endif() +endif(NOT WIN32) + +mark_as_advanced(CUDA_BUILD_CUBIN CUDA_BUILD_EMULATION CUDA_VERBOSE_BUILD) +mark_as_advanced(CUDA_SDK_ROOT_DIR CUDA_SEPARABLE_COMPILATION) diff --git a/cmake/paddlepaddle.cmake b/cmake/paddlepaddle.cmake index 72db4f23ddfa0fa34b6642e724d2c6def418c485..4caed8c2494338667d03c08d2a62ccaf3577cc7c 100644 --- a/cmake/paddlepaddle.cmake +++ b/cmake/paddlepaddle.cmake @@ -24,6 +24,8 @@ INCLUDE_DIRECTORIES(${CMAKE_BINARY_DIR}/Paddle/fluid_install_dir) # Reference https://stackoverflow.com/questions/45414507/pass-a-list-of-prefix-paths-to-externalproject-add-in-cmake-args set(prefix_path "${THIRD_PARTY_PATH}/install/gflags|${THIRD_PARTY_PATH}/install/leveldb|${THIRD_PARTY_PATH}/install/snappy|${THIRD_PARTY_PATH}/install/gtest|${THIRD_PARTY_PATH}/install/protobuf|${THIRD_PARTY_PATH}/install/zlib|${THIRD_PARTY_PATH}/install/glog") +message( "WITH_GPU = ${WITH_GPU}") + # If minimal .a is need, you can set WITH_DEBUG_SYMBOLS=OFF ExternalProject_Add( extern_paddle @@ -47,7 +49,7 @@ ExternalProject_Add( -DWITH_MKL=${WITH_MKL} -DWITH_AVX=${WITH_AVX} -DWITH_MKLDNN=OFF - -DWITH_GPU=OFF + -DWITH_GPU=${WITH_GPU} -DWITH_FLUID_ONLY=ON -DWITH_TESTING=OFF -DWITH_DISTRIBUTE=OFF diff --git a/demo-client/CMakeLists.txt b/demo-client/CMakeLists.txt index e30ca0ead42d93f3613f8702f36013316499047f..5e7208090ca4c47f724be38e92b8685684367501 100644 --- a/demo-client/CMakeLists.txt +++ b/demo-client/CMakeLists.txt @@ -20,6 +20,11 @@ target_link_libraries(ximage -Wl,--whole-archive sdk-cpp -Wl,--no-whole-archive -lpthread -lcrypto -lm -lrt -lssl -ldl -lz) +add_executable(ximage_press ${CMAKE_CURRENT_LIST_DIR}/src/ximage_press.cpp) +target_link_libraries(ximage_press -Wl,--whole-archive sdk-cpp + -Wl,--no-whole-archive -lpthread -lcrypto -lm -lrt -lssl -ldl + -lz) + add_executable(echo ${CMAKE_CURRENT_LIST_DIR}/src/echo.cpp) target_link_libraries(echo -Wl,--whole-archive sdk-cpp -Wl,--no-whole-archive -lpthread -lcrypto -lm -lrt -lssl -ldl @@ -51,6 +56,9 @@ target_link_libraries(text_classification_press -Wl,--whole-archive sdk-cpp -Wl, install(TARGETS ximage RUNTIME DESTINATION ${PADDLE_SERVING_INSTALL_DIR}/demo/client/image_classification/bin) +install(TARGETS ximage_press + RUNTIME DESTINATION + ${PADDLE_SERVING_INSTALL_DIR}/demo/client/image_classification/bin) install(DIRECTORY ${CMAKE_CURRENT_LIST_DIR}/conf DESTINATION ${PADDLE_SERVING_INSTALL_DIR}/demo/client/image_classification/) install(DIRECTORY ${CMAKE_CURRENT_LIST_DIR}/data/images DESTINATION diff --git a/demo-client/data/images/ILSVRC2012_val_00000001.jpeg b/demo-client/data/images/ILSVRC2012_val_00000001.jpeg new file mode 100644 index 0000000000000000000000000000000000000000..fcd0de8866f404b87c2881435efe8aa2372f7b36 Binary files /dev/null and b/demo-client/data/images/ILSVRC2012_val_00000001.jpeg differ diff --git a/demo-client/data/images/ILSVRC2012_val_00000002.jpeg b/demo-client/data/images/ILSVRC2012_val_00000002.jpeg new file mode 100644 index 0000000000000000000000000000000000000000..34476b6e5b2a1ef892f4957e8a5d7b54449aceb1 Binary files /dev/null and b/demo-client/data/images/ILSVRC2012_val_00000002.jpeg differ diff --git a/demo-client/data/images/ILSVRC2012_val_00000003.jpeg b/demo-client/data/images/ILSVRC2012_val_00000003.jpeg new file mode 100644 index 0000000000000000000000000000000000000000..95438c5f18dace758c6a62cd0955c4212cfb1c1e Binary files /dev/null and b/demo-client/data/images/ILSVRC2012_val_00000003.jpeg differ diff --git a/demo-client/data/images/ILSVRC2012_val_00000004.jpeg b/demo-client/data/images/ILSVRC2012_val_00000004.jpeg new file mode 100644 index 0000000000000000000000000000000000000000..d6c3df86e0a15baba0833b4b7a9c996526e5ac79 Binary files /dev/null and b/demo-client/data/images/ILSVRC2012_val_00000004.jpeg differ diff --git a/demo-client/data/images/ILSVRC2012_val_00000005.jpeg b/demo-client/data/images/ILSVRC2012_val_00000005.jpeg new file mode 100644 index 0000000000000000000000000000000000000000..166f2897e3965f86a0711bce1f929e63c6dd900e Binary files /dev/null and b/demo-client/data/images/ILSVRC2012_val_00000005.jpeg differ diff --git a/demo-client/data/images/ILSVRC2012_val_00000006.jpeg b/demo-client/data/images/ILSVRC2012_val_00000006.jpeg new file mode 100644 index 0000000000000000000000000000000000000000..2f589369058e002d823c8d9c783311a576f9ebdb Binary files /dev/null and b/demo-client/data/images/ILSVRC2012_val_00000006.jpeg differ diff --git a/demo-client/data/images/ILSVRC2012_val_00000007.jpeg b/demo-client/data/images/ILSVRC2012_val_00000007.jpeg new file mode 100644 index 0000000000000000000000000000000000000000..a043ebf28db967c6e1aefb0913fb1540762e3ad0 Binary files /dev/null and b/demo-client/data/images/ILSVRC2012_val_00000007.jpeg differ diff --git a/demo-client/data/images/ILSVRC2012_val_00000008.jpeg b/demo-client/data/images/ILSVRC2012_val_00000008.jpeg new file mode 100644 index 0000000000000000000000000000000000000000..4844c969cc8f4c16adc0809e566f80af62048ed6 Binary files /dev/null and b/demo-client/data/images/ILSVRC2012_val_00000008.jpeg differ diff --git a/demo-client/data/images/ILSVRC2012_val_00000009.jpeg b/demo-client/data/images/ILSVRC2012_val_00000009.jpeg new file mode 100644 index 0000000000000000000000000000000000000000..592eb3a5022ad5df1333dac2935a4cc4ff156aaa Binary files /dev/null and b/demo-client/data/images/ILSVRC2012_val_00000009.jpeg differ diff --git a/demo-client/data/images/ILSVRC2012_val_00000010.jpeg b/demo-client/data/images/ILSVRC2012_val_00000010.jpeg new file mode 100644 index 0000000000000000000000000000000000000000..6cad056bcf3cd66b586248455073b66af0e9afb5 Binary files /dev/null and b/demo-client/data/images/ILSVRC2012_val_00000010.jpeg differ diff --git a/demo-client/data/images/val.txt b/demo-client/data/images/val.txt new file mode 100644 index 0000000000000000000000000000000000000000..da362d6145f3ad9afb5ba2c02b6dd4f6b758ac55 --- /dev/null +++ b/demo-client/data/images/val.txt @@ -0,0 +1,10 @@ +ILSVRC2012_val_00000001.JPEG 65 +ILSVRC2012_val_00000002.JPEG 970 +ILSVRC2012_val_00000003.JPEG 230 +ILSVRC2012_val_00000004.JPEG 809 +ILSVRC2012_val_00000005.JPEG 516 +ILSVRC2012_val_00000006.JPEG 57 +ILSVRC2012_val_00000007.JPEG 334 +ILSVRC2012_val_00000008.JPEG 415 +ILSVRC2012_val_00000009.JPEG 674 +ILSVRC2012_val_00000010.JPEG 332 diff --git a/demo-client/src/ximage_press.cpp b/demo-client/src/ximage_press.cpp new file mode 100644 index 0000000000000000000000000000000000000000..5a71508e79d8d32598b88be4ad26373ef743850a --- /dev/null +++ b/demo-client/src/ximage_press.cpp @@ -0,0 +1,300 @@ +// Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include +#include + +#include +#include // NOLINT +#include "sdk-cpp/builtin_format.pb.h" +#include "sdk-cpp/image_class.pb.h" +#include "sdk-cpp/include/common.h" +#include "sdk-cpp/include/predictor_sdk.h" + +#ifndef BCLOUD +using json2pb::JsonToProtoMessage; +#endif + +using baidu::paddle_serving::sdk_cpp::Predictor; +using baidu::paddle_serving::sdk_cpp::PredictorApi; +using baidu::paddle_serving::predictor::format::XImageReqInstance; +using baidu::paddle_serving::predictor::format::DensePrediction; +using baidu::paddle_serving::predictor::image_classification::Request; +using baidu::paddle_serving::predictor::image_classification::Response; + +DEFINE_int32(concurrency, 1, "Set the max concurrent number of requests"); +DEFINE_int32(requests, 100, "Number of requests to send per thread"); +DEFINE_int32(batch_size, 1, "Batch size"); + +std::atomic g_concurrency(0); + +std::vector> g_round_time; +std::vector g_image_data; +std::vector g_image_lengths; + +const std::vector g_image_paths{ + "./data/images/ILSVRC2012_val_00000001.jpeg", + "./data/images/ILSVRC2012_val_00000002.jpeg", + "./data/images/ILSVRC2012_val_00000003.jpeg", + "./data/images/ILSVRC2012_val_00000004.jpeg", + "./data/images/ILSVRC2012_val_00000005.jpeg", + "./data/images/ILSVRC2012_val_00000006.jpeg", + "./data/images/ILSVRC2012_val_00000007.jpeg", + "./data/images/ILSVRC2012_val_00000008.jpeg", + "./data/images/ILSVRC2012_val_00000009.jpeg", + "./data/images/ILSVRC2012_val_00000010.jpeg"}; + +int prepare_data() { + for (auto x : g_image_paths) { + FILE* fp = fopen(x.c_str(), "rb"); + if (!fp) { + LOG(ERROR) << "Failed open image: " << x.c_str(); + continue; + } + + fseek(fp, 0L, SEEK_END); + size_t isize = ftell(fp); + char* ibuf = new (std::nothrow) char[isize]; + if (!ibuf) { + LOG(ERROR) << "Failed malloc image buffer"; + fclose(fp); + return -1; + } + + fseek(fp, 0, SEEK_SET); + fread(ibuf, sizeof(ibuf[0]), isize, fp); + g_image_data.push_back(ibuf); + g_image_lengths.push_back(isize); + + fclose(fp); + } + + return 0; +} + +int create_req(Request& req) { // NOLINT + for (int i = 0; i < FLAGS_batch_size; ++i) { + XImageReqInstance* ins = req.add_instances(); + if (!ins) { + LOG(ERROR) << "Failed create req instance"; + return -1; + } + + int id = i % g_image_data.size(); + ins->set_image_binary(g_image_data[id], g_image_lengths[id]); + ins->set_image_length(g_image_lengths[id]); + } + + return 0; +} + +void extract_res(const Request& req, const Response& res) { + uint32_t sample_size = res.predictions_size(); + std::string err_string; + for (uint32_t si = 0; si < sample_size; ++si) { + DensePrediction json_msg; + std::string json = res.predictions(si).response_json(); + butil::IOBuf buf; + buf.clear(); + buf.append(json); + butil::IOBufAsZeroCopyInputStream wrapper(buf); + if (!JsonToProtoMessage(&wrapper, &json_msg, &err_string)) { + LOG(ERROR) << "Failed parse json from str:" << json; + return; + } + + uint32_t csize = json_msg.categories_size(); + if (csize <= 0) { + LOG(ERROR) << "sample-" << si << "has no" + << "categories props"; + continue; + } + float max_prop = json_msg.categories(0); + uint32_t max_idx = 0; + for (uint32_t ci = 1; ci < csize; ++ci) { + if (json_msg.categories(ci) > max_prop) { + max_prop = json_msg.categories(ci); + max_idx = ci; + } + } + + LOG(INFO) << "instance " << si << "has class " << max_idx; + } // end for +} + +void thread_worker(PredictorApi* api, int thread_id) { + Request req; + Response res; + + api->thrd_initialize(); + + for (int i = 0; i < FLAGS_requests; ++i) { + api->thrd_clear(); + + Predictor* predictor = api->fetch_predictor("ximage"); + if (!predictor) { + LOG(ERROR) << "Failed fetch predictor: ximage"; + return; + } + + req.Clear(); + res.Clear(); + + if (create_req(req) != 0) { + return; + } + + while (g_concurrency.load() >= FLAGS_concurrency) { + } + g_concurrency++; +#if 1 + LOG(INFO) << "Current concurrency " << g_concurrency.load(); +#endif + + timeval start; + timeval end; + + gettimeofday(&start, NULL); + if (predictor->inference(&req, &res) != 0) { + LOG(ERROR) << "failed call predictor with req:" << req.ShortDebugString(); + return; + } + gettimeofday(&end, NULL); + + g_round_time[thread_id].push_back(end.tv_sec * 1000 + end.tv_usec / 1000 - + start.tv_sec * 1000 - + start.tv_usec / 1000); + + extract_res(req, res); + res.Clear(); + + g_concurrency--; +#if 1 + LOG(INFO) << "Done. Currenct concurrency " << g_concurrency.load(); +#endif + } // for (int i = 0; i < FLAGS_requests; ++i) + + api->thrd_finalize(); +} + +int main(int argc, char** argv) { + google::ParseCommandLineFlags(&argc, &argv, true); + + PredictorApi api; + +// initialize logger instance +#ifdef BCLOUD + logging::LoggingSettings settings; + settings.logging_dest = logging::LOG_TO_FILE; + + std::string filename(argv[0]); + filename = filename.substr(filename.find_last_of('/') + 1); + settings.log_file = (std::string("./log/") + filename + ".log").c_str(); + settings.delete_old = logging::DELETE_OLD_LOG_FILE; + logging::InitLogging(settings); + + logging::ComlogSinkOptions cso; + cso.process_name = filename; + cso.enable_wf_device = true; + logging::ComlogSink::GetInstance()->Setup(&cso); +#else + struct stat st_buf; + int ret = 0; + if ((ret = stat("./log", &st_buf)) != 0) { + mkdir("./log", 0777); + ret = stat("./log", &st_buf); + if (ret != 0) { + LOG(WARNING) << "Log path ./log not exist, and create fail"; + return -1; + } + } + FLAGS_log_dir = "./log"; + google::InitGoogleLogging(strdup(argv[0])); +#endif + + g_round_time.resize(FLAGS_concurrency); + + if (api.create("./conf", "predictors.prototxt") != 0) { + LOG(ERROR) << "Failed create predictors api!"; + return -1; + } + + if (prepare_data() != 0) { + LOG(ERROR) << "Prepare data fail"; + return -1; + } + + std::vector worker_threads; + int i = 0; + for (; i < FLAGS_concurrency; ++i) { + worker_threads.push_back(new std::thread(thread_worker, &api, i)); + } + + for (i = 0; i < FLAGS_concurrency; ++i) { + worker_threads[i]->join(); + delete worker_threads[i]; + } + + api.destroy(); + + std::vector round_times; + for (auto x : g_round_time) { + round_times.insert(round_times.end(), x.begin(), x.end()); + } + + std::sort(round_times.begin(), round_times.end()); + + int percent_pos_50 = round_times.size() * 0.5; + int percent_pos_80 = round_times.size() * 0.8; + int percent_pos_90 = round_times.size() * 0.9; + int percent_pos_99 = round_times.size() * 0.99; + int percent_pos_999 = round_times.size() * 0.999; + + uint64_t total_ms = 0; + for (auto x : round_times) { + total_ms += x; + } + + LOG(INFO) << "Batch size: " << FLAGS_batch_size; + LOG(INFO) << "Total requests: " << round_times.size(); + LOG(INFO) << "Max concurrency: " << FLAGS_concurrency; + LOG(INFO) << "Total ms (absolute time): " << total_ms / FLAGS_concurrency; + + double qps = 0.0; + if (total_ms != 0) { + qps = (static_cast(FLAGS_concurrency * FLAGS_requests) / + (total_ms / FLAGS_concurrency)) * + 1000; + } + + LOG(INFO) << "QPS: " << qps << "/s"; + + LOG(INFO) << "Latency statistics: "; + if (round_times.size() != 0) { + LOG(INFO) << "Average ms: " + << static_cast(total_ms) / round_times.size(); + LOG(INFO) << "50 percent ms: " << round_times[percent_pos_50]; + LOG(INFO) << "80 percent ms: " << round_times[percent_pos_80]; + LOG(INFO) << "90 percent ms: " << round_times[percent_pos_90]; + LOG(INFO) << "99 percent ms: " << round_times[percent_pos_99]; + LOG(INFO) << "99.9 percent ms: " << round_times[percent_pos_999]; + } else { + LOG(INFO) << "N/A"; + } + + return 0; +} + +/* vim: set expandtab ts=4 sw=4 sts=4 tw=100: */ diff --git a/demo-serving/CMakeLists.txt b/demo-serving/CMakeLists.txt index f9c6cab63a6c7e1fe04253e8aae6cbd8d8478e06..238fadf8934ca6bc8051e3f2d662c6fbe99170db 100644 --- a/demo-serving/CMakeLists.txt +++ b/demo-serving/CMakeLists.txt @@ -18,9 +18,17 @@ include(proto/CMakeLists.txt) add_executable(serving ${serving_srcs}) add_dependencies(serving pdcodegen fluid_cpu_engine pdserving paddle_fluid opencv_imgcodecs) +if (WITH_GPU) + add_dependencies(serving fluid_gpu_engine) +endif() target_include_directories(serving PUBLIC ${CMAKE_CURRENT_BINARY_DIR}/../predictor ) + +if(WITH_GPU) + target_link_libraries(serving ${CUDA_LIBRARIES} -Wl,--whole-archive fluid_gpu_engine + -Wl,--no-whole-archive) +endif() target_link_libraries(serving opencv_imgcodecs ${opencv_depend_libs} -Wl,--whole-archive fluid_cpu_engine -Wl,--no-whole-archive pdserving paddle_fluid ${paddle_depend_libs} diff --git a/demo-serving/op/classify_op.cpp b/demo-serving/op/classify_op.cpp index a714e7cd820113280f3deabbe0e95922cdee8d9a..22c3393866c002985d307168fb64b4cd88d4974e 100644 --- a/demo-serving/op/classify_op.cpp +++ b/demo-serving/op/classify_op.cpp @@ -35,7 +35,6 @@ int ClassifyOp::inference() { } const TensorVector* in = &reader_out->tensors; - uint32_t sample_size = in->size(); TensorVector* out = butil::get_object(); if (!out) { @@ -43,20 +42,21 @@ int ClassifyOp::inference() { return -1; } - if (sample_size <= 0) { - LOG(INFO) << "No samples need to to predicted"; - return 0; + if (in->size() != 1) { + LOG(ERROR) << "Samples should have been packed into a single tensor"; + return -1; } + int batch_size = in->at(0).shape[0]; // call paddle fluid model for inferencing if (InferManager::instance().infer( - IMAGE_CLASSIFICATION_MODEL_NAME, in, out, sample_size)) { + IMAGE_CLASSIFICATION_MODEL_NAME, in, out, batch_size)) { LOG(ERROR) << "Failed do infer in fluid model: " << IMAGE_CLASSIFICATION_MODEL_NAME; return -1; } - if (out->size() != sample_size) { + if (out->size() != in->size()) { LOG(ERROR) << "Output size not eq input size: " << in->size() << out->size(); return -1; @@ -64,24 +64,35 @@ int ClassifyOp::inference() { // copy output tensor into response ClassifyResponse* res = mutable_data(); + const paddle::PaddleTensor& out_tensor = (*out)[0]; + +#if 0 + int out_shape_size = out_tensor.shape.size(); + LOG(ERROR) << "out_tensor.shpae"; + for (int i = 0; i < out_shape_size; ++i) { + LOG(ERROR) << out_tensor.shape[i] << ":"; + } + + if (out_shape_size != 2) { + return -1; + } +#endif + + int sample_size = out_tensor.shape[0]; +#if 0 + LOG(ERROR) << "Output sample size " << sample_size; +#endif for (uint32_t si = 0; si < sample_size; si++) { - const paddle::PaddleTensor& out_tensor = (*out)[si]; DensePrediction* ins = res->add_predictions(); if (!ins) { LOG(ERROR) << "Failed append new out tensor"; return -1; } - uint32_t shape_size = out_tensor.shape.size(); - if (out_tensor.shape.size() != 2 || out_tensor.shape[0] != 1) { - LOG(ERROR) << "Not valid classification out shape" - << ", shape size: " << out_tensor.shape.size(); - return -1; - } - // assign output data - uint32_t data_size = out_tensor.data.length() / sizeof(float); - float* data = reinterpret_cast(out_tensor.data.data()); + uint32_t data_size = out_tensor.shape[1]; + float* data = reinterpret_cast(out_tensor.data.data() + + si * sizeof(float) * data_size); for (uint32_t di = 0; di < data_size; ++di) { ins->add_categories(data[di]); } @@ -95,10 +106,6 @@ int ClassifyOp::inference() { out->clear(); butil::return_object(out); - LOG(INFO) << "Response in image classification:" - << "length:" << res->ByteSize() << "," - << "data:" << res->ShortDebugString(); - return 0; } diff --git a/demo-serving/op/reader_op.cpp b/demo-serving/op/reader_op.cpp index 8c97702de33215f9b33e201c0b02de6a2a8d7d08..d50c9ebd24be48c32d59aa641bace52cf556a337 100644 --- a/demo-serving/op/reader_op.cpp +++ b/demo-serving/op/reader_op.cpp @@ -51,6 +51,26 @@ int ReaderOp::inference() { resize.height = iresize[0]; resize.width = iresize[1]; + paddle::PaddleTensor in_tensor; + in_tensor.name = "tensor"; + in_tensor.dtype = paddle::FLOAT32; + // shape assignment + in_tensor.shape.push_back(sample_size); // batch_size + in_tensor.shape.push_back(3); + in_tensor.shape.push_back(resize.width); + in_tensor.shape.push_back(resize.height); + + // tls resource assignment + size_t dense_capacity = 3 * resize.width * resize.height; + size_t len = dense_capacity * sizeof(float) * sample_size; + float* data = + reinterpret_cast(MempoolWrapper::instance().malloc(len)); + if (data == NULL) { + LOG(ERROR) << "Failed create temp float array, " + << "size=" << dense_capacity * sample_size * sizeof(float); + return -1; + } + for (uint32_t si = 0; si < sample_size; si++) { // parse image object from x-image const XImageReqInstance& ins = req->instances(si); @@ -103,50 +123,31 @@ int ReaderOp::inference() { const int H = _image_8u_rgb.rows; const int W = _image_8u_rgb.cols; const int C = _image_8u_rgb.channels(); - size_t dense_capacity = H * W * C; - - paddle::PaddleTensor in_tensor; - in_tensor.name = "tensor"; - in_tensor.dtype = paddle::FLOAT32; - - // shape assignment - in_tensor.shape.push_back(1); // batch_size - - // accoreding to training stage, the instance shape should be - // in order of C-W-H. - in_tensor.shape.push_back(C); - in_tensor.shape.push_back(W); - in_tensor.shape.push_back(H); + if (H != resize.height || W != resize.width || C != 3) { + LOG(ERROR) << "Image " << si << " has incompitable size"; + return -1; + } LOG(INFO) << "Succ read one image, C: " << C << ", W: " << W << ", H: " << H; - // tls resource assignment - size_t len = dense_capacity * sizeof(float); - float* data = - reinterpret_cast(MempoolWrapper::instance().malloc(len)); - if (data == NULL) { - LOG(ERROR) << "Failed create temp float array, " - << "size=" << dense_capacity; - return -1; - } - + float* data_ptr = data + dense_capacity * si; for (int h = 0; h < H; h++) { // p points to a new line unsigned char* p = _image_8u_rgb.ptr(h); for (int w = 0; w < W; w++) { for (int c = 0; c < C; c++) { // HWC(row,column,channel) -> CWH - data[W * H * c + W * h + w] = (p[C * w + c] - pmean[c]) * scale[c]; + data_ptr[W * H * c + W * h + w] = + (p[C * w + c] - pmean[c]) * scale[c]; } } } - - paddle::PaddleBuf pbuf(data, len); - in_tensor.data = pbuf; - - in->push_back(in_tensor); } + paddle::PaddleBuf pbuf(data, len); + in_tensor.data = pbuf; + + in->push_back(in_tensor); return 0; } diff --git a/demo-serving/op/write_json_op.cpp b/demo-serving/op/write_json_op.cpp index 25aece0e4533ab5a535e86d2c4cbff259754436c..e0b372d7c8fce86ced556ea397c9dcbb05111f66 100644 --- a/demo-serving/op/write_json_op.cpp +++ b/demo-serving/op/write_json_op.cpp @@ -16,7 +16,7 @@ #include #ifdef BCLOUD -#include "pb_to_json.h" +#include "pb_to_json.h" // NOLINT #else #include "json2pb/pb_to_json.h" #endif @@ -70,7 +70,7 @@ int WriteJsonOp::inference() { } } - LOG(INFO) << "Succ write json:" << classify_out->ShortDebugString(); + LOG(INFO) << "Succ write json"; return 0; } diff --git a/doc/FAQ.md b/doc/FAQ.md index a48e5c4dbc6562a8035d4ea0731b06183e2e8f3c..2ba9ec9d5e0a5d7c8f0ccc3ebfc480f21170751d 100644 --- a/doc/FAQ.md +++ b/doc/FAQ.md @@ -5,3 +5,22 @@ - 如果在inferservice_file里指定了port:xxx，那么就去申请该端口号； - 否则，如果在gflags.conf里指定了--port:xxx，那就去申请该端口号； - 否则，使用程序里指定的默认端口号：8010。 + +## 2. GPU预测中为何请求的响应时间波动会非常大？ +PaddleServing依托PaddlePaddle预测库执行预测计算；在GPU设备上，由于同一个进程内目前共用1个GPU stream，进程内的多个请求的预测计算会被严格串行。所以如果有2个请求同时到达某个Serving实例，不管该实例启动时创建了多少个worker线程，都不能起到加速作用，后到的请求会被排队，直到前面请求计算完成。 + +## 3. 如何充分利用GPU卡的计算能力？ +如问题2所说，由于预测库的限制，单个Serving进程只能绑定单张GPU卡，且进程内共用1个GPU stream，所有请求必须串行计算。 + +为提高GPU卡使用率，目前可以想到的方法是：在单张GPU卡上启动多个Serving进程，每个进程绑定一个GPU stream，多个stream并行计算。这种方法是否能起到加速作用，受限于多个因素，主要有： + +1. 单个stream占用GPU算力；假如单个stream已经将GPU算力占用超过50%，那么增加stream很可能会导致2个stream的job分别排队，拖慢各自的响应时间 +2. GPU显存：Serving进程需要将模型参数加载到显存中，并且计算时要在GPU显存池分配临时变量；假如单个Serving进程已经用掉超过50%的显存，则增加Serving进程会造成显存不足，导致进程报错退出 + +为此，可采用如下步骤，进行测试： + +1. 加载模型时，在model_toolkit.prototxt中，model type选择FLUID_GPU_ANALYSIS或FLUID_GPU_ANALYSIS_DIR；会对模型进行静态分析，进行一定程度显存优化 +2. 在步骤1完成后，启动单个Serving进程，启动参数:`--gpuid=N --bthread_concurrency=4 --bthread_min_concurrency=4`；启动一个client，进行并发度为1的压力测试，batch size从小到大，记下平响；由于算力的限制，当batch size增大到一定程度，应该会出现响应时间明显变大；或虽然没有明显变大，但已经不满足系统需求 +3. 再启动1个Serving进程，与步骤2启动时使用相同的参数略有不同: `--gpuid=N --bthread_concurrency=4 --bthread_min_concurrency=4 --port=8011` 其中--port=8011用来让新启动的进程使用一个新的服务端口；然后同时对这2个Serving进程进行压测，继续观察batch size从小到大时平均响应时间的变化，直到取得batch size和响应时间的折中 +4. 重复步骤2-3 +5. 以2-4步的测试，来决定：单张GPU卡可以由多少个Serving进程共用; 实际部署时，就在一张GPU卡上启动这么多个Serving进程同时提供服务 diff --git a/doc/INDEX.md b/doc/INDEX.md index c4e78be00435e3b936b9fa08785098028e53de17..c9399b2f5cf50cda10062023f009b92bfc0c8158 100644 --- a/doc/INDEX.md +++ b/doc/INDEX.md @@ -1,16 +1,19 @@ +[Design](DESIGN.md) -[Client Configure](CLIENT_CONFIGURE.md) +[Installation](INSTALL.md) -[How to Configure a Clustered Service](CLUSTERING.md) +[Getting Started](GETTING_STARTED.md) [Creating a Prediction Service](CREATING.md) -[Design](DESIGN.md) +[Client Configure](CLIENT_CONFIGURE.md) -[FAQ](FAQ.md) +[Server Side Configuration](SERVING_CONFIGURE.md) -[Getting Started](GETTING_STARTED.md) +[How to Configure a Clustered Service](CLUSTERING.md) -[Installation](INSTALL.md) +[Multiple Serving Instances over Single GPU Card](MULTI_SERVING_OVER_SINGLE_GPU_CARD.md) -[Server Side Configuration](SERVING_CONFIGURE.md) +[Benchmarking](BENCHMARKING.md) + +[FAQ](FAQ.md) diff --git a/doc/INSTALL.md b/doc/INSTALL.md index 64e482d6438f2d6e5da8ea6ce98e0933b18a0f7b..d3114e86efbf2cd5985811d4d39ec7e8069e3534 100644 --- a/doc/INSTALL.md +++ b/doc/INSTALL.md @@ -58,10 +58,9 @@ $ make install # CMake编译选项说明 -因Paddle Serving依托于PaddlePaddle项目进行构建，以下编译选项其实是传递给PaddlePaddle的编译选项： - | 编译选项 | 说明 | |----------|------| -| WITH_AVX | Compile PaddlePaddle with AVX intrinsics | -| WITH_MKL | Compile PaddlePaddle with MKLML library | +| WITH_AVX | For configuring PaddlePaddle. Compile PaddlePaddle with AVX intrinsics | +| WITH_MKL | For configuring PaddlePaddle. Compile PaddlePaddle with MKLML library | +| WITH_GPU | For configuring PaddlePaddle. Compile PaddlePaddle with NVIDIA GPU | | CLINET_ONLY | Compile client libraries and demos only | diff --git a/doc/MULTI_SERVING_OVER_SINGLE_GPU_CARD.md b/doc/MULTI_SERVING_OVER_SINGLE_GPU_CARD.md new file mode 100644 index 0000000000000000000000000000000000000000..1bbcaf16a38d551998402c0f52df581dc9dc9866 --- /dev/null +++ b/doc/MULTI_SERVING_OVER_SINGLE_GPU_CARD.md @@ -0,0 +1,32 @@ +# Multiple Serving Instances over Single GPU Card + +Paddle Serving依托PaddlePaddle预测库执行实际的预测计算。由于当前GPU预测库的限制，单个Serving实例只可以绑定1张GPU卡，且进程内所有worker线程共用1个GPU stream。也就是说，不管Serving启动多少个worker线程，所有的请求在GPU是严格串行计算的，起不到加速作用。这会带来一个问题，就是如果模型计算量不大，那么Serving进程实际上不会用满GPU的算力。 + +为了充分利用GPU卡的算力，考虑在单张卡上启动多个Serving实例，通过多个GPU stream，力争用满GPU的算力。启动命令可以如下所示： + +``` +bin/serving --gpuid=0 --bthread_concurrency=4 --bthread_min_concurrency=4 --port=8010& +bin/serving --gpuid=0 --bthread_concurrency=4 --bthread_min_concurrency=4 --port=8011& +``` + +上述2条命令，启动2个Serving实例，分别监听8010端口和8011端口。但他们都绑定同一张卡 (gpuid = 0)。 + +命令行参数含义： +``` +-gpuid=N：用于指定所绑定的GPU卡ID +-bthread_concurrency和bthread_min_concurrency共同限制该进程启动的worker数：由于在GPU预测模式下，增加worker线程数并不能提高并发能力，为了节省部分资源，干脆将他们限制掉；均设为4，是因为这是bthread允许的最小值。 +-port xxx：Serving实例监听的端口 +``` + +但是，上述方式究竟是否能在不影响响应时间等其他指标的前提下，起到提高GPU使用率作用，受到多个限制因素的制约，具体的： + +1. 单个stream占用GPU算力；假如单个stream已经将GPU算力占用超过50%，那么增加stream很可能会导致2个stream的job分别排队，拖慢各自的响应时间 +2. GPU显存：Serving进程需要将模型参数加载到显存中，并且计算时要在GPU显存池分配临时变量；假如单个Serving进程已经用掉超过50%的显存，则增加Serving进程会造成显存不足，导致进程报错退出 + +为此，可采用如下步骤，进行测试： + +1. 加载模型时，在model_toolkit.prototxt中，model type选择FLUID_GPU_ANALYSIS或FLUID_GPU_ANALYSIS_DIR；会对模型进行静态分析，进行一定程度显存优化 +2. 在步骤1完成后，启动单个Serving进程，启动参数:`--gpuid=N --bthread_concurrency=4 --bthread_min_concurrency=4`；启动一个client，进行并发度为1的压力测试，batch size从小到大，记下平响；由于算力的限制，当batch size增大到一定程度，应该会出现响应时间明显变大；或虽然没有明显变大，但已经不满足系统需求 +3. 再启动1个Serving进程，与步骤2启动时使用相同的参数略有不同: `--gpuid=N --bthread_concurrency=4 --bthread_min_concurrency=4 --port=8011` 其中--port=8011用来让新启动的进程使用一个新的服务端口；然后同时对这2个Serving进程进行压测，继续观察batch size从小到大时平均响应时间的变化，直到取得batch size和响应时间的折中 +4. 重复步骤2-3 +5. 以2-4步的测试，来决定：单张GPU卡可以由多少个Serving进程共用; 实际部署时，就在一张GPU卡上启动这么多个Serving进程同时提供服务 diff --git a/doc/SERVING_CONFIGURE.md b/doc/SERVING_CONFIGURE.md index 2b452532364616f080662022ff159c239a3bf0e5..f5887f5cd139b1e70b49f4eee2e2552658692103 100644 --- a/doc/SERVING_CONFIGURE.md +++ b/doc/SERVING_CONFIGURE.md @@ -142,6 +142,11 @@ type: 预测引擎的类型。可在inferencer-fluid-cpu/src/fluid_cpu_engine.cp |FLUID_CPU_ANALYSIS_DIR|使用fluid Analysis API；模型所有参数分开保存为独立的文件，整个模型放到一个目录中| |FLUID_CPU_NATIVE|使用fluid Native API；模型所有参数保存在一个文件| |FLUID_CPU_NATIVE_DIR|使用fluid Native API；模型所有参数分开保存为独立的文件，整个模型放到一个目录中| +|FLUID_GPU_ANALYSIS|GPU预测，使用fluid Analysis API；模型所有参数保存在一个文件| +|FLUID_GPU_ANALYSIS_DIR|GPU预测，使用fluid Analysis API；模型所有参数分开保存为独立的文件，整个模型放到一个目录中| +|FLUID_GPU_NATIVE|GPU预测，使用fluid Native API；模型所有参数保存在一个文件| +|FLUID_GPU_NATIVE_DIR|GPU预测，使用fluid Native API；模型所有参数分开保存为独立的文件，整个模型放到一个目录中| + **fluid Analysis API和fluid Native API的区别** @@ -182,8 +187,11 @@ enable_batch_align: |enable_model_toolkit|true|模型管理| |enable_protocol_list|baidu_std|brpc 通信协议列表| |log_dir|./log|log dir| -|num_threads|brpc server使用的系统线程数，默认为CPU核数| -|max_concurrency|并发处理的请求数，设为<=0则为不予限制，若大于0则限定brpc server端同时处理的请求数上限| +|num_threads||brpc server使用的系统线程数，默认为CPU核数| +|port|8010|Serving进程接收请求监听端口| +|gpuid|0|GPU预测时指定Serving进程使用的GPU device id。只允许绑定1张GPU卡| +|bthread_concurrency|9|BRPC底层bthread的concurrency。在使用GPU预测引擎时，为了限制并发worker数，可使用此参数| +|bthread_min_concurrency|4|BRPC底层bthread的min concurrency。在使用GPU预测引擎时，为限制并发worker数，可使用此参数。与bthread_concurrency结合使用| 可以通过在serving/conf/gflags.conf覆盖默认值，例如 ``` diff --git a/inferencer-fluid-cpu/include/fluid_cpu_engine.h b/inferencer-fluid-cpu/include/fluid_cpu_engine.h index f01f7fce418278ea45eee4cc3558a1da0b0dd094..82b924ac3423d9361d6871eda64f9f083d98402b 100644 --- a/inferencer-fluid-cpu/include/fluid_cpu_engine.h +++ b/inferencer-fluid-cpu/include/fluid_cpu_engine.h @@ -155,6 +155,8 @@ class FluidCpuNativeCore : public FluidFamilyCore { native_config.prog_file = data_path + "/__model__"; native_config.use_gpu = false; native_config.device = 0; + native_config.fraction_of_gpu_memory = 0; + AutoLock lock(GlobalPaddleCreateMutex::instance()); _core = paddle::CreatePaddlePredictor( @@ -209,6 +211,7 @@ class FluidCpuNativeDirCore : public FluidFamilyCore { native_config.model_dir = data_path; native_config.use_gpu = false; native_config.device = 0; + native_config.fraction_of_gpu_memory = 0; AutoLock lock(GlobalPaddleCreateMutex::instance()); _core = paddle::CreatePaddlePredictor( @@ -458,6 +461,7 @@ class FluidCpuNativeDirWithSigmoidCore : public FluidCpuWithSigmoidCore { native_config.model_dir = data_path; native_config.use_gpu = false; native_config.device = 0; + native_config.fraction_of_gpu_memory = 0; AutoLock lock(GlobalPaddleCreateMutex::instance()); _core->_fluid_core = paddle::CreatePaddlePredictor +#include +#include +#include +#include +#include "configure/include/configure_parser.h" +#include "configure/inferencer_configure.pb.h" +#ifdef BCLOUD +#ifdef WITH_GPU +#include "paddle/paddle_inference_api.h" +#else +#include "paddle/fluid/inference/api/paddle_inference_api.h" +#endif +#else +#include "paddle/fluid/inference/paddle_inference_api.h" +#endif +#include "predictor/framework/infer.h" + +DECLARE_int32(gpuid); + +namespace baidu { +namespace paddle_serving { +namespace fluid_gpu { + +using configure::SigmoidConf; + +class AutoLock { + public: + explicit AutoLock(pthread_mutex_t& mutex) : _mut(mutex) { + pthread_mutex_lock(&mutex); + } + + ~AutoLock() { pthread_mutex_unlock(&_mut); } + + private: + pthread_mutex_t& _mut; +}; + +class GlobalPaddleCreateMutex { + public: + pthread_mutex_t& mutex() { return _mut; } + + static pthread_mutex_t& instance() { + static GlobalPaddleCreateMutex gmutex; + return gmutex.mutex(); + } + + private: + GlobalPaddleCreateMutex() { pthread_mutex_init(&_mut, NULL); } + + pthread_mutex_t _mut; +}; + +class GlobalSigmoidCreateMutex { + public: + pthread_mutex_t& mutex() { return _mut; } + static pthread_mutex_t& instance() { + static GlobalSigmoidCreateMutex gmutex; + return gmutex.mutex(); + } + + private: + GlobalSigmoidCreateMutex() { pthread_mutex_init(&_mut, NULL); } + + pthread_mutex_t _mut; +}; + +// data interface +class FluidFamilyCore { + public: + virtual ~FluidFamilyCore() {} + virtual bool Run(const void* in_data, void* out_data) { + if (!_core->Run(*(std::vector*)in_data, + (std::vector*)out_data)) { + LOG(ERROR) << "Failed call Run with paddle predictor"; + return false; + } + + return true; + } + + virtual int create(const std::string& data_path) = 0; + + virtual int clone(void* origin_core) { + if (origin_core == NULL) { + LOG(ERROR) << "origin paddle Predictor is null."; + return -1; + } + paddle::PaddlePredictor* p_predictor = + (paddle::PaddlePredictor*)origin_core; + _core = p_predictor->Clone(); + if (_core.get() == NULL) { + LOG(ERROR) << "fail to clone paddle predictor: " << origin_core; + return -1; + } + return 0; + } + + virtual void* get() { return _core.get(); } + + protected: + std::unique_ptr _core; +}; + +// infer interface +class FluidGpuAnalysisCore : public FluidFamilyCore { + public: + int create(const std::string& data_path) { + if (access(data_path.c_str(), F_OK) == -1) { + LOG(ERROR) << "create paddle predictor failed, path not exits: " + << data_path; + return -1; + } + + paddle::AnalysisConfig analysis_config; + analysis_config.SetParamsFile(data_path + "/__params__"); + analysis_config.SetProgFile(data_path + "/__model__"); + analysis_config.EnableUseGpu(100, FLAGS_gpuid); + analysis_config.SetCpuMathLibraryNumThreads(1); + analysis_config.SwitchSpecifyInputNames(true); + analysis_config.EnableMemoryOptim(); + AutoLock lock(GlobalPaddleCreateMutex::instance()); + _core = + paddle::CreatePaddlePredictor(analysis_config); + if (NULL == _core.get()) { + LOG(ERROR) << "create paddle predictor failed, path: " << data_path; + return -1; + } + + LOG(WARNING) << "create paddle predictor sucess, path: " << data_path; + return 0; + } +}; + +class FluidGpuNativeCore : public FluidFamilyCore { + public: + int create(const std::string& data_path) { + if (access(data_path.c_str(), F_OK) == -1) { + LOG(ERROR) << "create paddle predictor failed, path not exits: " + << data_path; + return -1; + } + + paddle::NativeConfig native_config; + native_config.param_file = data_path + "/__params__"; + native_config.prog_file = data_path + "/__model__"; + native_config.use_gpu = true; + native_config.fraction_of_gpu_memory = 0.01; + native_config.device = FLAGS_gpuid; + AutoLock lock(GlobalPaddleCreateMutex::instance()); + _core = paddle::CreatePaddlePredictor( + native_config); + if (NULL == _core.get()) { + LOG(ERROR) << "create paddle predictor failed, path: " << data_path; + return -1; + } + + LOG(WARNING) << "create paddle predictor sucess, path: " << data_path; + return 0; + } +}; + +class FluidGpuAnalysisDirCore : public FluidFamilyCore { + public: + int create(const std::string& data_path) { + if (access(data_path.c_str(), F_OK) == -1) { + LOG(ERROR) << "create paddle predictor failed, path not exits: " + << data_path; + return -1; + } + + paddle::AnalysisConfig analysis_config; + analysis_config.SetModel(data_path); + analysis_config.EnableUseGpu(100, FLAGS_gpuid); + analysis_config.SwitchSpecifyInputNames(true); + analysis_config.SetCpuMathLibraryNumThreads(1); + analysis_config.EnableMemoryOptim(); + AutoLock lock(GlobalPaddleCreateMutex::instance()); + _core = + paddle::CreatePaddlePredictor(analysis_config); + if (NULL == _core.get()) { + LOG(ERROR) << "create paddle predictor failed, path: " << data_path; + return -1; + } + + LOG(WARNING) << "create paddle predictor sucess, path: " << data_path; + return 0; + } +}; + +class FluidGpuNativeDirCore : public FluidFamilyCore { + public: + int create(const std::string& data_path) { + if (access(data_path.c_str(), F_OK) == -1) { + LOG(ERROR) << "create paddle predictor failed, path not exits: " + << data_path; + return -1; + } + + paddle::NativeConfig native_config; + native_config.model_dir = data_path; + native_config.use_gpu = true; + native_config.fraction_of_gpu_memory = 0.01; + native_config.device = FLAGS_gpuid; + AutoLock lock(GlobalPaddleCreateMutex::instance()); + _core = paddle::CreatePaddlePredictor( + native_config); + if (NULL == _core.get()) { + LOG(ERROR) << "create paddle predictor failed, path: " << data_path; + return -1; + } + + LOG(WARNING) << "create paddle predictor sucess, path: " << data_path; + return 0; + } +}; + +class Parameter { + public: + Parameter() : _row(0), _col(0), _params(NULL) {} + ~Parameter() { + LOG(INFO) << "before destroy Parameter, file_name[" << _file_name << "]"; + destroy(); + } + + int init(int row, int col, const char* file_name) { + destroy(); + _file_name = file_name; + _row = row; + _col = col; + _params = reinterpret_cast(malloc(_row * _col * sizeof(float))); + if (_params == NULL) { + LOG(ERROR) << "Load " << _file_name << " malloc error."; + return -1; + } + LOG(WARNING) << "Load parameter file[" << _file_name << "] success."; + return 0; + } + + void destroy() { + _row = 0; + _col = 0; + if (_params != NULL) { + free(_params); + _params = NULL; + } + } + + int load() { + if (_params == NULL || _row <= 0 || _col <= 0) { + LOG(ERROR) << "load parameter error [not inited]."; + return -1; + } + + FILE* fs = fopen(_file_name.c_str(), "rb"); + if (fs == NULL) { + LOG(ERROR) << "load " << _file_name << " fopen error."; + return -1; + } + static const uint32_t MODEL_FILE_HEAD_LEN = 16; + char head[MODEL_FILE_HEAD_LEN] = {0}; + if (fread(head, 1, MODEL_FILE_HEAD_LEN, fs) != MODEL_FILE_HEAD_LEN) { + destroy(); + LOG(ERROR) << "Load " << _file_name << " read head error."; + if (fs != NULL) { + fclose(fs); + fs = NULL; + } + return -1; + } + + uint32_t matrix_size = _row * _col; + if (matrix_size == fread(_params, sizeof(float), matrix_size, fs)) { + if (fs != NULL) { + fclose(fs); + fs = NULL; + } + LOG(INFO) << "load " << _file_name << " read ok."; + return 0; + } else { + LOG(ERROR) << "load " << _file_name << " read error."; + destroy(); + if (fs != NULL) { + fclose(fs); + fs = NULL; + } + return -1; + } + return 0; + } + + public: + std::string _file_name; + int _row; + int _col; + float* _params; +}; + +class SigmoidModel { + public: + ~SigmoidModel() {} + int load(const char* sigmoid_w_file, + const char* sigmoid_b_file, + float exp_max, + float exp_min) { + AutoLock lock(GlobalSigmoidCreateMutex::instance()); + if (0 != _sigmoid_w.init(2, 1, sigmoid_w_file) || 0 != _sigmoid_w.load()) { + LOG(ERROR) << "load params sigmoid_w failed."; + return -1; + } + LOG(WARNING) << "load sigmoid_w [" << _sigmoid_w._params[0] << "] [" + << _sigmoid_w._params[1] << "]."; + if (0 != _sigmoid_b.init(2, 1, sigmoid_b_file) || 0 != _sigmoid_b.load()) { + LOG(ERROR) << "load params sigmoid_b failed."; + return -1; + } + LOG(WARNING) << "load sigmoid_b [" << _sigmoid_b._params[0] << "] [" + << _sigmoid_b._params[1] << "]."; + _exp_max_input = exp_max; + _exp_min_input = exp_min; + return 0; + } + + int softmax(float x, double& o) { // NOLINT + float _y0 = x * _sigmoid_w._params[0] + _sigmoid_b._params[0]; + float _y1 = x * _sigmoid_w._params[1] + _sigmoid_b._params[1]; + _y0 = (_y0 > _exp_max_input) + ? _exp_max_input + : ((_y0 < _exp_min_input) ? _exp_min_input : _y0); + _y1 = (_y1 > _exp_max_input) + ? _exp_max_input + : ((_y1 < _exp_min_input) ? _exp_min_input : _y1); + o = 1.0f / (1.0f + exp(_y0 - _y1)); + return 0; + } + + public: + Parameter _sigmoid_w; + Parameter _sigmoid_b; + float _exp_max_input; + float _exp_min_input; +}; + +class SigmoidFluidModel { + public: + int softmax(float x, double& o) { // NOLINT + return _sigmoid_core->softmax(x, o); + } // NOLINT + + std::unique_ptr Clone() { + std::unique_ptr clone_model; + clone_model.reset(new SigmoidFluidModel()); + clone_model->_sigmoid_core = _sigmoid_core; + clone_model->_fluid_core = _fluid_core->Clone(); + return std::move(clone_model); + } + + public: + std::unique_ptr _fluid_core; + std::shared_ptr _sigmoid_core; +}; + +class FluidGpuWithSigmoidCore : public FluidFamilyCore { + public: + virtual ~FluidGpuWithSigmoidCore() {} + + public: + int create(const std::string& model_path) { + size_t pos = model_path.find_last_of("/\\"); + std::string conf_path = model_path.substr(0, pos); + std::string conf_file = model_path.substr(pos); + configure::SigmoidConf conf; + if (configure::read_proto_conf(conf_path, conf_file, &conf) != 0) { + LOG(ERROR) << "failed load model path: " << model_path; + return -1; + } + + _core.reset(new SigmoidFluidModel); + + std::string fluid_model_data_path = conf.dnn_model_path(); + int ret = load_fluid_model(fluid_model_data_path); + if (ret < 0) { + LOG(ERROR) << "fail to load fluid model."; + return -1; + } + const char* sigmoid_w_file = conf.sigmoid_w_file().c_str(); + const char* sigmoid_b_file = conf.sigmoid_b_file().c_str(); + float exp_max = conf.exp_max_input(); + float exp_min = conf.exp_min_input(); + _core->_sigmoid_core.reset(new SigmoidModel); + LOG(INFO) << "create sigmoid core[" << _core->_sigmoid_core.get() + << "], use count[" << _core->_sigmoid_core.use_count() << "]."; + ret = _core->_sigmoid_core->load( + sigmoid_w_file, sigmoid_b_file, exp_max, exp_min); + if (ret < 0) { + LOG(ERROR) << "fail to load sigmoid model."; + return -1; + } + return 0; + } + + virtual bool Run(const void* in_data, void* out_data) { + if (!_core->_fluid_core->Run( + *(std::vector*)in_data, + (std::vector*)out_data)) { + LOG(ERROR) << "Failed call Run with paddle predictor"; + return false; + } + + return true; + } + + virtual int clone(SigmoidFluidModel* origin_core) { + if (origin_core == NULL) { + LOG(ERROR) << "origin paddle Predictor is null."; + return -1; + } + _core = origin_core->Clone(); + if (_core.get() == NULL) { + LOG(ERROR) << "fail to clone paddle predictor: " << origin_core; + return -1; + } + LOG(INFO) << "clone sigmoid core[" << _core->_sigmoid_core.get() + << "] use count[" << _core->_sigmoid_core.use_count() << "]."; + return 0; + } + + virtual SigmoidFluidModel* get() { return _core.get(); } + + virtual int load_fluid_model(const std::string& data_path) = 0; + + int softmax(float x, double& o) { // NOLINT + return _core->_sigmoid_core->softmax(x, o); + } + + protected: + std::unique_ptr _core; +}; + +class FluidGpuNativeDirWithSigmoidCore : public FluidGpuWithSigmoidCore { + public: + int load_fluid_model(const std::string& data_path) { + if (access(data_path.c_str(), F_OK) == -1) { + LOG(ERROR) << "create paddle predictor failed, path not exits: " + << data_path; + return -1; + } + + paddle::NativeConfig native_config; + native_config.model_dir = data_path; + native_config.use_gpu = true; + native_config.fraction_of_gpu_memory = 0.01; + native_config.device = FLAGS_gpuid; + AutoLock lock(GlobalPaddleCreateMutex::instance()); + _core->_fluid_core = + paddle::CreatePaddlePredictor( + native_config); + if (NULL == _core.get()) { + LOG(ERROR) << "create paddle predictor failed, path: " << data_path; + return -1; + } + + LOG(WARNING) << "create paddle predictor sucess, path: " << data_path; + return 0; + } +}; + +class FluidGpuAnalysisDirWithSigmoidCore : public FluidGpuWithSigmoidCore { + public: + int load_fluid_model(const std::string& data_path) { + if (access(data_path.c_str(), F_OK) == -1) { + LOG(ERROR) << "create paddle predictor failed, path not exits: " + << data_path; + return -1; + } + + paddle::AnalysisConfig analysis_config; + analysis_config.SetModel(data_path); + analysis_config.EnableUseGpu(100, FLAGS_gpuid); + analysis_config.SwitchSpecifyInputNames(true); + analysis_config.SetCpuMathLibraryNumThreads(1); + analysis_config.EnableMemoryOptim(); + AutoLock lock(GlobalPaddleCreateMutex::instance()); + _core->_fluid_core = + paddle::CreatePaddlePredictor(analysis_config); + if (NULL == _core.get()) { + LOG(ERROR) << "create paddle predictor failed, path: " << data_path; + return -1; + } + + LOG(WARNING) << "create paddle predictor sucess, path: " << data_path; + return 0; + } +}; + +} // namespace fluid_gpu +} // namespace paddle_serving +} // namespace baidu diff --git a/inferencer-fluid-gpu/src/fluid_gpu_engine.cpp b/inferencer-fluid-gpu/src/fluid_gpu_engine.cpp new file mode 100644 index 0000000000000000000000000000000000000000..70e89b3127724756eae82651a5ac2a9007f00e90 --- /dev/null +++ b/inferencer-fluid-gpu/src/fluid_gpu_engine.cpp @@ -0,0 +1,59 @@ +// Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "inferencer-fluid-gpu/include/fluid_gpu_engine.h" +#include "predictor/framework/factory.h" + +DEFINE_int32(gpuid, 0, "GPU device id to use"); + +namespace baidu { +namespace paddle_serving { +namespace fluid_gpu { + +REGIST_FACTORY_OBJECT_IMPL_WITH_NAME( + ::baidu::paddle_serving::predictor::FluidInferEngine, + ::baidu::paddle_serving::predictor::InferEngine, + "FLUID_GPU_ANALYSIS"); + +REGIST_FACTORY_OBJECT_IMPL_WITH_NAME( + ::baidu::paddle_serving::predictor::FluidInferEngine< + FluidGpuAnalysisDirCore>, + ::baidu::paddle_serving::predictor::InferEngine, + "FLUID_GPU_ANALYSIS_DIR"); + +REGIST_FACTORY_OBJECT_IMPL_WITH_NAME( + ::baidu::paddle_serving::predictor::FluidInferEngine< + FluidGpuAnalysisDirWithSigmoidCore>, + ::baidu::paddle_serving::predictor::InferEngine, + "FLUID_GPU_ANALYSIS_DIR_SIGMOID"); + +REGIST_FACTORY_OBJECT_IMPL_WITH_NAME( + ::baidu::paddle_serving::predictor::FluidInferEngine, + ::baidu::paddle_serving::predictor::InferEngine, + "FLUID_GPU_NATIVE"); + +REGIST_FACTORY_OBJECT_IMPL_WITH_NAME( + ::baidu::paddle_serving::predictor::FluidInferEngine, + ::baidu::paddle_serving::predictor::InferEngine, + "FLUID_GPU_NATIVE_DIR"); + +REGIST_FACTORY_OBJECT_IMPL_WITH_NAME( + ::baidu::paddle_serving::predictor::FluidInferEngine< + FluidGpuNativeDirWithSigmoidCore>, + ::baidu::paddle_serving::predictor::InferEngine, + "FLUID_GPU_NATIVE_DIR_SIGMOID"); + +} // namespace fluid_gpu +} // namespace paddle_serving +} // namespace baidu diff --git a/predictor/framework/infer.h b/predictor/framework/infer.h index 3754a7ea828b5fcbecf7aa3c9436488174fef3da..abf0f78883ef7d666a000286aa691b7f7fcce4b2 100644 --- a/predictor/framework/infer.h +++ b/predictor/framework/infer.h @@ -461,7 +461,7 @@ class CloneDBReloadableInferEngine }; template -class FluidInferEngine : public DBReloadableInferEngine { +class FluidInferEngine : public CloneDBReloadableInferEngine { public: FluidInferEngine() {} ~FluidInferEngine() {} diff --git a/predictor/src/pdserving.cpp b/predictor/src/pdserving.cpp index b24b8795e27b8f13db39a30cd2e7ba9d9e2c4633..c9c7be3f9656f2b9ca7a22cd948c64f23ebc4988 100644 --- a/predictor/src/pdserving.cpp +++ b/predictor/src/pdserving.cpp @@ -69,7 +69,30 @@ DEFINE_bool(V, false, "print version, bool"); DEFINE_bool(g, false, "user defined gflag path"); DECLARE_string(flagfile); -void pthread_worker_start_fn() { Resource::instance().thread_initialize(); } +namespace bthread { +extern pthread_mutex_t g_task_control_mutex; +} +pthread_mutex_t g_worker_start_fn_mutex = PTHREAD_MUTEX_INITIALIZER; + +void pthread_worker_start_fn() { + while (pthread_mutex_lock(&g_worker_start_fn_mutex) != 0) { + } + + // Try to avoid deadlock in bthread + int lock_status = pthread_mutex_trylock(&bthread::g_task_control_mutex); + if (lock_status == EBUSY || lock_status == EAGAIN) { + pthread_mutex_unlock(&bthread::g_task_control_mutex); + } + Resource::instance().thread_initialize(); + + // Try to avoid deadlock in bthread + if (lock_status == EBUSY || lock_status == EAGAIN) { + while (pthread_mutex_lock(&bthread::g_task_control_mutex) != 0) { + } + } + + pthread_mutex_unlock(&g_worker_start_fn_mutex); +} static void g_change_server_port() { InferServiceConf conf; @@ -111,7 +134,7 @@ int main(int argc, char** argv) { g_change_server_port(); - // initialize logger instance +// initialize logger instance #ifdef BCLOUD logging::LoggingSettings settings; settings.logging_dest = logging::LOG_TO_FILE; @@ -183,6 +206,8 @@ int main(int argc, char** argv) { } LOG(INFO) << "Succ call pthread worker start function"; + FLAGS_logtostderr = false; + if (ServerManager::instance().start_and_wait() != 0) { LOG(ERROR) << "Failed start server and wait!"; return -1;