diff --git a/README.md b/README.md index a0d46d5c1153bb90f314b572ca8e7e82946d70ff..d15fe64bfd5a21ed379a3b63fc76b2e254a05ff4 100644 --- a/README.md +++ b/README.md @@ -47,9 +47,10 @@ nvidia-docker exec -it test bash ```shell pip install paddle-serving-client==0.4.0 pip install paddle-serving-server==0.4.0 # CPU +pip install paddle-serving-app==0.2.0 pip install paddle-serving-server-gpu==0.4.0.post9 # GPU with CUDA9.0 pip install paddle-serving-server-gpu==0.4.0.post10 # GPU with CUDA10.0 -pip install paddle-serving-server-gpu==0.4.0.trt # GPU with CUDA10.1+TensorRT +pip install paddle-serving-server-gpu==0.4.0.100 # GPU with CUDA10.1+TensorRT ``` You may need to use a domestic mirror source (in China, you can use the Tsinghua mirror source, add `-i https://pypi.tuna.tsinghua.edu.cn/simple` to pip command) to speed up the download. diff --git a/README_CN.md b/README_CN.md index 571b7b00c1252093887a1b5562e03437f51837c4..4e43ee56489d3b65e0174222f1de306bcb1ad4f4 100644 --- a/README_CN.md +++ b/README_CN.md @@ -49,9 +49,10 @@ nvidia-docker exec -it test bash ```shell pip install paddle-serving-client==0.4.0 pip install paddle-serving-server==0.4.0 # CPU +pip install paddle-serving-app==0.2.0 pip install paddle-serving-server-gpu==0.4.0.post9 # GPU with CUDA9.0 pip install paddle-serving-server-gpu==0.4.0.post10 # GPU with CUDA10.0 -pip install paddle-serving-server-gpu==0.4.0.trt # GPU with CUDA10.1+TensorRT +pip install paddle-serving-server-gpu==0.4.0.100 # GPU with CUDA10.1+TensorRT ``` 您可能需要使用国内镜像源（例如清华源, 在pip命令中添加`-i https://pypi.tuna.tsinghua.edu.cn/simple`）来加速下载。 diff --git a/cmake/external/boost.cmake b/cmake/external/boost.cmake index 12412a51a0fd1aaa9702bd4547fb935d94012ada..117b8727f68b90c60ece896d5890d41ba04aac8e 100644 --- a/cmake/external/boost.cmake +++ b/cmake/external/boost.cmake @@ -22,8 +22,8 @@ set(BOOST_PROJECT "extern_boost") # version of boost, say, 1.66.0, doesn't build on CentOS 6. We # checked that the devtools package of CentOS 6 installs boost 1.41.0. # So we use 1.41.0 here. -set(BOOST_VER "1.41.0") -set(BOOST_TAR "boost_1_41_0" CACHE STRING "" FORCE) +set(BOOST_VER "1.74.0") +set(BOOST_TAR "boost_1_74_0" CACHE STRING "" FORCE) set(BOOST_URL "http://paddlepaddledeps.cdn.bcebos.com/${BOOST_TAR}.tar.gz" CACHE STRING "" FORCE) MESSAGE(STATUS "BOOST_TAR: ${BOOST_TAR}, BOOST_URL: ${BOOST_URL}") diff --git a/cmake/external/brpc.cmake b/cmake/external/brpc.cmake index 39412f6950b7d4fe71f294079b69707b202f0876..42eae8d4512c013e5457c2aceaa93e6308a87b8e 100644 --- a/cmake/external/brpc.cmake +++ b/cmake/external/brpc.cmake @@ -13,6 +13,9 @@ # limitations under the License. INCLUDE(ExternalProject) +set(BRPC_CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-narrowing") +set(BRPC_CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-narrowing") +set(BRPC_CMAKE_CPP_FLAGS "${CMAKE_CPP_FLAGS} -Wno-narrowing") find_package(OpenSSL REQUIRED) @@ -41,13 +44,14 @@ ExternalProject_Add( ${EXTERNAL_PROJECT_LOG_ARGS} # TODO(gongwb): change to de newst repo when they changed. GIT_REPOSITORY "https://github.com/wangjiawei04/brpc" - GIT_TAG "6d79e0b17f25107c35b705ea58d888083f59ff47" + GIT_TAG "serving-0.4.1" PREFIX ${BRPC_SOURCES_DIR} UPDATE_COMMAND "" CMAKE_ARGS -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} - -DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS} - -DCMAKE_C_FLAGS=${CMAKE_C_FLAGS} + -DCMAKE_CXX_FLAGS=${BRPC_CMAKE_CXX_FLAGS} + -DCMAKE_C_FLAGS=${BRPC_CMAKE_C_FLAGS} + -DCMAKE_CPP_FLAGS=${BRPC_CMAKE_CPP_FLAGS} -DCMAKE_INSTALL_PREFIX=${BRPC_INSTALL_DIR} -DCMAKE_INSTALL_LIBDIR=${BRPC_INSTALL_DIR}/lib -DCMAKE_POSITION_INDEPENDENT_CODE=ON diff --git a/cmake/paddlepaddle.cmake b/cmake/paddlepaddle.cmake index 4b7d3ed1f620bfcd2e1e214c49c57ee3848129e7..ad95b3ef6db215fddf165d0718d46037749af31f 100644 --- a/cmake/paddlepaddle.cmake +++ b/cmake/paddlepaddle.cmake @@ -31,11 +31,11 @@ message( "WITH_GPU = ${WITH_GPU}") # Paddle Version should be one of: # latest: latest develop build # version number like 1.5.2 -SET(PADDLE_VERSION "1.8.4") +SET(PADDLE_VERSION "2.0.0-rc1") if (WITH_GPU) if (WITH_TRT) - SET(PADDLE_LIB_VERSION "${PADDLE_VERSION}-gpu-cuda10.1-cudnn7.6-avx-mkl-trt6") + SET(PADDLE_LIB_VERSION "${PADDLE_VERSION}-gpu-cuda10.1-cudnn7-avx-mkl-trt6") else() SET(PADDLE_LIB_VERSION "${PADDLE_VERSION}-gpu-cuda10-cudnn7-avx-mkl") endif() @@ -51,7 +51,7 @@ else() endif() endif() -SET(PADDLE_LIB_PATH "http://paddle-inference-lib.bj.bcebos.com/${PADDLE_LIB_VERSION}/fluid_inference.tgz") +SET(PADDLE_LIB_PATH "http://paddle-inference-lib.bj.bcebos.com/${PADDLE_LIB_VERSION}/paddle_inference.tgz") MESSAGE(STATUS "PADDLE_LIB_PATH=${PADDLE_LIB_PATH}") if (WITH_GPU OR WITH_MKLML) if (WITH_TRT) diff --git a/core/configure/CMakeLists.txt b/core/configure/CMakeLists.txt index 9d9487dc9e2513388b70d03e5ac1d875079d95f4..8476192dd33c8fdf2583c3c5fc48b8d3e0ba0b9e 100644 --- a/core/configure/CMakeLists.txt +++ b/core/configure/CMakeLists.txt @@ -14,10 +14,6 @@ list(APPEND configure_srcs ${CMAKE_CURRENT_LIST_DIR}/src/configure_parser.cpp) add_library(configure ${configure_srcs}) add_dependencies(configure brpc) -add_executable(test_configure - ${CMAKE_CURRENT_LIST_DIR}/tests/test_configure.cpp) -target_link_libraries(test_configure configure protobuf) - install(TARGETS configure ARCHIVE DESTINATION ${PADDLE_SERVING_INSTALL_DIR}/lib ) @@ -45,19 +41,19 @@ add_custom_target(sdk_configure_py_proto_init ALL COMMAND ${CMAKE_COMMAND} -E to add_dependencies(sdk_configure_py_proto sdk_configure_py_proto_init) add_custom_command(TARGET sdk_configure_py_proto POST_BUILD COMMAND ${CMAKE_COMMAND} -E make_directory ${PADDLE_SERVING_BINARY_DIR}/python/paddle_serving_client/proto - COMMAND cp *.py ${PADDLE_SERVING_BINARY_DIR}/python/paddle_serving_client/proto + COMMAND cp -f *.py ${PADDLE_SERVING_BINARY_DIR}/python/paddle_serving_client/proto COMMENT "Copy generated python proto into directory paddle_serving_client/proto." WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) add_custom_command(TARGET general_model_config_py_proto POST_BUILD COMMAND ${CMAKE_COMMAND} -E make_directory ${PADDLE_SERVING_BINARY_DIR}/python/paddle_serving_client/proto - COMMAND cp *.py ${PADDLE_SERVING_BINARY_DIR}/python/paddle_serving_client/proto + COMMAND cp -f *.py ${PADDLE_SERVING_BINARY_DIR}/python/paddle_serving_client/proto COMMENT "Copy generated general_model_config proto file into directory paddle_serving_client/proto." WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) add_custom_command(TARGET multi_lang_general_model_service_py_proto POST_BUILD COMMAND ${CMAKE_COMMAND} -E make_directory ${PADDLE_SERVING_BINARY_DIR}/python/paddle_serving_client/proto - COMMAND cp *.py ${PADDLE_SERVING_BINARY_DIR}/python/paddle_serving_client/proto + COMMAND cp -f *.py ${PADDLE_SERVING_BINARY_DIR}/python/paddle_serving_client/proto COMMENT "Copy generated multi_lang_general_model_service proto file into directory paddle_serving_client/proto." WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) endif() @@ -65,7 +61,7 @@ endif() if (APP) add_custom_command(TARGET general_model_config_py_proto POST_BUILD COMMAND ${CMAKE_COMMAND} -E make_directory ${PADDLE_SERVING_BINARY_DIR}/python/paddle_serving_app/proto - COMMAND cp *.py ${PADDLE_SERVING_BINARY_DIR}/python/paddle_serving_app/proto + COMMAND cp -f *.py ${PADDLE_SERVING_BINARY_DIR}/python/paddle_serving_app/proto COMMENT "Copy generated general_model_config proto file into directory paddle_serving_app/proto." WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) endif() @@ -77,26 +73,26 @@ add_dependencies(server_config_py_proto server_config_py_proto_init) if (NOT WITH_GPU) add_custom_command(TARGET server_config_py_proto POST_BUILD COMMAND ${CMAKE_COMMAND} -E make_directory ${PADDLE_SERVING_BINARY_DIR}/python/paddle_serving_server/proto - COMMAND cp *.py ${PADDLE_SERVING_BINARY_DIR}/python/paddle_serving_server/proto + COMMAND cp -f *.py ${PADDLE_SERVING_BINARY_DIR}/python/paddle_serving_server/proto COMMENT "Copy generated python proto into directory paddle_serving_server/proto." WORKING_DIRECTORY ${CMAKE_CURRENT_BINRARY_DIR}) add_custom_command(TARGET general_model_config_py_proto POST_BUILD COMMAND ${CMAKE_COMMAND} -E make_directory ${PADDLE_SERVING_BINARY_DIR}/python/paddle_serving_server/proto - COMMAND cp *.py ${PADDLE_SERVING_BINARY_DIR}/python/paddle_serving_server/proto + COMMAND cp -f *.py ${PADDLE_SERVING_BINARY_DIR}/python/paddle_serving_server/proto COMMENT "Copy generated general_model_config proto file into directory paddle_serving_server/proto." WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) add_custom_command(TARGET multi_lang_general_model_service_py_proto POST_BUILD COMMAND ${CMAKE_COMMAND} -E make_directory ${PADDLE_SERVING_BINARY_DIR}/python/paddle_serving_server/proto - COMMAND cp *.py ${PADDLE_SERVING_BINARY_DIR}/python/paddle_serving_server/proto + COMMAND cp -f *.py ${PADDLE_SERVING_BINARY_DIR}/python/paddle_serving_server/proto COMMENT "Copy generated multi_lang_general_model_service proto file into directory paddle_serving_server/proto." WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) else() add_custom_command(TARGET server_config_py_proto POST_BUILD COMMAND ${CMAKE_COMMAND} -E make_directory ${PADDLE_SERVING_BINARY_DIR}/python/paddle_serving_server_gpu/proto - COMMAND cp *.py + COMMAND cp -f *.py ${PADDLE_SERVING_BINARY_DIR}/python/paddle_serving_server_gpu/proto COMMENT "Copy generated python proto into directory paddle_serving_server_gpu/proto." @@ -105,7 +101,7 @@ add_custom_command(TARGET server_config_py_proto POST_BUILD add_custom_command(TARGET general_model_config_py_proto POST_BUILD COMMAND ${CMAKE_COMMAND} -E make_directory ${PADDLE_SERVING_BINARY_DIR}/python/paddle_serving_server_gpu/proto - COMMAND cp *.py + COMMAND cp -f *.py ${PADDLE_SERVING_BINARY_DIR}/python/paddle_serving_server_gpu/proto COMMENT "Copy generated general_model_config proto file into directory paddle_serving_server_gpu/proto." @@ -113,7 +109,7 @@ add_custom_command(TARGET general_model_config_py_proto POST_BUILD add_custom_command(TARGET multi_lang_general_model_service_py_proto POST_BUILD COMMAND ${CMAKE_COMMAND} -E make_directory ${PADDLE_SERVING_BINARY_DIR}/python/paddle_serving_server_gpu/proto - COMMAND cp *.py ${PADDLE_SERVING_BINARY_DIR}/python/paddle_serving_server_gpu/proto + COMMAND cp -f *.py ${PADDLE_SERVING_BINARY_DIR}/python/paddle_serving_server_gpu/proto COMMENT "Copy generated multi_lang_general_model_service proto file into directory paddle_serving_server_gpu/proto." WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) endif() diff --git a/core/general-server/op/general_dist_kv_infer_op.cpp b/core/general-server/op/general_dist_kv_infer_op.cpp index 6809907226511f7de576f1e2bbdc21b7ac401422..f1662c2ea4d17cc72b09fc9fd3cb849aef780b1b 100644 --- a/core/general-server/op/general_dist_kv_infer_op.cpp +++ b/core/general-server/op/general_dist_kv_infer_op.cpp @@ -38,145 +38,7 @@ using baidu::paddle_serving::predictor::general_model::FetchInst; using baidu::paddle_serving::predictor::InferManager; using baidu::paddle_serving::predictor::PaddleGeneralModelConfig; -int GeneralDistKVInferOp::inference() { - VLOG(2) << "Going to run inference"; - const std::vector pre_node_names = pre_names(); - if (pre_node_names.size() != 1) { - LOG(ERROR) << "This op(" << op_name() - << ") can only have one predecessor op, but received " - << pre_node_names.size(); - return -1; - } - const std::string pre_name = pre_node_names[0]; - - const GeneralBlob *input_blob = get_depend_argument(pre_name); - uint64_t log_id = input_blob->GetLogId(); - VLOG(2) << "(logid=" << log_id << ") Get precedent op name: " << pre_name; - GeneralBlob *output_blob = mutable_data(); - - if (!input_blob) { - LOG(ERROR) << "(logid=" << log_id - << ") Failed mutable depended argument, op:" << pre_name; - return -1; - } - - const TensorVector *in = &input_blob->tensor_vector; - TensorVector *out = &output_blob->tensor_vector; - int batch_size = input_blob->GetBatchSize(); - VLOG(2) << "(logid=" << log_id << ") input batch size: " << batch_size; - std::vector keys; - std::vector values; - int sparse_count = 0; - int dense_count = 0; - std::vector> dataptr_size_pairs; - size_t key_len = 0; - for (size_t i = 0; i < in->size(); ++i) { - if (in->at(i).dtype != paddle::PaddleDType::INT64) { - ++dense_count; - continue; - } - ++sparse_count; - size_t elem_num = 1; - for (size_t s = 0; s < in->at(i).shape.size(); ++s) { - elem_num *= in->at(i).shape[s]; - } - key_len += elem_num; - int64_t *data_ptr = static_cast(in->at(i).data.data()); - dataptr_size_pairs.push_back(std::make_pair(data_ptr, elem_num)); - } - keys.resize(key_len); - int key_idx = 0; - for (size_t i = 0; i < dataptr_size_pairs.size(); ++i) { - std::copy(dataptr_size_pairs[i].first, - dataptr_size_pairs[i].first + dataptr_size_pairs[i].second, - keys.begin() + key_idx); - key_idx += dataptr_size_pairs[i].second; - } - Timer timeline; - int64_t cube_start = timeline.TimeStampUS(); - timeline.Start(); - rec::mcube::CubeAPI *cube = rec::mcube::CubeAPI::instance(); - std::vector table_names = cube->get_table_names(); - if (table_names.size() == 0) { - LOG(ERROR) << "(logid=" << log_id - << ") cube init error or cube config not given."; - return -1; - } - int ret = cube->seek(table_names[0], keys, &values); - int64_t cube_end = timeline.TimeStampUS(); - if (values.size() != keys.size() || values[0].buff.size() == 0) { - LOG(ERROR) << "(logid=" << log_id << ") cube value return null"; - } - size_t EMBEDDING_SIZE = values[0].buff.size() / sizeof(float); - TensorVector sparse_out; - sparse_out.resize(sparse_count); - TensorVector dense_out; - dense_out.resize(dense_count); - int cube_val_idx = 0; - int sparse_idx = 0; - int dense_idx = 0; - std::unordered_map in_out_map; - baidu::paddle_serving::predictor::Resource &resource = - baidu::paddle_serving::predictor::Resource::instance(); - std::shared_ptr model_config = - resource.get_general_model_config(); - for (size_t i = 0; i < in->size(); ++i) { - if (in->at(i).dtype != paddle::PaddleDType::INT64) { - dense_out[dense_idx] = in->at(i); - ++dense_idx; - continue; - } - - sparse_out[sparse_idx].lod.resize(in->at(i).lod.size()); - for (size_t x = 0; x < sparse_out[sparse_idx].lod.size(); ++x) { - sparse_out[sparse_idx].lod[x].resize(in->at(i).lod[x].size()); - std::copy(in->at(i).lod[x].begin(), - in->at(i).lod[x].end(), - sparse_out[sparse_idx].lod[x].begin()); - } - sparse_out[sparse_idx].dtype = paddle::PaddleDType::FLOAT32; - sparse_out[sparse_idx].shape.push_back( - sparse_out[sparse_idx].lod[0].back()); - sparse_out[sparse_idx].shape.push_back(EMBEDDING_SIZE); - sparse_out[sparse_idx].name = model_config->_feed_name[i]; - sparse_out[sparse_idx].data.Resize(sparse_out[sparse_idx].lod[0].back() * - EMBEDDING_SIZE * sizeof(float)); - float *dst_ptr = static_cast(sparse_out[sparse_idx].data.data()); - for (int x = 0; x < sparse_out[sparse_idx].lod[0].back(); ++x) { - float *data_ptr = dst_ptr + x * EMBEDDING_SIZE; - memcpy(data_ptr, - values[cube_val_idx].buff.data(), - values[cube_val_idx].buff.size()); - cube_val_idx++; - } - ++sparse_idx; - } - TensorVector infer_in; - infer_in.insert(infer_in.end(), dense_out.begin(), dense_out.end()); - infer_in.insert(infer_in.end(), sparse_out.begin(), sparse_out.end()); - - output_blob->SetBatchSize(batch_size); - output_blob->SetLogId(log_id); - - VLOG(2) << "(logid=" << log_id << ") infer batch size: " << batch_size; - - int64_t start = timeline.TimeStampUS(); - - if (InferManager::instance().infer( - engine_name().c_str(), &infer_in, out, batch_size)) { - LOG(ERROR) << "(logid=" << log_id - << ") Failed do infer in fluid model: " << engine_name(); - return -1; - } - - int64_t end = timeline.TimeStampUS(); - CopyBlobInfo(input_blob, output_blob); - AddBlobInfo(output_blob, cube_start); - AddBlobInfo(output_blob, cube_end); - AddBlobInfo(output_blob, start); - AddBlobInfo(output_blob, end); - return 0; -} +int GeneralDistKVInferOp::inference() { return 0; } DEFINE_OP(GeneralDistKVInferOp); } // namespace serving diff --git a/core/general-server/op/general_dist_kv_quant_infer_op.cpp b/core/general-server/op/general_dist_kv_quant_infer_op.cpp index 93ce76f3d3399ac62435352d2271154ab7f84235..7d347702768c13b997ea97291a8f9fde0ce042a2 100644 --- a/core/general-server/op/general_dist_kv_quant_infer_op.cpp +++ b/core/general-server/op/general_dist_kv_quant_infer_op.cpp @@ -188,21 +188,6 @@ int GeneralDistKVQuantInferOp::inference() { VLOG(2) << "(logid=" << log_id << ") infer batch size: " << batch_size; - Timer timeline; - int64_t start = timeline.TimeStampUS(); - timeline.Start(); - - if (InferManager::instance().infer( - engine_name().c_str(), &infer_in, out, batch_size)) { - LOG(ERROR) << "(logid=" << log_id - << ") Failed do infer in fluid model: " << engine_name(); - return -1; - } - - int64_t end = timeline.TimeStampUS(); - CopyBlobInfo(input_blob, output_blob); - AddBlobInfo(output_blob, start); - AddBlobInfo(output_blob, end); return 0; } DEFINE_OP(GeneralDistKVQuantInferOp); diff --git a/core/general-server/op/general_infer_op.cpp b/core/general-server/op/general_infer_op.cpp index b9478542c71e04b0f3f80b277da7d8d41f636d3d..5b9df8064d6c7f50b269fc67b157494ac53e22e2 100644 --- a/core/general-server/op/general_infer_op.cpp +++ b/core/general-server/op/general_infer_op.cpp @@ -44,45 +44,9 @@ int GeneralInferOp::inference() { << pre_node_names.size(); return -1; } - const std::string pre_name = pre_node_names[0]; - - const GeneralBlob *input_blob = get_depend_argument(pre_name); - uint64_t log_id = input_blob->GetLogId(); - VLOG(2) << "(logid=" << log_id << ") Get precedent op name: " << pre_name; - GeneralBlob *output_blob = mutable_data(); - output_blob->SetLogId(log_id); - - if (!input_blob) { - LOG(ERROR) << "(logid=" << log_id - << ") Failed mutable depended argument, op:" << pre_name; + if (InferManager::instance().infer(engine_name().c_str())) { return -1; } - - const TensorVector *in = &input_blob->tensor_vector; - TensorVector *out = &output_blob->tensor_vector; - - int batch_size = input_blob->_batch_size; - VLOG(2) << "(logid=" << log_id << ") input batch size: " << batch_size; - - output_blob->_batch_size = batch_size; - - VLOG(2) << "(logid=" << log_id << ") infer batch size: " << batch_size; - - Timer timeline; - int64_t start = timeline.TimeStampUS(); - timeline.Start(); - - if (InferManager::instance().infer( - engine_name().c_str(), in, out, batch_size)) { - LOG(ERROR) << "(logid=" << log_id - << ") Failed do infer in fluid model: " << engine_name().c_str(); - return -1; - } - - int64_t end = timeline.TimeStampUS(); - CopyBlobInfo(input_blob, output_blob); - AddBlobInfo(output_blob, start); - AddBlobInfo(output_blob, end); return 0; } DEFINE_OP(GeneralInferOp); diff --git a/core/general-server/op/general_reader_op.cpp b/core/general-server/op/general_reader_op.cpp index 0329fac6b9bb6eda59f3f6f1589cd00c3eec0fd9..24259e24d7f00b52eb35170bc9b887ecf301f157 100644 --- a/core/general-server/op/general_reader_op.cpp +++ b/core/general-server/op/general_reader_op.cpp @@ -20,6 +20,7 @@ #include "core/general-server/op/general_infer_helper.h" #include "core/predictor/framework/infer.h" #include "core/predictor/framework/memory.h" +#include "core/predictor/framework/resource.h" #include "core/util/include/timer.h" namespace baidu { @@ -32,6 +33,7 @@ using baidu::paddle_serving::predictor::general_model::Tensor; using baidu::paddle_serving::predictor::general_model::Request; using baidu::paddle_serving::predictor::general_model::FeedInst; using baidu::paddle_serving::predictor::PaddleGeneralModelConfig; +using baidu::paddle_serving::predictor::InferManager; int conf_check(const Request *req, const std::shared_ptr &model_config) { @@ -71,75 +73,34 @@ int conf_check(const Request *req, int GeneralReaderOp::inference() { // reade request from client + // TODO: only support one engine here + std::string engine_name = "general_infer_0"; const Request *req = dynamic_cast(get_request_message()); uint64_t log_id = req->log_id(); int input_var_num = 0; std::vector elem_type; std::vector elem_size; std::vector capacity; - - GeneralBlob *res = mutable_data(); - TensorVector *out = &res->tensor_vector; - - res->SetLogId(log_id); - - if (!res) { - LOG(ERROR) << "(logid=" << log_id - << ") Failed get op tls reader object output"; - } - - Timer timeline; - int64_t start = timeline.TimeStampUS(); int var_num = req->insts(0).tensor_array_size(); - VLOG(2) << "(logid=" << log_id << ") var num: " << var_num; - - VLOG(2) << "(logid=" << log_id - << ") start to call load general model_conf op"; - baidu::paddle_serving::predictor::Resource &resource = baidu::paddle_serving::predictor::Resource::instance(); - - VLOG(2) << "(logid=" << log_id << ") get resource pointer done."; std::shared_ptr model_config = resource.get_general_model_config(); - - VLOG(2) << "(logid=" << log_id << ") print general model config done."; - - // TODO(guru4elephant): how to do conditional check? - /* - int ret = conf_check(req, model_config); - if (ret != 0) { - LOG(ERROR) << "model conf of server:"; - resource.print_general_model_config(model_config); - return 0; - } - */ - // package tensor - elem_type.resize(var_num); elem_size.resize(var_num); capacity.resize(var_num); - // prepare basic information for input for (int i = 0; i < var_num; ++i) { - paddle::PaddleTensor lod_tensor; - elem_type[i] = req->insts(0).tensor_array(i).elem_type(); - VLOG(2) << "var[" << i << "] has elem type: " << elem_type[i]; - if (elem_type[i] == 0) { // int64 - elem_size[i] = sizeof(int64_t); - lod_tensor.dtype = paddle::PaddleDType::INT64; - } else if (elem_type[i] == 1) { - elem_size[i] = sizeof(float); - lod_tensor.dtype = paddle::PaddleDType::FLOAT32; - } else if (elem_type[i] == 2) { - elem_size[i] = sizeof(int32_t); - lod_tensor.dtype = paddle::PaddleDType::INT32; - } - // implement lod tensor here + std::string tensor_name = model_config->_feed_name[i]; + VLOG(2) << "(logid=" << log_id << ") get tensor name: " << tensor_name; + auto lod_tensor = InferManager::instance().GetInputHandle( + engine_name.c_str(), tensor_name.c_str()); + std::vector> lod; + std::vector shape; + // get lod info here if (req->insts(0).tensor_array(i).lod_size() > 0) { - VLOG(2) << "(logid=" << log_id << ") var[" << i << "] is lod_tensor"; - lod_tensor.lod.resize(1); + lod.resize(1); for (int k = 0; k < req->insts(0).tensor_array(i).lod_size(); ++k) { - lod_tensor.lod[0].push_back(req->insts(0).tensor_array(i).lod(k)); + lod[0].push_back(req->insts(0).tensor_array(i).lod(k)); } capacity[i] = 1; for (int k = 0; k < req->insts(0).tensor_array(i).shape_size(); ++k) { @@ -147,7 +108,7 @@ int GeneralReaderOp::inference() { VLOG(2) << "(logid=" << log_id << ") shape for var[" << i << "]: " << dim; capacity[i] *= dim; - lod_tensor.shape.push_back(dim); + shape.push_back(dim); } VLOG(2) << "(logid=" << log_id << ") var[" << i << "] is tensor, capacity: " << capacity[i]; @@ -158,92 +119,41 @@ int GeneralReaderOp::inference() { VLOG(2) << "(logid=" << log_id << ") shape for var[" << i << "]: " << dim; capacity[i] *= dim; - lod_tensor.shape.push_back(dim); + shape.push_back(dim); } VLOG(2) << "(logid=" << log_id << ") var[" << i << "] is tensor, capacity: " << capacity[i]; } - lod_tensor.name = model_config->_feed_name[i]; - out->push_back(lod_tensor); - } - // specify the memory needed for output tensor_vector - for (int i = 0; i < var_num; ++i) { - if (out->at(i).lod.size() == 1) { - int tensor_size = 0; - const Tensor &tensor = req->insts(0).tensor_array(i); - int data_len = 0; - if (tensor.int64_data_size() > 0) { - data_len = tensor.int64_data_size(); - } else if (tensor.float_data_size() > 0) { - data_len = tensor.float_data_size(); - } else if (tensor.int_data_size() > 0) { - data_len = tensor.int_data_size(); - } - VLOG(2) << "(logid=" << log_id << ") tensor size for var[" << i - << "]: " << data_len; - tensor_size += data_len; - - int cur_len = out->at(i).lod[0].back(); - VLOG(2) << "(logid=" << log_id << ") current len: " << cur_len; - - int sample_len = 0; - if (tensor.shape_size() == 1) { - sample_len = data_len; - } else { - sample_len = tensor.shape(0); - } - VLOG(2) << "(logid=" << log_id << ") new len: " << cur_len + sample_len; - out->at(i).data.Resize(tensor_size * elem_size[i]); - VLOG(2) << "(logid=" << log_id << ") var[" << i - << "] is lod_tensor and len=" << out->at(i).lod[0].back(); - } else { - out->at(i).data.Resize(capacity[i] * elem_size[i]); - VLOG(2) << "(logid=" << log_id << ") var[" << i - << "] is tensor and capacity=" << capacity[i]; - } - } - - // fill the data into output general_blob - for (int i = 0; i < var_num; ++i) { - if (elem_type[i] == 0) { - int64_t *dst_ptr = static_cast(out->at(i).data.data()); - VLOG(2) << "(logid=" << log_id << ") first element data in var[" << i - << "] is " << req->insts(0).tensor_array(i).int64_data(0); - int offset = 0; + lod_tensor->SetLoD(lod); + lod_tensor->Reshape(shape); + // insert data here + if (req->insts(0).tensor_array(i).elem_type() == 0) { + // TODO: Copy twice here, can optimize int elem_num = req->insts(0).tensor_array(i).int64_data_size(); + std::vector data(elem_num); + int64_t *dst_ptr = data.data(); for (int k = 0; k < elem_num; ++k) { - dst_ptr[offset + k] = req->insts(0).tensor_array(i).int64_data(k); + dst_ptr[k] = req->insts(0).tensor_array(i).int64_data(k); } - } else if (elem_type[i] == 1) { - float *dst_ptr = static_cast(out->at(i).data.data()); - VLOG(2) << "(logid=" << log_id << ") first element data in var[" << i - << "] is " << req->insts(0).tensor_array(i).float_data(0); - int offset = 0; + lod_tensor->CopyFromCpu(dst_ptr); + } else if (req->insts(0).tensor_array(i).elem_type() == 1) { int elem_num = req->insts(0).tensor_array(i).float_data_size(); + std::vector data(elem_num); + float *dst_ptr = data.data(); for (int k = 0; k < elem_num; ++k) { - dst_ptr[offset + k] = req->insts(0).tensor_array(i).float_data(k); + dst_ptr[k] = req->insts(0).tensor_array(i).float_data(k); } - } else if (elem_type[i] == 2) { - int32_t *dst_ptr = static_cast(out->at(i).data.data()); - VLOG(2) << "(logid=" << log_id << ") first element data in var[" << i - << "] is " << req->insts(0).tensor_array(i).int_data(0); - int offset = 0; + lod_tensor->CopyFromCpu(dst_ptr); + } else if (req->insts(0).tensor_array(i).elem_type() == 2) { int elem_num = req->insts(0).tensor_array(i).int_data_size(); + std::vector data(elem_num); + int32_t *dst_ptr = data.data(); for (int k = 0; k < elem_num; ++k) { - dst_ptr[offset + k] = req->insts(0).tensor_array(i).int_data(k); + dst_ptr[k] = req->insts(0).tensor_array(i).int_data(k); } + lod_tensor->CopyFromCpu(dst_ptr); } } - - VLOG(2) << "(logid=" << log_id << ") output size: " << out->size(); - timeline.Pause(); - int64_t end = timeline.TimeStampUS(); - res->p_size = 0; - res->_batch_size = 1; - AddBlobInfo(res, start); - AddBlobInfo(res, end); - - VLOG(2) << "(logid=" << log_id << ") read data from client success"; return 0; } DEFINE_OP(GeneralReaderOp); diff --git a/core/general-server/op/general_response_op.cpp b/core/general-server/op/general_response_op.cpp index 5f80510f79f8acf09aed9f7f65e84b9cfaa9a8ed..dbc24c4cb659e116e0d1b07b03c033ad8764e033 100644 --- a/core/general-server/op/general_response_op.cpp +++ b/core/general-server/op/general_response_op.cpp @@ -40,160 +40,60 @@ using baidu::paddle_serving::predictor::InferManager; using baidu::paddle_serving::predictor::PaddleGeneralModelConfig; int GeneralResponseOp::inference() { - const std::vector pre_node_names = pre_names(); - VLOG(2) << "pre node names size: " << pre_node_names.size(); - const GeneralBlob *input_blob; - uint64_t log_id = - get_depend_argument(pre_node_names[0])->GetLogId(); - const Request *req = dynamic_cast(get_request_message()); // response inst with only fetch_var_names Response *res = mutable_data(); - - Timer timeline; - // double response_time = 0.0; - // timeline.Start(); - int64_t start = timeline.TimeStampUS(); - - VLOG(2) << "(logid=" << log_id - << ") start to call load general model_conf op"; baidu::paddle_serving::predictor::Resource &resource = baidu::paddle_serving::predictor::Resource::instance(); - - VLOG(2) << "(logid=" << log_id << ") get resource pointer done."; std::shared_ptr model_config = resource.get_general_model_config(); - - VLOG(2) << "(logid=" << log_id - << ") max body size : " << brpc::fLU64::FLAGS_max_body_size; - - std::vector fetch_index; - fetch_index.resize(req->fetch_var_names_size()); + std::vector capacity(req->fetch_var_names_size(), 1); + std::string engine_name = "general_infer_0"; + ModelOutput *output = res->add_outputs(); + FetchInst *fetch_inst = output->add_insts(); + FetchInst *fetch_p = output->mutable_insts(0); + std::vector outs = + InferManager::instance().GetOutputNames(engine_name.c_str()); for (int i = 0; i < req->fetch_var_names_size(); ++i) { - fetch_index[i] = - model_config->_fetch_alias_name_to_index[req->fetch_var_names(i)]; - } - - for (uint32_t pi = 0; pi < pre_node_names.size(); ++pi) { - const std::string &pre_name = pre_node_names[pi]; - VLOG(2) << "(logid=" << log_id << ") pre names[" << pi << "]: " << pre_name - << " (" << pre_node_names.size() << ")"; - input_blob = get_depend_argument(pre_name); - // fprintf(stderr, "input(%s) blob address %x\n", pre_names.c_str(), - // input_blob); - if (!input_blob) { - LOG(ERROR) << "(logid=" << log_id - << ") Failed mutable depended argument, op: " << pre_name; - return -1; + Tensor *tensor = fetch_inst->add_tensor_array(); + std::string tensor_name = outs[i]; + auto lod_tensor = InferManager::instance().GetOutputHandle( + engine_name.c_str(), tensor_name.c_str()); + std::vector shape = lod_tensor->shape(); + for (int k = 0; k < shape.size(); ++k) { + capacity[i] *= shape[k]; + tensor->add_shape(shape[k]); } - - const TensorVector *in = &input_blob->tensor_vector; - - ModelOutput *output = res->add_outputs(); - // To get the order of model return values - output->set_engine_name(pre_name); - FetchInst *fetch_inst = output->add_insts(); - - for (auto &idx : fetch_index) { - Tensor *tensor = fetch_inst->add_tensor_array(); - if (model_config->_is_lod_fetch[idx]) { - VLOG(2) << "(logid=" << log_id << ") out[" << idx << "] " - << model_config->_fetch_name[idx] << " is lod_tensor"; - for (int k = 0; k < in->at(idx).shape.size(); ++k) { - VLOG(2) << "(logid=" << log_id << ") shape[" << k - << "]: " << in->at(idx).shape[k]; - tensor->add_shape(in->at(idx).shape[k]); - } - } else { - VLOG(2) << "(logid=" << log_id << ") out[" << idx << "] " - << model_config->_fetch_name[idx] << " is tensor"; - for (int k = 0; k < in->at(idx).shape.size(); ++k) { - VLOG(2) << "(logid=" << log_id << ") shape[" << k - << "]: " << in->at(idx).shape[k]; - tensor->add_shape(in->at(idx).shape[k]); - } - } + auto dtype = lod_tensor->type(); + if (dtype == paddle::PaddleDType::INT64) { + std::vector datas(capacity[i]); + int64_t *data_ptr = datas.data(); + lod_tensor->CopyToCpu(data_ptr); + google::protobuf::RepeatedField tmp_data(data_ptr, + data_ptr + capacity[i]); + tensor->mutable_int64_data()->Swap(&tmp_data); + } else if (dtype == paddle::PaddleDType::FLOAT32) { + std::vector datas(capacity[i]); + float *data_ptr = datas.data(); + lod_tensor->CopyToCpu(data_ptr); + google::protobuf::RepeatedField tmp_data(data_ptr, + data_ptr + capacity[i]); + tensor->mutable_float_data()->Swap(&tmp_data); + } else if (dtype == paddle::PaddleDType::INT32) { + std::vector datas(capacity[i]); + int32_t *data_ptr = datas.data(); + lod_tensor->CopyToCpu(data_ptr); + google::protobuf::RepeatedField tmp_data(data_ptr, + data_ptr + capacity[i]); + tensor->mutable_int_data()->Swap(&tmp_data); } - - int var_idx = 0; - for (auto &idx : fetch_index) { - int cap = 1; - for (int j = 0; j < in->at(idx).shape.size(); ++j) { - cap *= in->at(idx).shape[j]; + std::vector> lod = lod_tensor->lod(); + if (lod.size() > 0) { + for (int j = 0; j < lod[0].size(); ++j) { + tensor->add_lod(lod[0][j]); } - - FetchInst *fetch_p = output->mutable_insts(0); - auto dtype = in->at(idx).dtype; - - if (dtype == paddle::PaddleDType::INT64) { - VLOG(2) << "(logid=" << log_id << ") Prepare int64 var [" - << model_config->_fetch_name[idx] << "]."; - int64_t *data_ptr = static_cast(in->at(idx).data.data()); - // from - // https://stackoverflow.com/questions/15499641/copy-a-stdvector-to-a-repeated-field-from-protobuf-with-memcpy - // `Swap` method is faster than `{}` method. - google::protobuf::RepeatedField tmp_data(data_ptr, - data_ptr + cap); - fetch_p->mutable_tensor_array(var_idx)->mutable_int64_data()->Swap( - &tmp_data); - } else if (dtype == paddle::PaddleDType::FLOAT32) { - VLOG(2) << "(logid=" << log_id << ") Prepare float var [" - << model_config->_fetch_name[idx] << "]."; - float *data_ptr = static_cast(in->at(idx).data.data()); - google::protobuf::RepeatedField tmp_data(data_ptr, - data_ptr + cap); - fetch_p->mutable_tensor_array(var_idx)->mutable_float_data()->Swap( - &tmp_data); - } else if (dtype == paddle::PaddleDType::INT32) { - VLOG(2) << "(logid=" << log_id << ")Prepare int32 var [" - << model_config->_fetch_name[idx] << "]."; - int32_t *data_ptr = static_cast(in->at(idx).data.data()); - google::protobuf::RepeatedField tmp_data(data_ptr, - data_ptr + cap); - fetch_p->mutable_tensor_array(var_idx)->mutable_int_data()->Swap( - &tmp_data); - } - - if (model_config->_is_lod_fetch[idx]) { - if (in->at(idx).lod.size() > 0) { - for (int j = 0; j < in->at(idx).lod[0].size(); ++j) { - fetch_p->mutable_tensor_array(var_idx)->add_lod( - in->at(idx).lod[0][j]); - } - } - } - - VLOG(2) << "(logid=" << log_id << ") fetch var [" - << model_config->_fetch_name[idx] << "] ready"; - var_idx++; } } - - if (req->profile_server()) { - int64_t end = timeline.TimeStampUS(); - // TODO(barriery): multi-model profile_time. - // At present, only the response_op is multi-input, so here we get - // the profile_time by hard coding. It needs to be replaced with - // a more elegant way. - for (uint32_t pi = 0; pi < pre_node_names.size(); ++pi) { - input_blob = get_depend_argument(pre_node_names[pi]); - VLOG(2) << "(logid=" << log_id - << ") p size for input blob: " << input_blob->p_size; - int profile_time_idx = -1; - if (pi == 0) { - profile_time_idx = 0; - } else { - profile_time_idx = input_blob->p_size - 2; - } - for (; profile_time_idx < input_blob->p_size; ++profile_time_idx) { - res->add_profile_time(input_blob->time_stamp[profile_time_idx]); - } - } - // TODO(guru4elephant): find more elegant way to do this - res->add_profile_time(start); - res->add_profile_time(end); - } - return 0; } diff --git a/core/predictor/CMakeLists.txt b/core/predictor/CMakeLists.txt index 637c7c15530273bc908ec2f8693a3d66989eebd2..10fcd0b23b2d76a3e693bc29e07f5add663dbcdf 100644 --- a/core/predictor/CMakeLists.txt +++ b/core/predictor/CMakeLists.txt @@ -12,13 +12,12 @@ set_source_files_properties( ${pdserving_srcs} PROPERTIES COMPILE_FLAGS "-Wno-strict-aliasing -Wno-unused-variable -Wno-non-virtual-dtor -Wno-error=non-virtual-dtor -Wno-error=delete-non-virtual-dtor") -add_dependencies(pdserving protobuf boost brpc leveldb pdcodegen configure) +add_dependencies(pdserving protobuf boost brpc leveldb pdcodegen configure extern_paddle paddle_fluid) if (WITH_TRT) add_definitions(-DWITH_TRT) endif() target_link_libraries(pdserving - brpc protobuf boost leveldb configure -lpthread -lcrypto -lm -lrt -lssl -ldl -lz) - + brpc protobuf boost leveldb configure -lpthread -lcrypto -lm -lrt -lssl -ldl -lz paddle_fluid ${paddle_depend_libs}) # install install(TARGETS pdserving RUNTIME DESTINATION ${PADDLE_SERVING_INSTALL_DIR}/bin diff --git a/core/predictor/framework/infer.h b/core/predictor/framework/infer.h index 431bc456326c1714dce48e2f6321bf58f3e021ce..8a6b6707b26474200b57769908e91055b5479a41 100644 --- a/core/predictor/framework/infer.h +++ b/core/predictor/framework/infer.h @@ -20,10 +20,9 @@ #include #include #include "core/predictor/common/inner_common.h" -#include "core/predictor/framework/bsf.h" #include "core/predictor/framework/factory.h" #include "core/predictor/framework/infer_data.h" - +#include "paddle_inference_api.h" // NOLINT namespace baidu { namespace paddle_serving { namespace predictor { @@ -105,9 +104,7 @@ class InferEngine { virtual int thrd_initialize() { return thrd_initialize_impl(); } virtual int thrd_clear() { return thrd_clear_impl(); } virtual int thrd_finalize() { return thrd_finalize_impl(); } - virtual int infer(const void* in, void* out, uint32_t batch_size = -1) { - return infer_impl1(in, out, batch_size); - } + virtual int infer() { return infer_impl(); } virtual int reload() = 0; @@ -120,11 +117,13 @@ class InferEngine { virtual int thrd_finalize_impl() = 0; virtual int thrd_clear_impl() = 0; virtual int proc_finalize_impl() = 0; - virtual int infer_impl1(const void* in, - void* out, - uint32_t batch_size = -1) = 0; - virtual int infer_impl2(const BatchTensor& in, - BatchTensor& out) = 0; // NOLINT + virtual std::vector GetInputNames() = 0; + virtual std::vector GetOutputNames() = 0; + virtual std::unique_ptr GetInputHandle( + const std::string& name) = 0; + virtual std::unique_ptr GetOutputHandle( + const std::string& name) = 0; + virtual int infer_impl() = 0; // end: framework inner call }; @@ -138,8 +137,6 @@ class ReloadableInferEngine : public InferEngine { uint64_t last_revision; }; - typedef im::bsf::Task TaskT; - virtual int load(const InferEngineCreationParams& params) = 0; int proc_initialize_impl(const configure::EngineDesc& conf, bool version) { @@ -201,45 +198,10 @@ class ReloadableInferEngine : public InferEngine { LOG(ERROR) << "Failed proc initialize impl"; return -1; } - - // init bsf framework - if (_infer_thread_num <= 0) { - return 0; - } - - im::bsf::TaskExecutor::instance()->set_thread_init_fn( - boost::bind(&InferEngine::thrd_initialize_impl, this)); - im::bsf::TaskExecutor::instance()->set_thread_reset_fn( - boost::bind(&InferEngine::thrd_clear_impl, this)); - im::bsf::TaskExecutor::instance()->set_thread_callback_fn( - boost::bind(&InferEngine::infer_impl2, this, _1, _2)); - im::bsf::TaskExecutor::instance()->set_batch_size(_infer_batch_size); - im::bsf::TaskExecutor::instance()->set_batch_align( - _infer_batch_align); - if (im::bsf::TaskExecutor::instance()->start(_infer_thread_num) != - 0) { - LOG(ERROR) << "Failed start bsf executor, threads:" << _infer_thread_num; - return -1; - } - - LOG(WARNING) << "Enable batch schedule framework, thread_num:" - << _infer_thread_num << ", batch_size:" << _infer_batch_size - << ", enable_batch_align:" << _infer_batch_align; - return 0; } - int infer(const void* in, void* out, uint32_t batch_size = -1) { - if (_infer_thread_num <= 0) { - return infer_impl1(in, out, batch_size); - } - - im::bsf::TaskManager task_manager; - task_manager.schedule(*(reinterpret_cast(in)), - *(reinterpret_cast(out))); - task_manager.wait(); - return 0; - } + int infer() { return infer_impl(); } int thrd_initialize() { if (_infer_thread_num > 0) { @@ -263,10 +225,6 @@ class ReloadableInferEngine : public InferEngine { return -1; } - if (_infer_thread_num > 0) { - im::bsf::TaskExecutor::instance()->stop(); - } - return 0; } @@ -417,10 +375,6 @@ class DBReloadableInferEngine : public ReloadableInferEngine { virtual int thrd_initialize_impl() { // memory pool to be inited in non-serving-threads - if (MempoolWrapper::instance().thread_initialize() != 0) { - LOG(ERROR) << "Failed thread initialize mempool"; - return -1; - } ModelData* md = new (std::nothrow) ModelData; if (!md || load_data(md, _infer_engine_params) != 0) { @@ -430,17 +384,12 @@ class DBReloadableInferEngine : public ReloadableInferEngine { } THREAD_SETSPECIFIC(_skey, md); - im::bsf::AutoMutex lock(_mutex); _reload_vec.push_back(md); return 0; } int thrd_clear_impl() { // for non-serving-threads - if (MempoolWrapper::instance().thread_clear() != 0) { - LOG(ERROR) << "Failed thread clear mempool"; - return -1; - } return 0; } @@ -538,12 +487,6 @@ class CloneDBReloadableInferEngine } virtual int thrd_initialize_impl() { - // memory pool to be inited in non-serving-threads - if (MempoolWrapper::instance().thread_initialize() != 0) { - LOG(ERROR) << "Failed thread initialize mempool"; - return -1; - } - ModelData* md = new (std::nothrow) ModelData; if (!md || load_data(md, _pd->cores[_pd->current_idx]) != 0) { LOG(ERROR) << "Failed clone thread data, origin_core[" @@ -552,7 +495,6 @@ class CloneDBReloadableInferEngine } THREAD_SETSPECIFIC(DBReloadableInferEngine::_skey, md); - im::bsf::AutoMutex lock(DBReloadableInferEngine::_mutex); DBReloadableInferEngine::_reload_vec.push_back(md); return 0; } @@ -571,8 +513,45 @@ class FluidInferEngine : public CloneDBReloadableInferEngine { public: // NOLINT FluidInferEngine() {} ~FluidInferEngine() {} + std::vector GetInputNames() { + FluidFamilyCore* core = + DBReloadableInferEngine::get_core(); + if (!core || !core->get()) { + LOG(ERROR) << "Failed get fluid core in GetInputHandle()"; + } + return core->GetInputNames(); + } - int infer_impl1(const void* in, void* out, uint32_t batch_size = -1) { + std::vector GetOutputNames() { + FluidFamilyCore* core = + DBReloadableInferEngine::get_core(); + if (!core || !core->get()) { + LOG(ERROR) << "Failed get fluid core in GetInputHandle()"; + } + return core->GetOutputNames(); + } + + std::unique_ptr GetInputHandle( + const std::string& name) { + FluidFamilyCore* core = + DBReloadableInferEngine::get_core(); + if (!core || !core->get()) { + LOG(ERROR) << "Failed get fluid core in GetInputHandle()"; + } + return core->GetInputHandle(name); + } + + std::unique_ptr GetOutputHandle( + const std::string& name) { + FluidFamilyCore* core = + DBReloadableInferEngine::get_core(); + if (!core || !core->get()) { + LOG(ERROR) << "Failed get fluid core in GetOutputHandle()"; + } + return core->GetOutputHandle(name); + } + + int infer_impl() { FluidFamilyCore* core = DBReloadableInferEngine::get_core(); if (!core || !core->get()) { @@ -580,16 +559,12 @@ class FluidInferEngine : public CloneDBReloadableInferEngine { return -1; } - if (!core->Run(in, out)) { + if (!core->Run()) { LOG(ERROR) << "Failed run fluid family core"; return -1; } return 0; } - - int infer_impl2(const BatchTensor& in, BatchTensor& out) { // NOLINT - return infer_impl1(&in, &out); - } }; typedef FactoryPool StaticInferFactory; @@ -715,13 +690,45 @@ class VersionedInferEngine : public InferEngine { return _versions.begin()->second; } - int infer(const void* in, void* out, uint32_t batch_size) { + int infer() { InferEngine* engine = default_engine(); if (!engine) { LOG(WARNING) << "fail to get default engine"; return -1; } - return engine->infer(in, out, batch_size); + return engine->infer(); + } + + std::vector GetInputNames() { + InferEngine* engine = default_engine(); + if (!engine) { + LOG(WARNING) << "fail to get default engine"; + } + return engine->GetInputNames(); + } + std::vector GetOutputNames() { + InferEngine* engine = default_engine(); + if (!engine) { + LOG(WARNING) << "fail to get default engine"; + } + return engine->GetOutputNames(); + } + std::unique_ptr GetInputHandle( + const std::string& name) { + InferEngine* engine = default_engine(); + if (!engine) { + LOG(WARNING) << "fail to get default engine"; + } + return engine->GetInputHandle(name); + } + + std::unique_ptr GetOutputHandle( + const std::string& name) { + InferEngine* engine = default_engine(); + if (!engine) { + LOG(WARNING) << "fail to get default engine"; + } + return engine->GetOutputHandle(name); } template @@ -740,14 +747,47 @@ class VersionedInferEngine : public InferEngine { } // versioned inference interface - int infer(const void* in, void* out, uint32_t batch_size, uint64_t version) { + int infer(uint64_t version) { auto iter = _versions.find(version); if (iter == _versions.end()) { LOG(ERROR) << "Not found version engine: " << version; return -1; } - return iter->second->infer(in, out, batch_size); + return iter->second->infer(); + } + std::vector GetInputNames(uint64_t version) { + auto iter = _versions.find(version); + if (iter == _versions.end()) { + LOG(ERROR) << "Not found version engine: " << version; + } + return iter->second->GetInputNames(); + } + + std::vector GetOutputNames(uint64_t version) { + auto iter = _versions.find(version); + if (iter == _versions.end()) { + LOG(ERROR) << "Not found version engine: " << version; + } + return iter->second->GetOutputNames(); + } + + std::unique_ptr GetInputHandle( + uint64_t version, const std::string& name) { + auto iter = _versions.find(version); + if (iter == _versions.end()) { + LOG(ERROR) << "Not found version engine: " << version; + } + return iter->second->GetInputHandle(name); + } + + std::unique_ptr GetOutputHandle( + uint64_t version, const std::string& name) { + auto iter = _versions.find(version); + if (iter == _versions.end()) { + LOG(ERROR) << "Not found version engine: " << version; + } + return iter->second->GetOutputHandle(name); } template @@ -774,12 +814,7 @@ class VersionedInferEngine : public InferEngine { int thrd_finalize_impl() { return -1; } int thrd_clear_impl() { return -1; } int proc_finalize_impl() { return -1; } - int infer_impl1(const void* in, void* out, uint32_t batch_size = -1) { - return -1; - } - int infer_impl2(const BatchTensor& in, BatchTensor& out) { // NOLINT - return -1; - } // NOLINT + int infer_impl() { return -1; } private: boost::unordered_map _versions; @@ -877,16 +912,44 @@ class InferManager { } // Inference interface - int infer(const char* model_name, - const void* in, - void* out, - uint32_t batch_size = -1) { + int infer(const char* model_name) { auto it = _map.find(model_name); if (it == _map.end()) { LOG(WARNING) << "Cannot find engine in map, model name:" << model_name; return -1; } - return it->second->infer(in, out, batch_size); + return it->second->infer(); + } + + std::vector GetInputNames(const char* model_name) { + auto it = _map.find(model_name); + if (it == _map.end()) { + LOG(WARNING) << "Cannot find engine in map, model name:" << model_name; + } + return it->second->GetInputNames(); + } + std::vector GetOutputNames(const char* model_name) { + auto it = _map.find(model_name); + if (it == _map.end()) { + LOG(WARNING) << "Cannot find engine in map, model name:" << model_name; + } + return it->second->GetOutputNames(); + } + std::unique_ptr GetInputHandle( + const char* model_name, const std::string& name) { + auto it = _map.find(model_name); + if (it == _map.end()) { + LOG(WARNING) << "Cannot find engine in map, model name:" << model_name; + } + return it->second->GetInputHandle(name); + } + std::unique_ptr GetOutputHandle( + const char* model_name, const std::string& name) { + auto it = _map.find(model_name); + if (it == _map.end()) { + LOG(WARNING) << "Cannot find engine in map, model name:" << model_name; + } + return it->second->GetOutputHandle(name); } template @@ -906,19 +969,48 @@ class InferManager { } // Versioned inference interface - int infer(const char* model_name, - const void* in, - void* out, - uint32_t batch_size, - uint64_t version) { + int infer(const char* model_name, uint64_t version) { auto it = _map.find(model_name); if (it == _map.end()) { LOG(WARNING) << "Cannot find engine in map, model name:" << model_name; return -1; } - return it->second->infer(in, out, batch_size, version); + return it->second->infer(version); + } + std::vector GetInputNames(const char* model_name, + uint64_t version) { + auto it = _map.find(model_name); + if (it == _map.end()) { + LOG(WARNING) << "Cannot find engine in map, model name:" << model_name; + } + return it->second->GetInputNames(version); } + std::vector GetOutputNames(const char* model_name, + uint64_t version) { + auto it = _map.find(model_name); + if (it == _map.end()) { + LOG(WARNING) << "Cannot find engine in map, model name:" << model_name; + } + return it->second->GetOutputNames(version); + } + + std::unique_ptr GetInputHandle( + const char* model_name, uint64_t version, const std::string& name) { + auto it = _map.find(model_name); + if (it == _map.end()) { + LOG(WARNING) << "Cannot find engine in map, model name:" << model_name; + } + return it->second->GetInputHandle(version, name); + } + std::unique_ptr GetOutputHandle( + const char* model_name, uint64_t version, const std::string& name) { + auto it = _map.find(model_name); + if (it == _map.end()) { + LOG(WARNING) << "Cannot find engine in map, model name:" << model_name; + } + return it->second->GetOutputHandle(version, name); + } template T* get_core(const char* model_name, uint64_t version) { auto it = _map.find(model_name); diff --git a/doc/COMPILE_CN.md b/doc/COMPILE_CN.md index 0a31cb1b42017eeea12dfd891431b25c24d87777..9691808eda61a77808a971cc99648a7212b5747c 100644 --- a/doc/COMPILE_CN.md +++ b/doc/COMPILE_CN.md @@ -122,6 +122,7 @@ make -j10 export CUDA_PATH='/usr/local' export CUDNN_LIBRARY='/usr/local/cuda/lib64/' export CUDA_CUDART_LIBRARY="/usr/local/cuda/lib64/" +export TENSORRT_LIBRARY_PATH="/usr/local/TensorRT-6.0.1.5/targets/x86_64-linux-gnu/" mkdir server-build-trt && cd server-build-trt cmake -DPYTHON_INCLUDE_DIR=$PYTHONROOT/include/python2.7/ \ diff --git a/doc/PIPELINE_SERVING_CN.md b/doc/PIPELINE_SERVING_CN.md index 668901f92dc01a8abe22efc339c9202316155b14..268a962dcfb9af1ea6036340b4b8bf39d4c39f8f 100644 --- a/doc/PIPELINE_SERVING_CN.md +++ b/doc/PIPELINE_SERVING_CN.md @@ -676,7 +676,7 @@ service_throughput = 1 / 最慢OP的耗时 * 并发数 service_avg_cost = ∑op_concurrency 【关键路径】 Channel堆积： -channel_acc_size = QPS(down - up) * time +channel_acc_size = QPS(down - up) * time 批量预测平均耗时： avg_batch_cost = (N * pre + mid + post) / N diff --git a/doc/SAVE.md b/doc/SAVE.md index 8ebeb89c536f576bf73414fb06c1eb4bfde63ea0..8a909dc98d60579cd2861f5cdf38619264bae2fa 100644 --- a/doc/SAVE.md +++ b/doc/SAVE.md @@ -49,4 +49,4 @@ Arguments are the same as `inference_model_to_serving` API. | `serving_server` | str | `"serving_server"` | The path of model files and configuration files for server. | | `serving_client` | str | `"serving_client"` | The path of configuration files for client. | | `model_filename` | str | None | The name of file to load the inference program. If it is None, the default filename `__model__` will be used. | -| `paras_filename` | str | None | The name of file to load all parameters. It is only used for the case that all parameters were saved in a single binary file. If parameters were saved in separate files, set it as None. | +| `params_filename` | str | None | The name of file to load all parameters. It is only used for the case that all parameters were saved in a single binary file. If parameters were saved in separate files, set it as None. | diff --git a/doc/SAVE_CN.md b/doc/SAVE_CN.md index a05729ed9c01f421893403b4fc2a13bd42ad9fd4..3ede0471ab640a670fd5beb4ada68e0385b4c85b 100644 --- a/doc/SAVE_CN.md +++ b/doc/SAVE_CN.md @@ -50,4 +50,4 @@ python -m paddle_serving_client.convert --dirname ./your_inference_model_dir | `serving_server` | str | `"serving_server"` | 转换后的模型文件和配置文件的存储路径。默认值为serving_server | | `serving_client` | str | `"serving_client"` | 转换后的客户端配置文件存储路径。默认值为serving_client | | `model_filename` | str | None | 存储需要转换的模型Inference Program结构的文件名称。如果设置为None，则使用 `__model__` 作为默认的文件名 | -| `paras_filename` | str | None | 存储需要转换的模型所有参数的文件名称。当且仅当所有模型参数被保存在一个单独的二进制文件中，它才需要被指定。如果模型参数是存储在各自分离的文件中，设置它的值为None | +| `params_filename` | str | None | 存储需要转换的模型所有参数的文件名称。当且仅当所有模型参数被保存在一个单独的二进制文件中，它才需要被指定。如果模型参数是存储在各自分离的文件中，设置它的值为None | diff --git a/java/README.md b/java/README.md index 9990d1166d088acd5934fc36a267dd70e7cf0ee8..2346d13e20b4f81c454bd4bf731fe406015ab26f 100644 --- a/java/README.md +++ b/java/README.md @@ -117,4 +117,3 @@ The second is to deploy GPU Serving and Java Client separately. If they are on t **It should be noted that in the example, Java Pipeline Client code is in path /Java/Examples and /Java/src/main, and the Pipeline server code is in path /python/examples/pipeline/** - diff --git a/java/README_CN.md b/java/README_CN.md index 55b28f4cbd8661159c1e128de0fe8a0d4a5ed5ef..4c1df65fbeb78340187c9e603ff185751ebecf56 100644 --- a/java/README_CN.md +++ b/java/README_CN.md @@ -118,4 +118,4 @@ java -cp paddle-serving-sdk-java-examples-0.0.1-jar-with-dependencies.jar Pipeli **需要注意的是，Java Pipeline Client相关示例在/Java/Examples和/Java/src/main中，对应的Pipeline server在/python/examples/pipeline/中** - +**目前Serving已推出Pipeline模式（详见[Pipeline Serving](../doc/PIPELINE_SERVING_CN.md)），下个版本（0.4.1）面向Java的Pipeline Serving Client将会发布，敬请期待。** diff --git a/paddle_inference/inferencer-fluid-cpu/include/fluid_cpu_engine.h b/paddle_inference/inferencer-fluid-cpu/include/fluid_cpu_engine.h index a4d8dda71a7977185106bb1552cb8f39ef6bc50e..b20a4f4cf34e2f250788ae84c1b5b681d36cea4f 100644 --- a/paddle_inference/inferencer-fluid-cpu/include/fluid_cpu_engine.h +++ b/paddle_inference/inferencer-fluid-cpu/include/fluid_cpu_engine.h @@ -28,8 +28,6 @@ namespace baidu { namespace paddle_serving { namespace fluid_cpu { -using configure::SigmoidConf; - class AutoLock { public: explicit AutoLock(pthread_mutex_t& mutex) : _mut(mutex) { @@ -57,31 +55,36 @@ class GlobalPaddleCreateMutex { pthread_mutex_t _mut; }; -class GlobalSigmoidCreateMutex { - public: - pthread_mutex_t& mutex() { return _mut; } - static pthread_mutex_t& instance() { - static GlobalSigmoidCreateMutex gmutex; - return gmutex.mutex(); - } - - private: - GlobalSigmoidCreateMutex() { pthread_mutex_init(&_mut, NULL); } - - pthread_mutex_t _mut; -}; +using paddle_infer::Config; +using paddle_infer::Predictor; +using paddle_infer::Tensor; +using paddle_infer::CreatePredictor; // data interface class FluidFamilyCore { public: virtual ~FluidFamilyCore() {} - virtual bool Run(const void* in_data, void* out_data) { - if (!_core->Run(*(std::vector*)in_data, - (std::vector*)out_data)) { + virtual std::vector GetInputNames() { + return _core->GetInputNames(); + } + + virtual std::unique_ptr GetInputHandle(const std::string& name) { + return _core->GetInputHandle(name); + } + + virtual std::vector GetOutputNames() { + return _core->GetOutputNames(); + } + + virtual std::unique_ptr GetOutputHandle(const std::string& name) { + return _core->GetOutputHandle(name); + } + + virtual bool Run() { + if (!_core->Run()) { LOG(ERROR) << "Failed call Run with paddle predictor"; return false; } - return true; } @@ -92,8 +95,7 @@ class FluidFamilyCore { LOG(ERROR) << "origin paddle Predictor is null."; return -1; } - paddle::PaddlePredictor* p_predictor = - (paddle::PaddlePredictor*)origin_core; + Predictor* p_predictor = (Predictor*)origin_core; _core = p_predictor->Clone(); if (_core.get() == NULL) { LOG(ERROR) << "fail to clone paddle predictor: " << origin_core; @@ -105,7 +107,7 @@ class FluidFamilyCore { virtual void* get() { return _core.get(); } protected: - std::unique_ptr _core; + std::shared_ptr _core; }; // infer interface @@ -119,51 +121,19 @@ class FluidCpuAnalysisCore : public FluidFamilyCore { return -1; } - paddle::AnalysisConfig analysis_config; - analysis_config.SetParamsFile(data_path + "/__params__"); - analysis_config.SetProgFile(data_path + "/__model__"); - analysis_config.DisableGpu(); - analysis_config.SetCpuMathLibraryNumThreads(1); + Config config; + config.SetParamsFile(data_path + "/__params__"); + config.SetProgFile(data_path + "/__model__"); + config.DisableGpu(); + config.SetCpuMathLibraryNumThreads(1); if (params.enable_memory_optimization()) { - analysis_config.EnableMemoryOptim(); + config.EnableMemoryOptim(); } - analysis_config.SwitchSpecifyInputNames(true); - AutoLock lock(GlobalPaddleCreateMutex::instance()); - _core = - paddle::CreatePaddlePredictor(analysis_config); - if (NULL == _core.get()) { - LOG(ERROR) << "create paddle predictor failed, path: " << data_path; - return -1; - } - - VLOG(2) << "create paddle predictor sucess, path: " << data_path; - return 0; - } -}; - -class FluidCpuNativeCore : public FluidFamilyCore { - public: - int create(const predictor::InferEngineCreationParams& params) { - std::string data_path = params.get_path(); - if (access(data_path.c_str(), F_OK) == -1) { - LOG(ERROR) << "create paddle predictor failed, path not exits: " - << data_path; - return -1; - } - - paddle::NativeConfig native_config; - native_config.param_file = data_path + "/__params__"; - native_config.prog_file = data_path + "/__model__"; - native_config.use_gpu = false; - native_config.device = 0; - native_config.fraction_of_gpu_memory = 0; - + config.SwitchSpecifyInputNames(true); AutoLock lock(GlobalPaddleCreateMutex::instance()); - _core = paddle::CreatePaddlePredictor( - native_config); + _core = CreatePredictor(config); if (NULL == _core.get()) { LOG(ERROR) << "create paddle predictor failed, path: " << data_path; return -1; @@ -184,54 +154,24 @@ class FluidCpuAnalysisDirCore : public FluidFamilyCore { return -1; } - paddle::AnalysisConfig analysis_config; - analysis_config.SetModel(data_path); - analysis_config.DisableGpu(); - analysis_config.SwitchSpecifyInputNames(true); - analysis_config.SetCpuMathLibraryNumThreads(1); + Config config; + config.SetModel(data_path); + config.DisableGpu(); + config.SwitchSpecifyInputNames(true); + config.SetCpuMathLibraryNumThreads(1); if (params.enable_memory_optimization()) { - analysis_config.EnableMemoryOptim(); + config.EnableMemoryOptim(); } if (params.enable_ir_optimization()) { - analysis_config.SwitchIrOptim(true); + config.SwitchIrOptim(true); } else { - analysis_config.SwitchIrOptim(false); + config.SwitchIrOptim(false); } AutoLock lock(GlobalPaddleCreateMutex::instance()); - _core = - paddle::CreatePaddlePredictor(analysis_config); - if (NULL == _core.get()) { - LOG(ERROR) << "create paddle predictor failed, path: " << data_path; - return -1; - } - - VLOG(2) << "create paddle predictor sucess, path: " << data_path; - return 0; - } -}; - -class FluidCpuNativeDirCore : public FluidFamilyCore { - public: - int create(const predictor::InferEngineCreationParams& params) { - std::string data_path = params.get_path(); - if (access(data_path.c_str(), F_OK) == -1) { - LOG(ERROR) << "create paddle predictor failed, path not exits: " - << data_path; - return -1; - } - - paddle::NativeConfig native_config; - native_config.model_dir = data_path; - native_config.use_gpu = false; - native_config.device = 0; - native_config.fraction_of_gpu_memory = 0; - AutoLock lock(GlobalPaddleCreateMutex::instance()); - _core = paddle::CreatePaddlePredictor( - native_config); + _core = CreatePredictor(config); if (NULL == _core.get()) { LOG(ERROR) << "create paddle predictor failed, path: " << data_path; return -1; @@ -323,214 +263,6 @@ class Parameter { float* _params; }; -class SigmoidModel { - public: - ~SigmoidModel() {} - int load(const char* sigmoid_w_file, - const char* sigmoid_b_file, - float exp_max, - float exp_min) { - AutoLock lock(GlobalSigmoidCreateMutex::instance()); - if (0 != _sigmoid_w.init(2, 1, sigmoid_w_file) || 0 != _sigmoid_w.load()) { - LOG(ERROR) << "load params sigmoid_w failed."; - return -1; - } - VLOG(2) << "load sigmoid_w [" << _sigmoid_w._params[0] << "] [" - << _sigmoid_w._params[1] << "]."; - if (0 != _sigmoid_b.init(2, 1, sigmoid_b_file) || 0 != _sigmoid_b.load()) { - LOG(ERROR) << "load params sigmoid_b failed."; - return -1; - } - VLOG(2) << "load sigmoid_b [" << _sigmoid_b._params[0] << "] [" - << _sigmoid_b._params[1] << "]."; - _exp_max_input = exp_max; - _exp_min_input = exp_min; - return 0; - } - - int softmax(float x, double& o) { // NOLINT - float _y0 = x * _sigmoid_w._params[0] + _sigmoid_b._params[0]; - float _y1 = x * _sigmoid_w._params[1] + _sigmoid_b._params[1]; - _y0 = (_y0 > _exp_max_input) - ? _exp_max_input - : ((_y0 < _exp_min_input) ? _exp_min_input : _y0); - _y1 = (_y1 > _exp_max_input) - ? _exp_max_input - : ((_y1 < _exp_min_input) ? _exp_min_input : _y1); - o = 1.0f / (1.0f + exp(_y0 - _y1)); - return 0; - } - - public: - Parameter _sigmoid_w; - Parameter _sigmoid_b; - float _exp_max_input; - float _exp_min_input; -}; - -class SigmoidFluidModel { - public: - int softmax(float x, double& o) { // NOLINT - return _sigmoid_core->softmax(x, o); - } // NOLINT - - std::unique_ptr Clone() { - std::unique_ptr clone_model; - clone_model.reset(new SigmoidFluidModel()); - clone_model->_sigmoid_core = _sigmoid_core; - clone_model->_fluid_core = _fluid_core->Clone(); - return std::move(clone_model); // NOLINT - } - - public: - std::unique_ptr _fluid_core; - std::shared_ptr _sigmoid_core; -}; - -class FluidCpuWithSigmoidCore : public FluidFamilyCore { - public: - virtual ~FluidCpuWithSigmoidCore() {} - - public: - int create(const predictor::InferEngineCreationParams& params) { - std::string model_path = params.get_path(); - size_t pos = model_path.find_last_of("/\\"); - std::string conf_path = model_path.substr(0, pos); - std::string conf_file = model_path.substr(pos); - configure::SigmoidConf conf; - if (configure::read_proto_conf(conf_path, conf_file, &conf) != 0) { - LOG(ERROR) << "failed load model path: " << model_path; - return -1; - } - - _core.reset(new SigmoidFluidModel); - - std::string fluid_model_data_path = conf.dnn_model_path(); - predictor::InferEngineCreationParams new_params(params); - new_params.set_path(fluid_model_data_path); - int ret = load_fluid_model(new_params); - if (ret < 0) { - LOG(ERROR) << "fail to load fluid model."; - return -1; - } - const char* sigmoid_w_file = conf.sigmoid_w_file().c_str(); - const char* sigmoid_b_file = conf.sigmoid_b_file().c_str(); - float exp_max = conf.exp_max_input(); - float exp_min = conf.exp_min_input(); - _core->_sigmoid_core.reset(new SigmoidModel); - VLOG(2) << "create sigmoid core[" << _core->_sigmoid_core.get() - << "], use count[" << _core->_sigmoid_core.use_count() << "]."; - ret = _core->_sigmoid_core->load( - sigmoid_w_file, sigmoid_b_file, exp_max, exp_min); - if (ret < 0) { - LOG(ERROR) << "fail to load sigmoid model."; - return -1; - } - return 0; - } - - virtual bool Run(const void* in_data, void* out_data) { - if (!_core->_fluid_core->Run( - *(std::vector*)in_data, - (std::vector*)out_data)) { - LOG(ERROR) << "Failed call Run with paddle predictor"; - return false; - } - - return true; - } - - virtual int clone(SigmoidFluidModel* origin_core) { - if (origin_core == NULL) { - LOG(ERROR) << "origin paddle Predictor is null."; - return -1; - } - _core = origin_core->Clone(); - if (_core.get() == NULL) { - LOG(ERROR) << "fail to clone paddle predictor: " << origin_core; - return -1; - } - VLOG(2) << "clone sigmoid core[" << _core->_sigmoid_core.get() - << "] use count[" << _core->_sigmoid_core.use_count() << "]."; - return 0; - } - - virtual SigmoidFluidModel* get() { return _core.get(); } - - virtual int load_fluid_model( - const predictor::InferEngineCreationParams& params) = 0; - - int softmax(float x, double& o) { // NOLINT - return _core->_sigmoid_core->softmax(x, o); - } - - protected: - std::unique_ptr _core; // NOLINT -}; - -class FluidCpuNativeDirWithSigmoidCore : public FluidCpuWithSigmoidCore { - public: - int load_fluid_model(const predictor::InferEngineCreationParams& params) { - std::string data_path = params.get_path(); - if (access(data_path.c_str(), F_OK) == -1) { - LOG(ERROR) << "create paddle predictor failed, path not exits: " - << data_path; - return -1; - } - - paddle::NativeConfig native_config; - native_config.model_dir = data_path; - native_config.use_gpu = false; - native_config.device = 0; - native_config.fraction_of_gpu_memory = 0; - AutoLock lock(GlobalPaddleCreateMutex::instance()); - _core->_fluid_core = - paddle::CreatePaddlePredictor( - native_config); - if (NULL == _core.get()) { - LOG(ERROR) << "create paddle predictor failed, path: " << data_path; - return -1; - } - - VLOG(2) << "create paddle predictor sucess, path: " << data_path; - return 0; - } -}; - -class FluidCpuAnalysisDirWithSigmoidCore : public FluidCpuWithSigmoidCore { - public: - int load_fluid_model(const predictor::InferEngineCreationParams& params) { - std::string data_path = params.get_path(); - if (access(data_path.c_str(), F_OK) == -1) { - LOG(ERROR) << "create paddle predictor failed, path not exits: " - << data_path; - return -1; - } - - paddle::AnalysisConfig analysis_config; - analysis_config.SetModel(data_path); - analysis_config.DisableGpu(); - analysis_config.SwitchSpecifyInputNames(true); - analysis_config.SetCpuMathLibraryNumThreads(1); - - if (params.enable_memory_optimization()) { - analysis_config.EnableMemoryOptim(); - } - - AutoLock lock(GlobalPaddleCreateMutex::instance()); - _core->_fluid_core = - paddle::CreatePaddlePredictor(analysis_config); - if (NULL == _core.get()) { - LOG(ERROR) << "create paddle predictor failed, path: " << data_path; - return -1; - } - - VLOG(2) << "create paddle predictor sucess, path: " << data_path; - return 0; - } -}; - } // namespace fluid_cpu } // namespace paddle_serving } // namespace baidu diff --git a/paddle_inference/inferencer-fluid-cpu/src/fluid_cpu_engine.cpp b/paddle_inference/inferencer-fluid-cpu/src/fluid_cpu_engine.cpp index af3f93a8129282920f4cb6fd1d074e0c7eb46228..91cb0bd20c97e53952f95bb05a25582242793f57 100644 --- a/paddle_inference/inferencer-fluid-cpu/src/fluid_cpu_engine.cpp +++ b/paddle_inference/inferencer-fluid-cpu/src/fluid_cpu_engine.cpp @@ -30,28 +30,6 @@ REGIST_FACTORY_OBJECT_IMPL_WITH_NAME( ::baidu::paddle_serving::predictor::InferEngine, "FLUID_CPU_ANALYSIS_DIR"); -REGIST_FACTORY_OBJECT_IMPL_WITH_NAME( - ::baidu::paddle_serving::predictor::FluidInferEngine< - FluidCpuAnalysisDirWithSigmoidCore>, - ::baidu::paddle_serving::predictor::InferEngine, - "FLUID_CPU_ANALYSIS_DIR_SIGMOID"); - -REGIST_FACTORY_OBJECT_IMPL_WITH_NAME( - ::baidu::paddle_serving::predictor::FluidInferEngine, - ::baidu::paddle_serving::predictor::InferEngine, - "FLUID_CPU_NATIVE"); - -REGIST_FACTORY_OBJECT_IMPL_WITH_NAME( - ::baidu::paddle_serving::predictor::FluidInferEngine, - ::baidu::paddle_serving::predictor::InferEngine, - "FLUID_CPU_NATIVE_DIR"); - -REGIST_FACTORY_OBJECT_IMPL_WITH_NAME( - ::baidu::paddle_serving::predictor::FluidInferEngine< - FluidCpuNativeDirWithSigmoidCore>, - ::baidu::paddle_serving::predictor::InferEngine, - "FLUID_CPU_NATIVE_DIR_SIGMOID"); - } // namespace fluid_cpu } // namespace paddle_serving } // namespace baidu diff --git a/paddle_inference/inferencer-fluid-gpu/include/fluid_gpu_engine.h b/paddle_inference/inferencer-fluid-gpu/include/fluid_gpu_engine.h index 3782c967823d07c23ba02e5ce0f388dc6b46e181..3d59a5009471ff5c76e037a941a0da87377684ab 100644 --- a/paddle_inference/inferencer-fluid-gpu/include/fluid_gpu_engine.h +++ b/paddle_inference/inferencer-fluid-gpu/include/fluid_gpu_engine.h @@ -61,31 +61,36 @@ class GlobalPaddleCreateMutex { pthread_mutex_t _mut; }; -class GlobalSigmoidCreateMutex { - public: - pthread_mutex_t& mutex() { return _mut; } - static pthread_mutex_t& instance() { - static GlobalSigmoidCreateMutex gmutex; - return gmutex.mutex(); - } - - private: - GlobalSigmoidCreateMutex() { pthread_mutex_init(&_mut, NULL); } - - pthread_mutex_t _mut; -}; +using paddle_infer::Config; +using paddle_infer::Predictor; +using paddle_infer::Tensor; +using paddle_infer::CreatePredictor; // data interface class FluidFamilyCore { public: virtual ~FluidFamilyCore() {} - virtual bool Run(const void* in_data, void* out_data) { - if (!_core->Run(*(std::vector*)in_data, - (std::vector*)out_data)) { + virtual std::vector GetInputNames() { + return _core->GetInputNames(); + } + + virtual std::unique_ptr GetInputHandle(const std::string& name) { + return _core->GetInputHandle(name); + } + + virtual std::vector GetOutputNames() { + return _core->GetOutputNames(); + } + + virtual std::unique_ptr GetOutputHandle(const std::string& name) { + return _core->GetOutputHandle(name); + } + + virtual bool Run() { + if (!_core->Run()) { LOG(ERROR) << "Failed call Run with paddle predictor"; return false; } - return true; } @@ -96,8 +101,7 @@ class FluidFamilyCore { LOG(ERROR) << "origin paddle Predictor is null."; return -1; } - paddle::PaddlePredictor* p_predictor = - (paddle::PaddlePredictor*)origin_core; + Predictor* p_predictor = (Predictor*)origin_core; _core = p_predictor->Clone(); if (_core.get() == NULL) { LOG(ERROR) << "fail to clone paddle predictor: " << origin_core; @@ -109,7 +113,7 @@ class FluidFamilyCore { virtual void* get() { return _core.get(); } protected: - std::unique_ptr _core; + std::shared_ptr _core; }; // infer interface @@ -123,51 +127,19 @@ class FluidGpuAnalysisCore : public FluidFamilyCore { return -1; } - paddle::AnalysisConfig analysis_config; - analysis_config.SetParamsFile(data_path + "/__params__"); - analysis_config.SetProgFile(data_path + "/__model__"); - analysis_config.EnableUseGpu(100, FLAGS_gpuid); - analysis_config.SetCpuMathLibraryNumThreads(1); + Config config; + config.SetParamsFile(data_path + "/__params__"); + config.SetProgFile(data_path + "/__model__"); + config.EnableUseGpu(100, FLAGS_gpuid); + config.SetCpuMathLibraryNumThreads(1); if (params.enable_memory_optimization()) { - analysis_config.EnableMemoryOptim(); + config.EnableMemoryOptim(); } - analysis_config.SwitchSpecifyInputNames(true); - + config.SwitchSpecifyInputNames(true); AutoLock lock(GlobalPaddleCreateMutex::instance()); - _core = - paddle::CreatePaddlePredictor(analysis_config); - if (NULL == _core.get()) { - LOG(ERROR) << "create paddle predictor failed, path: " << data_path; - return -1; - } - - VLOG(2) << "create paddle predictor sucess, path: " << data_path; - return 0; - } -}; - -class FluidGpuNativeCore : public FluidFamilyCore { - public: - int create(const predictor::InferEngineCreationParams& params) { - std::string data_path = params.get_path(); - if (access(data_path.c_str(), F_OK) == -1) { - LOG(ERROR) << "create paddle predictor failed, path not exits: " - << data_path; - return -1; - } - - paddle::NativeConfig native_config; - native_config.param_file = data_path + "/__params__"; - native_config.prog_file = data_path + "/__model__"; - native_config.use_gpu = true; - native_config.fraction_of_gpu_memory = 0.01; - native_config.device = FLAGS_gpuid; - AutoLock lock(GlobalPaddleCreateMutex::instance()); - _core = paddle::CreatePaddlePredictor( - native_config); + _core = CreatePredictor(config); if (NULL == _core.get()) { LOG(ERROR) << "create paddle predictor failed, path: " << data_path; return -1; @@ -188,110 +160,38 @@ class FluidGpuAnalysisDirCore : public FluidFamilyCore { return -1; } - paddle::AnalysisConfig analysis_config; - analysis_config.SetModel(data_path); - analysis_config.EnableUseGpu(1500, FLAGS_gpuid); - analysis_config.SwitchSpecifyInputNames(true); - analysis_config.SetCpuMathLibraryNumThreads(1); + Config config; + config.SetModel(data_path); + config.EnableUseGpu(1500, FLAGS_gpuid); + config.SwitchSpecifyInputNames(true); + config.SetCpuMathLibraryNumThreads(1); if (params.enable_memory_optimization()) { - analysis_config.EnableMemoryOptim(); + config.EnableMemoryOptim(); } - -#if 0 // todo: support flexible shape - - int min_seq_len = 1; - int max_seq_len = 512; - int opt_seq_len = 128; - int head_number = 12; - int batch = 50; - - std::vector min_in_shape = {batch, min_seq_len, 1}; - std::vector max_in_shape = {batch, max_seq_len, 1}; - std::vector opt_in_shape = {batch, opt_seq_len, 1}; - - std::string input1_name = "src_text_a_ids"; - std::string input2_name = "pos_text_a_ids"; - std::string input3_name = "sent_text_a_ids"; - std::string input4_name = "stack_0.tmp_0"; - - std::map> min_input_shape = { - {input1_name, min_in_shape}, - {input2_name, min_in_shape}, - {input3_name, min_in_shape}, - {input4_name, {batch, head_number, min_seq_len, min_seq_len}}, - }; - - std::map> max_input_shape = { - {input1_name, max_in_shape}, - {input2_name, max_in_shape}, - {input3_name, max_in_shape}, - {input4_name, {batch, head_number, max_seq_len, max_seq_len}}, - }; - std::map> opt_input_shape = { - {input1_name, opt_in_shape}, - {input2_name, opt_in_shape}, - {input3_name, opt_in_shape}, - {input4_name, {batch, head_number, opt_seq_len, opt_seq_len}}, - }; - - analysis_config.SetTRTDynamicShapeInfo( - min_input_shape, max_input_shape, opt_input_shape); -#endif int max_batch = 32; int min_subgraph_size = 3; if (params.use_trt()) { - analysis_config.EnableTensorRtEngine( - 1 << 20, - max_batch, - min_subgraph_size, - paddle::AnalysisConfig::Precision::kFloat32, - false, - false); + config.EnableTensorRtEngine(1 << 20, + max_batch, + min_subgraph_size, + Config::Precision::kFloat32, + false, + false); LOG(INFO) << "create TensorRT predictor"; } else { if (params.enable_memory_optimization()) { - analysis_config.EnableMemoryOptim(); + config.EnableMemoryOptim(); } if (params.enable_ir_optimization()) { - analysis_config.SwitchIrOptim(true); + config.SwitchIrOptim(true); } else { - analysis_config.SwitchIrOptim(false); + config.SwitchIrOptim(false); } } AutoLock lock(GlobalPaddleCreateMutex::instance()); - _core = - paddle::CreatePaddlePredictor(analysis_config); - if (NULL == _core.get()) { - LOG(ERROR) << "create paddle predictor failed, path: " << data_path; - return -1; - } - - VLOG(2) << "create paddle predictor sucess, path: " << data_path; - return 0; - } -}; - -class FluidGpuNativeDirCore : public FluidFamilyCore { - public: - int create(const predictor::InferEngineCreationParams& params) { - std::string data_path = params.get_path(); - if (access(data_path.c_str(), F_OK) == -1) { - LOG(ERROR) << "create paddle predictor failed, path not exits: " - << data_path; - return -1; - } - - paddle::NativeConfig native_config; - native_config.model_dir = data_path; - native_config.use_gpu = true; - native_config.fraction_of_gpu_memory = 0.01; - native_config.device = FLAGS_gpuid; - AutoLock lock(GlobalPaddleCreateMutex::instance()); - _core = paddle::CreatePaddlePredictor( - native_config); + _core = CreatePredictor(config); if (NULL == _core.get()) { LOG(ERROR) << "create paddle predictor failed, path: " << data_path; return -1; @@ -383,214 +283,6 @@ class Parameter { float* _params; }; -class SigmoidModel { - public: - ~SigmoidModel() {} - int load(const char* sigmoid_w_file, - const char* sigmoid_b_file, - float exp_max, - float exp_min) { - AutoLock lock(GlobalSigmoidCreateMutex::instance()); - if (0 != _sigmoid_w.init(2, 1, sigmoid_w_file) || 0 != _sigmoid_w.load()) { - LOG(ERROR) << "load params sigmoid_w failed."; - return -1; - } - VLOG(2) << "load sigmoid_w [" << _sigmoid_w._params[0] << "] [" - << _sigmoid_w._params[1] << "]."; - if (0 != _sigmoid_b.init(2, 1, sigmoid_b_file) || 0 != _sigmoid_b.load()) { - LOG(ERROR) << "load params sigmoid_b failed."; - return -1; - } - VLOG(2) << "load sigmoid_b [" << _sigmoid_b._params[0] << "] [" - << _sigmoid_b._params[1] << "]."; - _exp_max_input = exp_max; - _exp_min_input = exp_min; - return 0; - } - - int softmax(float x, double& o) { // NOLINT - float _y0 = x * _sigmoid_w._params[0] + _sigmoid_b._params[0]; - float _y1 = x * _sigmoid_w._params[1] + _sigmoid_b._params[1]; - _y0 = (_y0 > _exp_max_input) - ? _exp_max_input - : ((_y0 < _exp_min_input) ? _exp_min_input : _y0); - _y1 = (_y1 > _exp_max_input) - ? _exp_max_input - : ((_y1 < _exp_min_input) ? _exp_min_input : _y1); - o = 1.0f / (1.0f + exp(_y0 - _y1)); - return 0; - } - - public: - Parameter _sigmoid_w; - Parameter _sigmoid_b; - float _exp_max_input; - float _exp_min_input; -}; - -class SigmoidFluidModel { - public: - int softmax(float x, double& o) { // NOLINT - return _sigmoid_core->softmax(x, o); - } // NOLINT - - std::unique_ptr Clone() { - std::unique_ptr clone_model; - clone_model.reset(new SigmoidFluidModel()); - clone_model->_sigmoid_core = _sigmoid_core; - clone_model->_fluid_core = _fluid_core->Clone(); - return std::move(clone_model); - } - - public: - std::unique_ptr _fluid_core; - std::shared_ptr _sigmoid_core; -}; - -class FluidGpuWithSigmoidCore : public FluidFamilyCore { - public: - virtual ~FluidGpuWithSigmoidCore() {} - - public: - int create(const predictor::InferEngineCreationParams& params) { - std::string model_path = params.get_path(); - size_t pos = model_path.find_last_of("/\\"); - std::string conf_path = model_path.substr(0, pos); - std::string conf_file = model_path.substr(pos); - configure::SigmoidConf conf; - if (configure::read_proto_conf(conf_path, conf_file, &conf) != 0) { - LOG(ERROR) << "failed load model path: " << model_path; - return -1; - } - - _core.reset(new SigmoidFluidModel); - - std::string fluid_model_data_path = conf.dnn_model_path(); - predictor::InferEngineCreationParams new_params(params); - new_params.set_path(fluid_model_data_path); - int ret = load_fluid_model(new_params); - if (ret < 0) { - LOG(ERROR) << "fail to load fluid model."; - return -1; - } - const char* sigmoid_w_file = conf.sigmoid_w_file().c_str(); - const char* sigmoid_b_file = conf.sigmoid_b_file().c_str(); - float exp_max = conf.exp_max_input(); - float exp_min = conf.exp_min_input(); - _core->_sigmoid_core.reset(new SigmoidModel); - LOG(INFO) << "create sigmoid core[" << _core->_sigmoid_core.get() - << "], use count[" << _core->_sigmoid_core.use_count() << "]."; - ret = _core->_sigmoid_core->load( - sigmoid_w_file, sigmoid_b_file, exp_max, exp_min); - if (ret < 0) { - LOG(ERROR) << "fail to load sigmoid model."; - return -1; - } - return 0; - } - - virtual bool Run(const void* in_data, void* out_data) { - if (!_core->_fluid_core->Run( - *(std::vector*)in_data, - (std::vector*)out_data)) { - LOG(ERROR) << "Failed call Run with paddle predictor"; - return false; - } - - return true; - } - - virtual int clone(SigmoidFluidModel* origin_core) { - if (origin_core == NULL) { - LOG(ERROR) << "origin paddle Predictor is null."; - return -1; - } - _core = origin_core->Clone(); - if (_core.get() == NULL) { - LOG(ERROR) << "fail to clone paddle predictor: " << origin_core; - return -1; - } - LOG(INFO) << "clone sigmoid core[" << _core->_sigmoid_core.get() - << "] use count[" << _core->_sigmoid_core.use_count() << "]."; - return 0; - } - - virtual SigmoidFluidModel* get() { return _core.get(); } - - virtual int load_fluid_model( - const predictor::InferEngineCreationParams& params) = 0; - - int softmax(float x, double& o) { // NOLINT - return _core->_sigmoid_core->softmax(x, o); - } - - protected: - std::unique_ptr _core; -}; - -class FluidGpuNativeDirWithSigmoidCore : public FluidGpuWithSigmoidCore { - public: - int load_fluid_model(const predictor::InferEngineCreationParams& params) { - std::string data_path = params.get_path(); - if (access(data_path.c_str(), F_OK) == -1) { - LOG(ERROR) << "create paddle predictor failed, path not exits: " - << data_path; - return -1; - } - - paddle::NativeConfig native_config; - native_config.model_dir = data_path; - native_config.use_gpu = true; - native_config.fraction_of_gpu_memory = 0.01; - native_config.device = FLAGS_gpuid; - AutoLock lock(GlobalPaddleCreateMutex::instance()); - _core->_fluid_core = - paddle::CreatePaddlePredictor( - native_config); - if (NULL == _core.get()) { - LOG(ERROR) << "create paddle predictor failed, path: " << data_path; - return -1; - } - - VLOG(2) << "create paddle predictor sucess, path: " << data_path; - return 0; - } -}; - -class FluidGpuAnalysisDirWithSigmoidCore : public FluidGpuWithSigmoidCore { - public: - int load_fluid_model(const predictor::InferEngineCreationParams& params) { - std::string data_path = params.get_path(); - if (access(data_path.c_str(), F_OK) == -1) { - LOG(ERROR) << "create paddle predictor failed, path not exits: " - << data_path; - return -1; - } - - paddle::AnalysisConfig analysis_config; - analysis_config.SetModel(data_path); - analysis_config.EnableUseGpu(100, FLAGS_gpuid); - analysis_config.SwitchSpecifyInputNames(true); - analysis_config.SetCpuMathLibraryNumThreads(1); - - if (params.enable_memory_optimization()) { - analysis_config.EnableMemoryOptim(); - } - - AutoLock lock(GlobalPaddleCreateMutex::instance()); - _core->_fluid_core = - paddle::CreatePaddlePredictor(analysis_config); - if (NULL == _core.get()) { - LOG(ERROR) << "create paddle predictor failed, path: " << data_path; - return -1; - } - - VLOG(2) << "create paddle predictor sucess, path: " << data_path; - return 0; - } -}; - } // namespace fluid_gpu } // namespace paddle_serving } // namespace baidu diff --git a/paddle_inference/inferencer-fluid-gpu/src/fluid_gpu_engine.cpp b/paddle_inference/inferencer-fluid-gpu/src/fluid_gpu_engine.cpp index 7447a417338a37716eff025721126e4c817408a6..c00ea8719414f5ac324ac62e3e36128ad6035f91 100644 --- a/paddle_inference/inferencer-fluid-gpu/src/fluid_gpu_engine.cpp +++ b/paddle_inference/inferencer-fluid-gpu/src/fluid_gpu_engine.cpp @@ -32,28 +32,6 @@ REGIST_FACTORY_OBJECT_IMPL_WITH_NAME( ::baidu::paddle_serving::predictor::InferEngine, "FLUID_GPU_ANALYSIS_DIR"); -REGIST_FACTORY_OBJECT_IMPL_WITH_NAME( - ::baidu::paddle_serving::predictor::FluidInferEngine< - FluidGpuAnalysisDirWithSigmoidCore>, - ::baidu::paddle_serving::predictor::InferEngine, - "FLUID_GPU_ANALYSIS_DIR_SIGMOID"); - -REGIST_FACTORY_OBJECT_IMPL_WITH_NAME( - ::baidu::paddle_serving::predictor::FluidInferEngine, - ::baidu::paddle_serving::predictor::InferEngine, - "FLUID_GPU_NATIVE"); - -REGIST_FACTORY_OBJECT_IMPL_WITH_NAME( - ::baidu::paddle_serving::predictor::FluidInferEngine, - ::baidu::paddle_serving::predictor::InferEngine, - "FLUID_GPU_NATIVE_DIR"); - -REGIST_FACTORY_OBJECT_IMPL_WITH_NAME( - ::baidu::paddle_serving::predictor::FluidInferEngine< - FluidGpuNativeDirWithSigmoidCore>, - ::baidu::paddle_serving::predictor::InferEngine, - "FLUID_GPU_NATIVE_DIR_SIGMOID"); - } // namespace fluid_gpu } // namespace paddle_serving } // namespace baidu diff --git a/python/examples/fit_a_line/local_train.py b/python/examples/fit_a_line/local_train.py index 900b4a674a96434f4e848d1d4fd8f2ebed79f148..3e0f8880a4d006b346712f2592d6c44986882193 100644 --- a/python/examples/fit_a_line/local_train.py +++ b/python/examples/fit_a_line/local_train.py @@ -16,7 +16,7 @@ import sys import paddle import paddle.fluid as fluid - +paddle.enable_static() train_reader = paddle.batch( paddle.reader.shuffle( paddle.dataset.uci_housing.train(), buf_size=500), diff --git a/python/examples/grpc_impl_example/fit_a_line/README_CN.md b/python/examples/grpc_impl_example/fit_a_line/README_CN.md index 93e0d1cf7262d620df18570401ed39db67f839ef..4b2bd59e7ba3a52952496b929689c6bd026bf0ce 100644 --- a/python/examples/grpc_impl_example/fit_a_line/README_CN.md +++ b/python/examples/grpc_impl_example/fit_a_line/README_CN.md @@ -38,20 +38,9 @@ python test_asyn_client.py python test_batch_client.py ``` -### 通用 pb 预测 - -``` shell -python test_general_pb_client.py -``` - ### 预测超时 ``` shell python test_timeout_client.py ``` -### List 输入 - -``` shell -python test_list_input_client.py -``` diff --git a/python/examples/grpc_impl_example/fit_a_line/test_asyn_client.py b/python/examples/grpc_impl_example/fit_a_line/test_asyn_client.py index b01a9372585bae42abca213fe8fb8a55505dfe57..eb0e1c2dcaad998a51b370f63655299ce8d93889 100644 --- a/python/examples/grpc_impl_example/fit_a_line/test_asyn_client.py +++ b/python/examples/grpc_impl_example/fit_a_line/test_asyn_client.py @@ -18,7 +18,7 @@ import functools import time import threading import grpc - +import numpy as np client = Client() client.connect(["127.0.0.1:9393"]) @@ -43,7 +43,8 @@ x = [ ] task_count = 0 for i in range(3): - future = client.predict(feed={"x": x}, fetch=["price"], asyn=True) + new_data = np.array(x).astype("float32").reshape((1,13)) + future = client.predict(feed={"x": new_data}, fetch=["price"], batch=False, asyn=True) task_count += 1 future.add_done_callback(functools.partial(call_back)) diff --git a/python/examples/grpc_impl_example/fit_a_line/test_batch_client.py b/python/examples/grpc_impl_example/fit_a_line/test_batch_client.py index 0630a0a960e5e40a7507454feb57418c8cfbdc68..30da59342571dfc2353a5177476ac5d229b91181 100644 --- a/python/examples/grpc_impl_example/fit_a_line/test_batch_client.py +++ b/python/examples/grpc_impl_example/fit_a_line/test_batch_client.py @@ -13,7 +13,7 @@ # limitations under the License. # pylint: disable=doc-string-missing from paddle_serving_client import MultiLangClient as Client - +import numpy as np client = Client() client.connect(["127.0.0.1:9393"]) @@ -24,8 +24,11 @@ x = [ ] for i in range(3): - batch_feed = [{"x": x} for j in range(batch_size)] - fetch_map = client.predict(feed=batch_feed, fetch=["price"]) + new_data = np.array(x).astype("float32").reshape((1, 1, 13)) + batch_data = np.concatenate([new_data, new_data, new_data], axis=0) + print(batch_data.shape) + fetch_map = client.predict(feed={"x":batch_data}, fetch=["price"], batch=True) + if fetch_map["serving_status_code"] == 0: print(fetch_map) else: diff --git a/python/examples/grpc_impl_example/fit_a_line/test_general_pb_client.py b/python/examples/grpc_impl_example/fit_a_line/test_general_pb_client.py deleted file mode 100644 index b2744906b0dcd321f86a1b8117a78307e24578e5..0000000000000000000000000000000000000000 --- a/python/examples/grpc_impl_example/fit_a_line/test_general_pb_client.py +++ /dev/null @@ -1,30 +0,0 @@ -# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# pylint: disable=doc-string-missing - -from paddle_serving_client import MultiLangClient as Client - -client = Client() -client.connect(["127.0.0.1:9393"]) - -x = [ - 0.0137, -0.1136, 0.2553, -0.0692, 0.0582, -0.0727, -0.1583, -0.0584, 0.6283, - 0.4919, 0.1856, 0.0795, -0.0332 -] -for i in range(3): - fetch_map = client.predict(feed={"x": x}, fetch=["price"], is_python=False) - if fetch_map["serving_status_code"] == 0: - print(fetch_map) - else: - print(fetch_map["serving_status_code"]) diff --git a/python/examples/grpc_impl_example/fit_a_line/test_numpy_input_client.py b/python/examples/grpc_impl_example/fit_a_line/test_numpy_input_client.py deleted file mode 100644 index e98c1e87bb48613e4226cf5378063aec7c5b4093..0000000000000000000000000000000000000000 --- a/python/examples/grpc_impl_example/fit_a_line/test_numpy_input_client.py +++ /dev/null @@ -1,31 +0,0 @@ -# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# pylint: disable=doc-string-missing - -from paddle_serving_client import MultiLangClient as Client -import numpy as np - -client = Client() -client.connect(["127.0.0.1:9393"]) - -x = [ - 0.0137, -0.1136, 0.2553, -0.0692, 0.0582, -0.0727, -0.1583, -0.0584, 0.6283, - 0.4919, 0.1856, 0.0795, -0.0332 -] -for i in range(3): - fetch_map = client.predict(feed={"x": np.array(x)}, fetch=["price"]) - if fetch_map["serving_status_code"] == 0: - print(fetch_map) - else: - print(fetch_map["serving_status_code"]) diff --git a/python/examples/grpc_impl_example/fit_a_line/test_sync_client.py b/python/examples/grpc_impl_example/fit_a_line/test_sync_client.py index 89530dc2f2a33ef44b2dbde52975634f4b4d8295..dbc9a7bbdd31e37726edef4eb71de08c90ec39d2 100644 --- a/python/examples/grpc_impl_example/fit_a_line/test_sync_client.py +++ b/python/examples/grpc_impl_example/fit_a_line/test_sync_client.py @@ -14,16 +14,27 @@ # pylint: disable=doc-string-missing from paddle_serving_client import MultiLangClient as Client - +import numpy as np client = Client() client.connect(["127.0.0.1:9393"]) +""" +for data in test_reader(): + new_data = np.zeros((1, 1, 13)).astype("float32") + new_data[0] = data[0][0] + fetch_map = client.predict( + feed={"x": new_data}, fetch=["price"], batch=True) + print("{} {}".format(fetch_map["price"][0], data[0][1][0])) + print(fetch_map) +""" + x = [ 0.0137, -0.1136, 0.2553, -0.0692, 0.0582, -0.0727, -0.1583, -0.0584, 0.6283, 0.4919, 0.1856, 0.0795, -0.0332 ] for i in range(3): - fetch_map = client.predict(feed={"x": x}, fetch=["price"]) + new_data = np.array(x).astype("float32").reshape((1,13)) + fetch_map = client.predict(feed={"x": new_data}, fetch=["price"], batch=False) if fetch_map["serving_status_code"] == 0: print(fetch_map) else: diff --git a/python/examples/grpc_impl_example/fit_a_line/test_timeout_client.py b/python/examples/grpc_impl_example/fit_a_line/test_timeout_client.py index f90fab38533aabf3daa7627ee0b79c56892444dd..082fc9080ec49a0fc2bcaef68842a1c1695faf7c 100644 --- a/python/examples/grpc_impl_example/fit_a_line/test_timeout_client.py +++ b/python/examples/grpc_impl_example/fit_a_line/test_timeout_client.py @@ -15,17 +15,18 @@ from paddle_serving_client import MultiLangClient as Client import grpc - +import numpy as np client = Client() client.connect(["127.0.0.1:9393"]) -client.set_rpc_timeout_ms(1) +client.set_rpc_timeout_ms(40) x = [ 0.0137, -0.1136, 0.2553, -0.0692, 0.0582, -0.0727, -0.1583, -0.0584, 0.6283, 0.4919, 0.1856, 0.0795, -0.0332 ] for i in range(3): - fetch_map = client.predict(feed={"x": x}, fetch=["price"]) + new_data = np.array(x).astype("float32").reshape((1,13)) + fetch_map = client.predict(feed={"x": new_data}, fetch=["price"], batch=False) if fetch_map["serving_status_code"] == 0: print(fetch_map) elif fetch_map["serving_status_code"] == grpc.StatusCode.DEADLINE_EXCEEDED: diff --git a/python/examples/grpc_impl_example/yolov4/test_client.py b/python/examples/grpc_impl_example/yolov4/test_client.py index a55763880f7852f0297d7e6c7f44f8c3a206dc60..49573bb79ef5be09fc39f882c980d3c048d5ceba 100644 --- a/python/examples/grpc_impl_example/yolov4/test_client.py +++ b/python/examples/grpc_impl_example/yolov4/test_client.py @@ -27,7 +27,7 @@ preprocess = Sequential([ postprocess = RCNNPostprocess("label_list.txt", "output", [608, 608]) client = Client() client.connect(['127.0.0.1:9393']) -# client.set_rpc_timeout_ms(10000) +client.set_rpc_timeout_ms(15000) im = preprocess(sys.argv[1]) fetch_map = client.predict( @@ -35,7 +35,8 @@ fetch_map = client.predict( "image": im, "im_size": np.array(list(im.shape[1:])), }, - fetch=["save_infer_model/scale_0.tmp_0"]) + fetch=["save_infer_model/scale_0.tmp_0"], batch=False) +print(fetch_map) fetch_map.pop("serving_status_code") fetch_map["image"] = sys.argv[1] postprocess(fetch_map) diff --git a/python/paddle_serving_client/__init__.py b/python/paddle_serving_client/__init__.py index 6f3908fd6445854f7c398d6b228112b99898028d..b2094b3b29b9fedfacd01af179841a135c36f9f9 100644 --- a/python/paddle_serving_client/__init__.py +++ b/python/paddle_serving_client/__init__.py @@ -522,78 +522,48 @@ class MultiLangClient(object): req.fetch_var_names.extend(fetch) req.is_python = is_python req.log_id = log_id - feed_batch = None - if isinstance(feed, dict): - feed_batch = [feed] - elif isinstance(feed, list): - feed_batch = feed - else: - raise Exception("{} not support".format(type(feed))) - req.feed_var_names.extend(feed_batch[0].keys()) - init_feed_names = False - for feed_data in feed_batch: - inst = multi_lang_general_model_service_pb2.FeedInst() - for name in req.feed_var_names: - tensor = multi_lang_general_model_service_pb2.Tensor() - var = feed_data[name] - v_type = self.feed_types_[name] - if is_python: - data = None - if isinstance(var, list): - if v_type == 0: # int64 - data = np.array(var, dtype="int64") - elif v_type == 1: # float32 - data = np.array(var, dtype="float32") - elif v_type == 2: # int32 - data = np.array(var, dtype="int32") - else: - raise Exception("error tensor value type.") - elif isinstance(var, np.ndarray): - data = var - if v_type == 0: - if data.dtype != 'int64': - data = data.astype("int64") - elif v_type == 1: - if data.dtype != 'float32': - data = data.astype("float32") - elif v_type == 2: - if data.dtype != 'int32': - data = data.astype("int32") - else: - raise Exception("error tensor value type.") + feed_var_names = [] + for key in feed.keys(): + if '.lod' not in key: + feed_var_names.append(key) + req.feed_var_names.extend(feed_var_names) + inst = multi_lang_general_model_service_pb2.FeedInst() + for name in req.feed_var_names: + tensor = multi_lang_general_model_service_pb2.Tensor() + var = feed[name] + v_type = self.feed_types_[name] + if is_python: + data = None + if isinstance(var, list): + if v_type == 0: # int64 + data = np.array(var, dtype="int64") + elif v_type == 1: # float32 + data = np.array(var, dtype="float32") + elif v_type == 2: # int32 + data = np.array(var, dtype="int32") else: - raise Exception("var must be list or ndarray.") - tensor.data = data.tobytes() - else: - if isinstance(var, np.ndarray): - if v_type == 0: # int64 - tensor.int64_data.extend( - var.reshape(-1).astype("int64").tolist()) - elif v_type == 1: - tensor.float_data.extend( - var.reshape(-1).astype('float32').tolist()) - elif v_type == 2: - tensor.int_data.extend( - var.reshape(-1).astype('int32').tolist()) - else: - raise Exception("error tensor value type.") - elif isinstance(var, list): - if v_type == 0: - tensor.int64_data.extend(self._flatten_list(var)) - elif v_type == 1: - tensor.float_data.extend(self._flatten_list(var)) - elif v_type == 2: - tensor.int_data.extend(self._flatten_list(var)) - else: - raise Exception("error tensor value type.") + raise Exception("error tensor value type.") + elif isinstance(var, np.ndarray): + data = var + if v_type == 0: + if data.dtype != 'int64': + data = data.astype("int64") + elif v_type == 1: + if data.dtype != 'float32': + data = data.astype("float32") + elif v_type == 2: + if data.dtype != 'int32': + data = data.astype("int32") else: - raise Exception("var must be list or ndarray.") - if isinstance(var, np.ndarray): - tensor.shape.extend(list(var.shape)) + raise Exception("error tensor value type.") else: - tensor.shape.extend(self.feed_shapes_[name]) - inst.tensor_array.append(tensor) - req.insts.append(inst) + raise Exception("var must be list or ndarray.") + tensor.data = data.tobytes() + tensor.shape.extend(list(var.shape)) + if "{}.lod".format(name) in feed.keys(): + tensor.lod.extend(feed["{}.lod".format(name)]) + inst.tensor_array.append(tensor) + req.insts.append(inst) return req def _unpack_inference_response(self, resp, fetch, is_python, @@ -652,10 +622,17 @@ class MultiLangClient(object): def predict(self, feed, fetch, + batch=True, need_variant_tag=False, asyn=False, is_python=True, log_id=0): + if isinstance(feed, dict) is False: + raise ValueError("Type Error. grpc feed must be dict.") + if batch is False: + for key in feed: + if ".lod" not in key: + feed[key] = feed[key][np.newaxis, :] if not asyn: try: self.profile_.record('py_prepro_0') diff --git a/python/paddle_serving_client/io/__init__.py b/python/paddle_serving_client/io/__init__.py index 5ffa6262ec9187d649c207bf753f3d051cd48778..e6aa9947ca3326d8ff8e2bce012c37bffdb69b8d 100644 --- a/python/paddle_serving_client/io/__init__.py +++ b/python/paddle_serving_client/io/__init__.py @@ -23,7 +23,90 @@ from paddle.fluid.io import save_inference_model import paddle.fluid as fluid from ..proto import general_model_config_pb2 as model_conf import os +import paddle +import paddle.nn.functional as F +from paddle.jit import to_static +def save_dygraph_model(serving_model_folder, client_config_folder, model): + paddle.jit.save(model, "serving_tmp") + loaded_layer = paddle.jit.load(path=".", model_filename="serving_tmp.pdmodel", params_filename="serving_tmp.pdiparams") + feed_target_names = [x.name for x in loaded_layer._input_spec()] + fetch_target_names = [x.name for x in loaded_layer._output_spec()] + + inference_program = loaded_layer.program() + feed_var_dict = { + x: inference_program.global_block().var(x) + for x in feed_target_names + } + fetch_var_dict = { + x: inference_program.global_block().var(x) + for x in fetch_target_names + } + config = model_conf.GeneralModelConfig() + + #int64 = 0; float32 = 1; int32 = 2; + for key in feed_var_dict: + feed_var = model_conf.FeedVar() + feed_var.alias_name = key + feed_var.name = feed_var_dict[key].name + feed_var.is_lod_tensor = feed_var_dict[key].lod_level >= 1 + if feed_var_dict[key].dtype == core.VarDesc.VarType.INT64: + feed_var.feed_type = 0 + if feed_var_dict[key].dtype == core.VarDesc.VarType.FP32: + feed_var.feed_type = 1 + if feed_var_dict[key].dtype == core.VarDesc.VarType.INT32: + feed_var.feed_type = 2 + if feed_var.is_lod_tensor: + feed_var.shape.extend([-1]) + else: + tmp_shape = [] + for v in feed_var_dict[key].shape: + if v >= 0: + tmp_shape.append(v) + feed_var.shape.extend(tmp_shape) + config.feed_var.extend([feed_var]) + for key in fetch_var_dict: + fetch_var = model_conf.FetchVar() + fetch_var.alias_name = key + fetch_var.name = fetch_var_dict[key].name + fetch_var.is_lod_tensor = 1 + if fetch_var_dict[key].dtype == core.VarDesc.VarType.INT64: + fetch_var.fetch_type = 0 + if fetch_var_dict[key].dtype == core.VarDesc.VarType.FP32: + fetch_var.fetch_type = 1 + if fetch_var_dict[key].dtype == core.VarDesc.VarType.INT32: + fetch_var.fetch_type = 2 + if fetch_var.is_lod_tensor: + fetch_var.shape.extend([-1]) + else: + tmp_shape = [] + for v in fetch_var_dict[key].shape: + if v >= 0: + tmp_shape.append(v) + fetch_var.shape.extend(tmp_shape) + config.fetch_var.extend([fetch_var]) + cmd = "mkdir -p {}".format(client_config_folder) + os.system(cmd) + cmd = "mkdir -p {}".format(serving_model_folder) + os.system(cmd) + cmd = "mv {} {}/__model__".format("serving_tmp.pdmodel", serving_model_folder) + os.system(cmd) + cmd = "mv {} {}/__params__".format("serving_tmp.pdiparams", serving_model_folder) + os.system(cmd) + cmd = "rm -rf serving_tmp.pd*" + os.system(cmd) + with open("{}/serving_client_conf.prototxt".format(client_config_folder), + "w") as fout: + fout.write(str(config)) + with open("{}/serving_server_conf.prototxt".format(serving_model_folder), + "w") as fout: + fout.write(str(config)) + with open("{}/serving_client_conf.stream.prototxt".format( + client_config_folder), "wb") as fout: + fout.write(config.SerializeToString()) + with open("{}/serving_server_conf.stream.prototxt".format( + serving_model_folder), "wb") as fout: + fout.write(config.SerializeToString()) def save_model(server_model_folder, client_config_folder, @@ -44,6 +127,8 @@ def save_model(server_model_folder, feed_var_names, target_vars, executor, + model_filename="__model__", + params_filename="__params__", main_program=main_program) config = model_conf.GeneralModelConfig() diff --git a/python/paddle_serving_server/__init__.py b/python/paddle_serving_server/__init__.py index 30f4583a3b785dfe8824a5c14014c5e816fbc27e..a46d0f246cc471b7c98f678b3e87d95e601db774 100644 --- a/python/paddle_serving_server/__init__.py +++ b/python/paddle_serving_server/__init__.py @@ -230,11 +230,15 @@ class Server(object): engine.enable_ir_optimization = self.ir_optimization engine.static_optimization = False engine.force_update_static_cache = False + if os.path.exists('{}/__params__'.format(model_config_path)): + suffix = "" + else: + suffix = "_DIR" if device == "cpu": - engine.type = "FLUID_CPU_ANALYSIS_DIR" + engine.type = "FLUID_CPU_ANALYSIS" + suffix elif device == "gpu": - engine.type = "FLUID_GPU_ANALYSIS_DIR" + engine.type = "FLUID_GPU_ANALYSIS" + suffix self.model_toolkit_conf.engines.extend([engine]) @@ -523,35 +527,26 @@ class MultiLangServerServiceServicer(multi_lang_general_model_service_pb2_grpc. fetch_names = list(request.fetch_var_names) is_python = request.is_python log_id = request.log_id - feed_batch = [] - for feed_inst in request.insts: - feed_dict = {} - for idx, name in enumerate(feed_names): - var = feed_inst.tensor_array[idx] - v_type = self.feed_types_[name] - data = None - if is_python: - if v_type == 0: # int64 - data = np.frombuffer(var.data, dtype="int64") - elif v_type == 1: # float32 - data = np.frombuffer(var.data, dtype="float32") - elif v_type == 2: # int32 - data = np.frombuffer(var.data, dtype="int32") - else: - raise Exception("error type.") + feed_dict = {} + feed_inst = request.insts[0] + for idx, name in enumerate(feed_names): + var = feed_inst.tensor_array[idx] + v_type = self.feed_types_[name] + data = None + if is_python: + if v_type == 0: # int64 + data = np.frombuffer(var.data, dtype="int64") + elif v_type == 1: # float32 + data = np.frombuffer(var.data, dtype="float32") + elif v_type == 2: # int32 + data = np.frombuffer(var.data, dtype="int32") else: - if v_type == 0: # int64 - data = np.array(list(var.int64_data), dtype="int64") - elif v_type == 1: # float32 - data = np.array(list(var.float_data), dtype="float32") - elif v_type == 2: # int32 - data = np.array(list(var.int_data), dtype="int32") - else: - raise Exception("error type.") - data.shape = list(feed_inst.tensor_array[idx].shape) - feed_dict[name] = data - feed_batch.append(feed_dict) - return feed_batch, fetch_names, is_python, log_id + raise Exception("error type.") + data.shape = list(feed_inst.tensor_array[idx].shape) + feed_dict[name] = data + if len(var.lod) > 0: + feed_dict["{}.lod".format()] = var.lod + return feed_dict, fetch_names, is_python, log_id def _pack_inference_response(self, ret, fetch_names, is_python): resp = multi_lang_general_model_service_pb2.InferenceResponse() @@ -608,6 +603,7 @@ class MultiLangServerServiceServicer(multi_lang_general_model_service_pb2_grpc. ret = self.bclient_.predict( feed=feed_dict, fetch=fetch_names, + batch=True, need_variant_tag=True, log_id=log_id) return self._pack_inference_response(ret, fetch_names, is_python) diff --git a/tools/serving_build.sh b/tools/serving_build.sh index 880c509e762131104478ad4b5b39f5e11ded0656..6bc142c36efad60ec26f7dac6200c3127aef8252 100644 --- a/tools/serving_build.sh +++ b/tools/serving_build.sh @@ -174,7 +174,7 @@ function python_test_fit_a_line() { # test web unsetproxy # maybe the proxy is used on iPipe, which makes web-test failed. - check_cmd "python -m paddle_serving_server.serve --model uci_housing_model --name uci --port 9393 --thread 4 --name uci > /dev/null &" + check_cmd "python test_server.py > /dev/null &" sleep 5 # wait for the server to start check_cmd "curl -H \"Content-Type:application/json\" -X POST -d '{\"feed\":[{\"x\": [0.0137, -0.1136, 0.2553, -0.0692, 0.0582, -0.0727, -0.1583, -0.0584, 0.6283, 0.4919, 0.1856, 0.0795, -0.0332]}], \"fetch\":[\"price\"]}' http://127.0.0.1:9393/uci/prediction" # check http code @@ -183,14 +183,6 @@ function python_test_fit_a_line() { echo "HTTP status code -ne 200" exit 1 fi - # test web batch - check_cmd "curl -H \"Content-Type:application/json\" -X POST -d '{\"feed\":[{\"x\": [0.0137, -0.1136, 0.2553, -0.0692, 0.0582, -0.0727, -0.1583, -0.0584, 0.6283, 0.4919, 0.1856, 0.0795, -0.0332]}, {\"x\": [0.0137, -0.1136, 0.2553, -0.0692, 0.0582, -0.0727, -0.1583, -0.0584, 0.6283, 0.4919, 0.1856, 0.0795, -0.0332]}], \"fetch\":[\"price\"]}' http://127.0.0.1:9393/uci/prediction" - # check http code - http_code=`curl -H "Content-Type:application/json" -X POST -d '{"feed":[{"x": [0.0137, -0.1136, 0.2553, -0.0692, 0.0582, -0.0727, -0.1583, -0.0584, 0.6283, 0.4919, 0.1856, 0.0795, -0.0332]}, {"x": [0.0137, -0.1136, 0.2553, -0.0692, 0.0582, -0.0727, -0.1583, -0.0584, 0.6283, 0.4919, 0.1856, 0.0795, -0.0332]}], "fetch":["price"]}' -s -w "%{http_code}" -o /dev/null http://127.0.0.1:9393/uci/prediction` - if [ ${http_code} -ne 200 ]; then - echo "HTTP status code -ne 200" - exit 1 - fi setproxy # recover proxy state kill_server_process ;; @@ -202,27 +194,6 @@ function python_test_fit_a_line() { check_cmd "python test_client.py uci_housing_client/serving_client_conf.prototxt > /dev/null" kill_server_process - # test web - #unsetproxy # maybe the proxy is used on iPipe, which makes web-test failed. - #check_cmd "python -m paddle_serving_server_gpu.serve --model uci_housing_model --port 9393 --thread 2 --gpu_ids 0 --name uci > /dev/null &" - #sleep 5 # wait for the server to start - #check_cmd "curl -H \"Content-Type:application/json\" -X POST -d '{\"feed\":[{\"x\": [0.0137, -0.1136, 0.2553, -0.0692, 0.0582, -0.0727, -0.1583, -0.0584, 0.6283, 0.4919, 0.1856, 0.0795, -0.0332]}], \"fetch\":[\"price\"]}' http://127.0.0.1:9393/uci/prediction" - # check http code - #http_code=`curl -H "Content-Type:application/json" -X POST -d '{"feed":[{"x": [0.0137, -0.1136, 0.2553, -0.0692, 0.0582, -0.0727, -0.1583, -0.0584, 0.6283, 0.4919, 0.1856, 0.0795, -0.0332]}], "fetch":["price"]}' -s -w "%{http_code}" -o /dev/null http://127.0.0.1:9393/uci/prediction` - #if [ ${http_code} -ne 200 ]; then - # echo "HTTP status code -ne 200" - # exit 1 - #fi - # test web batch - #check_cmd "curl -H \"Content-Type:application/json\" -X POST -d '{\"feed\":[{\"x\": [0.0137, -0.1136, 0.2553, -0.0692, 0.0582, -0.0727, -0.1583, -0.0584, 0.6283, 0.4919, 0.1856, 0.0795, -0.0332]}, {\"x\": [0.0137, -0.1136, 0.2553, -0.0692, 0.0582, -0.0727, -0.1583, -0.0584, 0.6283, 0.4919, 0.1856, 0.0795, -0.0332]}], \"fetch\":[\"price\"]}' http://127.0.0.1:9393/uci/prediction" - # check http code - #http_code=`curl -H "Content-Type:application/json" -X POST -d '{"feed":[{"x": [0.0137, -0.1136, 0.2553, -0.0692, 0.0582, -0.0727, -0.1583, -0.0584, 0.6283, 0.4919, 0.1856, 0.0795, -0.0332]}, {"x": [0.0137, -0.1136, 0.2553, -0.0692, 0.0582, -0.0727, -0.1583, -0.0584, 0.6283, 0.4919, 0.1856, 0.0795, -0.0332]}], "fetch":["price"]}' -s -w "%{http_code}" -o /dev/null http://127.0.0.1:9393/uci/prediction` - #if [ ${http_code} -ne 200 ]; then - # echo "HTTP status code -ne 200" - # exit 1 - #fi - #setproxy # recover proxy state - #kill_server_process ;; *) echo "error type" @@ -589,9 +560,6 @@ function python_test_grpc_impl() { sleep 5 # wait for the server to start check_cmd "python test_sync_client.py > /dev/null" check_cmd "python test_asyn_client.py > /dev/null" - check_cmd "python test_general_pb_client.py > /dev/null" - check_cmd "python test_numpy_input_client.py > /dev/null" - check_cmd "python test_batch_client.py > /dev/null" check_cmd "python test_timeout_client.py > /dev/null" kill_server_process kill_process_by_port 9393 @@ -600,9 +568,6 @@ function python_test_grpc_impl() { sleep 5 # wait for the server to start check_cmd "python test_sync_client.py > /dev/null" check_cmd "python test_asyn_client.py > /dev/null" - check_cmd "python test_general_pb_client.py > /dev/null" - check_cmd "python test_numpy_input_client.py > /dev/null" - check_cmd "python test_batch_client.py > /dev/null" check_cmd "python test_timeout_client.py > /dev/null" kill_server_process kill_process_by_port 9393 @@ -651,9 +616,7 @@ COMMENT sleep 5 # wait for the server to start check_cmd "python test_sync_client.py > /dev/null" check_cmd "python test_asyn_client.py > /dev/null" - check_cmd "python test_general_pb_client.py > /dev/null" - check_cmd "python test_numpy_input_client.py > /dev/null" - check_cmd "python test_batch_client.py > /dev/null" + #check_cmd "python test_batch_client.py > /dev/null" check_cmd "python test_timeout_client.py > /dev/null" kill_server_process kill_process_by_port 9393 @@ -662,9 +625,7 @@ COMMENT sleep 5 # wait for the server to start check_cmd "python test_sync_client.py > /dev/null" check_cmd "python test_asyn_client.py > /dev/null" - check_cmd "python test_general_pb_client.py > /dev/null" - check_cmd "python test_numpy_input_client.py > /dev/null" - check_cmd "python test_batch_client.py > /dev/null" + #check_cmd "python test_batch_client.py > /dev/null" check_cmd "python test_timeout_client.py > /dev/null" kill_server_process kill_process_by_port 9393