From c49f4bce791cf1afacbc94cb1f5dbdd3412493c1 Mon Sep 17 00:00:00 2001 From: zhangjun Date: Wed, 10 Mar 2021 09:59:57 +0000 Subject: [PATCH] fix and delete inferencer code --- core/general-server/CMakeLists.txt | 13 +- core/predictor/common/utils.h | 3 +- core/predictor/framework/infer.h | 8 +- .../inferencer-fluid-arm/CMakeLists.txt | 10 - .../include/fluid_arm_engine.h | 291 --------------- .../src/fluid_arm_engine.cpp | 35 -- .../inferencer-fluid-cpu/CMakeLists.txt | 10 - .../include/fluid_cpu_engine.h | 323 ----------------- .../src/fluid_cpu_engine.cpp | 42 --- .../inferencer-fluid-gpu/CMakeLists.txt | 11 - .../include/fluid_gpu_engine.h | 342 ------------------ .../src/fluid_gpu_engine.cpp | 42 --- .../paddle/include/paddle_engine.h | 17 +- paddle_inference/paddle/src/paddle_engine.cpp | 2 + 14 files changed, 26 insertions(+), 1123 deletions(-) delete mode 100644 paddle_inference/inferencer-fluid-arm/CMakeLists.txt delete mode 100644 paddle_inference/inferencer-fluid-arm/include/fluid_arm_engine.h delete mode 100644 paddle_inference/inferencer-fluid-arm/src/fluid_arm_engine.cpp delete mode 100644 paddle_inference/inferencer-fluid-cpu/CMakeLists.txt delete mode 100644 paddle_inference/inferencer-fluid-cpu/include/fluid_cpu_engine.h delete mode 100644 paddle_inference/inferencer-fluid-cpu/src/fluid_cpu_engine.cpp delete mode 100644 paddle_inference/inferencer-fluid-gpu/CMakeLists.txt delete mode 100644 paddle_inference/inferencer-fluid-gpu/include/fluid_gpu_engine.h delete mode 100644 paddle_inference/inferencer-fluid-gpu/src/fluid_gpu_engine.cpp diff --git a/core/general-server/CMakeLists.txt b/core/general-server/CMakeLists.txt index 48f154b0..88a5455d 100644 --- a/core/general-server/CMakeLists.txt +++ b/core/general-server/CMakeLists.txt @@ -2,13 +2,14 @@ include_directories(SYSTEM ${CMAKE_CURRENT_LIST_DIR}/../../) include(op/CMakeLists.txt) include(proto/CMakeLists.txt) add_executable(serving ${serving_srcs}) -add_dependencies(serving pdcodegen fluid_cpu_engine pdserving paddle_fluid cube-api utils) +add_dependencies(serving pdcodegen paddle_inference_engine pdserving paddle_fluid cube-api utils) + if (WITH_GPU) - add_dependencies(serving fluid_gpu_engine) + add_dependencies(serving paddle_inference_engine) endif() if (WITH_LITE) - add_dependencies(serving fluid_arm_engine) + add_dependencies(serving paddle_inference_engine) endif() target_include_directories(serving PUBLIC @@ -16,16 +17,16 @@ target_include_directories(serving PUBLIC ) include_directories(${CUDNN_ROOT}/include/) if(WITH_GPU) - target_link_libraries(serving -Wl,--whole-archive fluid_gpu_engine + target_link_libraries(serving -Wl,--whole-archive paddle_infence_engine -Wl,--no-whole-archive) endif() if(WITH_LITE) - target_link_libraries(serving -Wl,--whole-archive fluid_arm_engine + target_link_libraries(serving -Wl,--whole-archive paddle_inference_engine -Wl,--no-whole-archive) endif() -target_link_libraries(serving -Wl,--whole-archive fluid_cpu_engine +target_link_libraries(serving -Wl,--whole-archive paddle_inference_engine -Wl,--no-whole-archive) target_link_libraries(serving paddle_fluid ${paddle_depend_libs}) diff --git a/core/predictor/common/utils.h b/core/predictor/common/utils.h index 989b0951..052f90b1 100644 --- a/core/predictor/common/utils.h +++ b/core/predictor/common/utils.h @@ -14,6 +14,7 @@ #pragma once #include +#include #include "core/predictor/common/inner_common.h" #include "core/predictor/common/macros.h" @@ -148,7 +149,7 @@ class IsDerivedFrom { } }; -void ReadBinaryFile(const std::string& filename, std::string* contents) { +static void ReadBinaryFile(const std::string& filename, std::string* contents) { std::ifstream fin(filename, std::ios::in | std::ios::binary); fin.seekg(0, std::ios::end); contents->clear(); diff --git a/core/predictor/framework/infer.h b/core/predictor/framework/infer.h index 6c0e98e6..fcd3038c 100644 --- a/core/predictor/framework/infer.h +++ b/core/predictor/framework/infer.h @@ -104,7 +104,7 @@ class ReloadableInferEngine : public InferEngine { int proc_initialize_impl(const configure::EngineDesc& conf, bool version) { _reload_tag_file = conf.reloadable_meta(); _reload_mode_tag = conf.reloadable_type(); - _model_data_path = conf.model_data_path(); + _model_data_path = conf.model_dir(); _infer_thread_num = conf.runtime_thread_num(); _infer_batch_size = conf.batch_infer_size(); _infer_batch_align = conf.enable_batch_align(); @@ -162,7 +162,7 @@ class ReloadableInferEngine : public InferEngine { int reload() { if (check_need_reload()) { LOG(WARNING) << "begin reload model[" << _model_data_path << "]."; - return load(_infer_engine_params); + return load(_conf); } return 0; } @@ -310,7 +310,7 @@ class DBReloadableInferEngine : public ReloadableInferEngine { ModelData* md = new (std::nothrow) ModelData; if (!md || load_data(md, _conf) != 0) { LOG(ERROR) << "Failed create thread data from " - << _infer_engine_params.model_dir(); + << _conf.model_dir(); return -1; } @@ -370,7 +370,7 @@ class CloneDBReloadableInferEngine // 加载进程级模型数据 if (!_pd || DBReloadableInferEngine::load_data(_pd, conf) != 0) { - LOG(ERROR) << "Failed to create common model from [" << params.model_dir() + LOG(ERROR) << "Failed to create common model from [" << conf.model_dir() << "]."; return -1; } diff --git a/paddle_inference/inferencer-fluid-arm/CMakeLists.txt b/paddle_inference/inferencer-fluid-arm/CMakeLists.txt deleted file mode 100644 index cf415d9e..00000000 --- a/paddle_inference/inferencer-fluid-arm/CMakeLists.txt +++ /dev/null @@ -1,10 +0,0 @@ -FILE(GLOB fluid_arm_engine_srcs ${CMAKE_CURRENT_LIST_DIR}/src/*.cpp) -add_library(fluid_arm_engine ${fluid_arm_engine_srcs}) -target_include_directories(fluid_arm_engine PUBLIC - ${CMAKE_BINARY_DIR}/Paddle/fluid_install_dir/) -add_dependencies(fluid_arm_engine pdserving extern_paddle configure) -target_link_libraries(fluid_arm_engine pdserving paddle_fluid -lpthread -lcrypto -lm -lrt -lssl -ldl -lz) - -install(TARGETS fluid_arm_engine - ARCHIVE DESTINATION ${PADDLE_SERVING_INSTALL_DIR}/lib - ) diff --git a/paddle_inference/inferencer-fluid-arm/include/fluid_arm_engine.h b/paddle_inference/inferencer-fluid-arm/include/fluid_arm_engine.h deleted file mode 100644 index b3db6e1a..00000000 --- a/paddle_inference/inferencer-fluid-arm/include/fluid_arm_engine.h +++ /dev/null @@ -1,291 +0,0 @@ -// Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#pragma once - -#include -#include -#include -#include -#include -#include "core/configure/include/configure_parser.h" -#include "core/configure/inferencer_configure.pb.h" -#include "core/predictor/framework/infer.h" -#include "paddle_inference_api.h" // NOLINT - -namespace baidu { -namespace paddle_serving { -namespace fluid_arm { - -class AutoLock { - public: - explicit AutoLock(pthread_mutex_t& mutex) : _mut(mutex) { - pthread_mutex_lock(&mutex); - } - - ~AutoLock() { pthread_mutex_unlock(&_mut); } - - private: - pthread_mutex_t& _mut; -}; - -class GlobalPaddleCreateMutex { - public: - pthread_mutex_t& mutex() { return _mut; } - - static pthread_mutex_t& instance() { - static GlobalPaddleCreateMutex gmutex; - return gmutex.mutex(); - } - - private: - GlobalPaddleCreateMutex() { pthread_mutex_init(&_mut, NULL); } - - pthread_mutex_t _mut; -}; - -using paddle_infer::Config; -using paddle_infer::Predictor; -using paddle_infer::Tensor; -using paddle_infer::PrecisionType; -using paddle_infer::CreatePredictor; - -// data interface -class FluidFamilyCore { - public: - virtual ~FluidFamilyCore() {} - virtual std::vector GetInputNames() { - return _core->GetInputNames(); - } - - virtual std::unique_ptr GetInputHandle(const std::string& name) { - return _core->GetInputHandle(name); - } - - virtual std::vector GetOutputNames() { - return _core->GetOutputNames(); - } - - virtual std::unique_ptr GetOutputHandle(const std::string& name) { - return _core->GetOutputHandle(name); - } - - virtual bool Run() { - if (!_core->Run()) { - LOG(ERROR) << "Failed call Run with paddle predictor"; - return false; - } - return true; - } - - virtual int create(const predictor::InferEngineCreationParams& params) = 0; - - virtual int clone(void* origin_core) { - if (origin_core == NULL) { - LOG(ERROR) << "origin paddle Predictor is null."; - return -1; - } - Predictor* p_predictor = (Predictor*)origin_core; - _core = p_predictor->Clone(); - if (_core.get() == NULL) { - LOG(ERROR) << "fail to clone paddle predictor: " << origin_core; - return -1; - } - return 0; - } - - virtual void* get() { return _core.get(); } - - protected: - std::shared_ptr _core; -}; - -// infer interface -class FluidArmAnalysisCore : public FluidFamilyCore { - public: - int create(const predictor::InferEngineCreationParams& params) { - std::string data_path = params.get_path(); - if (access(data_path.c_str(), F_OK) == -1) { - LOG(ERROR) << "create paddle predictor failed, path not exits: " - << data_path; - return -1; - } - - Config config; - config.SetParamsFile(data_path + "/__params__"); - config.SetProgFile(data_path + "/__model__"); - config.DisableGpu(); - config.SetCpuMathLibraryNumThreads(1); - - if (params.use_lite()) { - config.EnableLiteEngine(PrecisionType::kFloat32, true); - } - - if (params.use_xpu()) { - config.EnableXpu(2 * 1024 * 1024); - } - - if (params.enable_memory_optimization()) { - config.EnableMemoryOptim(); - } - - if (params.enable_ir_optimization()) { - config.SwitchIrOptim(true); - } else { - config.SwitchIrOptim(false); - } - - config.SwitchSpecifyInputNames(true); - AutoLock lock(GlobalPaddleCreateMutex::instance()); - _core = CreatePredictor(config); - if (NULL == _core.get()) { - LOG(ERROR) << "create paddle predictor failed, path: " << data_path; - return -1; - } - - VLOG(2) << "create paddle predictor sucess, path: " << data_path; - return 0; - } -}; - -class FluidArmAnalysisDirCore : public FluidFamilyCore { - public: - int create(const predictor::InferEngineCreationParams& params) { - std::string data_path = params.get_path(); - if (access(data_path.c_str(), F_OK) == -1) { - LOG(ERROR) << "create paddle predictor failed, path not exits: " - << data_path; - return -1; - } - - Config config; - config.SetModel(data_path); - config.DisableGpu(); - config.SwitchSpecifyInputNames(true); - config.SetCpuMathLibraryNumThreads(1); - - if (params.use_lite()) { - config.EnableLiteEngine(PrecisionType::kFloat32, true); - } - - if (params.use_xpu()) { - config.EnableXpu(2 * 1024 * 1024); - } - - if (params.enable_memory_optimization()) { - config.EnableMemoryOptim(); - } - - if (params.enable_ir_optimization()) { - config.SwitchIrOptim(true); - } else { - config.SwitchIrOptim(false); - } - - AutoLock lock(GlobalPaddleCreateMutex::instance()); - _core = CreatePredictor(config); - if (NULL == _core.get()) { - LOG(ERROR) << "create paddle predictor failed, path: " << data_path; - return -1; - } - - VLOG(2) << "create paddle predictor sucess, path: " << data_path; - return 0; - } -}; - -class Parameter { - public: - Parameter() : _row(0), _col(0), _params(NULL) {} - ~Parameter() { - VLOG(2) << "before destroy Parameter, file_name[" << _file_name << "]"; - destroy(); - } - - int init(int row, int col, const char* file_name) { - destroy(); - _file_name = file_name; - _row = row; - _col = col; - _params = reinterpret_cast(malloc(_row * _col * sizeof(float))); - if (_params == NULL) { - LOG(ERROR) << "Load " << _file_name << " malloc error."; - return -1; - } - VLOG(2) << "Load parameter file[" << _file_name << "] success."; - return 0; - } - - void destroy() { - _row = 0; - _col = 0; - if (_params != NULL) { - free(_params); - _params = NULL; - } - } - - int load() { - if (_params == NULL || _row <= 0 || _col <= 0) { - LOG(ERROR) << "load parameter error [not inited]."; - return -1; - } - - FILE* fs = fopen(_file_name.c_str(), "rb"); - if (fs == NULL) { - LOG(ERROR) << "load " << _file_name << " fopen error."; - return -1; - } - static const uint32_t MODEL_FILE_HEAD_LEN = 16; - char head[MODEL_FILE_HEAD_LEN] = {0}; - if (fread(head, 1, MODEL_FILE_HEAD_LEN, fs) != MODEL_FILE_HEAD_LEN) { - destroy(); - LOG(ERROR) << "Load " << _file_name << " read head error."; - if (fs != NULL) { - fclose(fs); - fs = NULL; - } - return -1; - } - - uint32_t matrix_size = _row * _col; - if (matrix_size == fread(_params, sizeof(float), matrix_size, fs)) { - if (fs != NULL) { - fclose(fs); - fs = NULL; - } - VLOG(2) << "load " << _file_name << " read ok."; - return 0; - } else { - LOG(ERROR) << "load " << _file_name << " read error."; - destroy(); - if (fs != NULL) { - fclose(fs); - fs = NULL; - } - return -1; - } - return 0; - } - - public: - std::string _file_name; - int _row; - int _col; - float* _params; -}; - -} // namespace fluid_arm -} // namespace paddle_serving -} // namespace baidu diff --git a/paddle_inference/inferencer-fluid-arm/src/fluid_arm_engine.cpp b/paddle_inference/inferencer-fluid-arm/src/fluid_arm_engine.cpp deleted file mode 100644 index 2c853c63..00000000 --- a/paddle_inference/inferencer-fluid-arm/src/fluid_arm_engine.cpp +++ /dev/null @@ -1,35 +0,0 @@ -// Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "paddle_inference/inferencer-fluid-arm/include/fluid_arm_engine.h" -#include "core/predictor/framework/factory.h" - -namespace baidu { -namespace paddle_serving { -namespace fluid_arm { - -REGIST_FACTORY_OBJECT_IMPL_WITH_NAME( - ::baidu::paddle_serving::predictor::FluidInferEngine, - ::baidu::paddle_serving::predictor::InferEngine, - "FLUID_ARM_ANALYSIS"); - -REGIST_FACTORY_OBJECT_IMPL_WITH_NAME( - ::baidu::paddle_serving::predictor::FluidInferEngine< - FluidArmAnalysisDirCore>, - ::baidu::paddle_serving::predictor::InferEngine, - "FLUID_ARM_ANALYSIS_DIR"); - -} // namespace fluid_arm -} // namespace paddle_serving -} // namespace baidu diff --git a/paddle_inference/inferencer-fluid-cpu/CMakeLists.txt b/paddle_inference/inferencer-fluid-cpu/CMakeLists.txt deleted file mode 100644 index fe7cd91c..00000000 --- a/paddle_inference/inferencer-fluid-cpu/CMakeLists.txt +++ /dev/null @@ -1,10 +0,0 @@ -FILE(GLOB fluid_cpu_engine_srcs ${CMAKE_CURRENT_LIST_DIR}/src/*.cpp) -add_library(fluid_cpu_engine ${fluid_cpu_engine_srcs}) -target_include_directories(fluid_cpu_engine PUBLIC - ${CMAKE_BINARY_DIR}/Paddle/fluid_install_dir/) -add_dependencies(fluid_cpu_engine pdserving extern_paddle configure) -target_link_libraries(fluid_cpu_engine pdserving paddle_fluid -lpthread -lcrypto -lm -lrt -lssl -ldl -lz) - -install(TARGETS fluid_cpu_engine - ARCHIVE DESTINATION ${PADDLE_SERVING_INSTALL_DIR}/lib - ) diff --git a/paddle_inference/inferencer-fluid-cpu/include/fluid_cpu_engine.h b/paddle_inference/inferencer-fluid-cpu/include/fluid_cpu_engine.h deleted file mode 100644 index 681f2fe2..00000000 --- a/paddle_inference/inferencer-fluid-cpu/include/fluid_cpu_engine.h +++ /dev/null @@ -1,323 +0,0 @@ -// Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#pragma once - -#include -#include -#include -#include -#include -#include "core/configure/include/configure_parser.h" -#include "core/configure/inferencer_configure.pb.h" -#include "core/predictor/framework/infer.h" -#include "paddle_inference_api.h" // NOLINT - -namespace baidu { -namespace paddle_serving { -namespace fluid_cpu { - -class AutoLock { - public: - explicit AutoLock(pthread_mutex_t& mutex) : _mut(mutex) { - pthread_mutex_lock(&mutex); - } - - ~AutoLock() { pthread_mutex_unlock(&_mut); } - - private: - pthread_mutex_t& _mut; -}; - -class GlobalPaddleCreateMutex { - public: - pthread_mutex_t& mutex() { return _mut; } - - static pthread_mutex_t& instance() { - static GlobalPaddleCreateMutex gmutex; - return gmutex.mutex(); - } - - private: - GlobalPaddleCreateMutex() { pthread_mutex_init(&_mut, NULL); } - - pthread_mutex_t _mut; -}; - -using paddle_infer::Config; -using paddle_infer::Predictor; -using paddle_infer::Tensor; -using paddle_infer::CreatePredictor; - -// data interface -class FluidFamilyCore { - public: - virtual ~FluidFamilyCore() {} - virtual std::vector GetInputNames() { - return _core->GetInputNames(); - } - - virtual std::unique_ptr GetInputHandle(const std::string& name) { - return _core->GetInputHandle(name); - } - - virtual std::vector GetOutputNames() { - return _core->GetOutputNames(); - } - - virtual std::unique_ptr GetOutputHandle(const std::string& name) { - return _core->GetOutputHandle(name); - } - - virtual bool Run() { - if (!_core->Run()) { - LOG(ERROR) << "Failed call Run with paddle predictor"; - return false; - } - return true; - } - - virtual int create(const predictor::InferEngineCreationParams& params) = 0; - - virtual int clone(void* origin_core) { - if (origin_core == NULL) { - LOG(ERROR) << "origin paddle Predictor is null."; - return -1; - } - Predictor* p_predictor = (Predictor*)origin_core; - _core = p_predictor->Clone(); - if (_core.get() == NULL) { - LOG(ERROR) << "fail to clone paddle predictor: " << origin_core; - return -1; - } - return 0; - } - - virtual void* get() { return _core.get(); } - - protected: - std::shared_ptr _core; -}; - -// infer interface -class FluidCpuAnalysisCore : public FluidFamilyCore { - public: - int create(const predictor::InferEngineCreationParams& params) { - std::string data_path = params.get_path(); - if (access(data_path.c_str(), F_OK) == -1) { - LOG(ERROR) << "create paddle predictor failed, path not exits: " - << data_path; - return -1; - } - - Config config; - config.SetParamsFile(data_path + "/__params__"); - config.SetProgFile(data_path + "/__model__"); - config.DisableGpu(); - config.SetCpuMathLibraryNumThreads(1); - - if (params.enable_memory_optimization()) { - config.EnableMemoryOptim(); - } - - config.SwitchSpecifyInputNames(true); - AutoLock lock(GlobalPaddleCreateMutex::instance()); - _core = CreatePredictor(config); - if (NULL == _core.get()) { - LOG(ERROR) << "create paddle predictor failed, path: " << data_path; - return -1; - } - - VLOG(2) << "create paddle predictor sucess, path: " << data_path; - return 0; - } -}; - -class FluidCpuAnalysisDirCore : public FluidFamilyCore { - public: - int create(const predictor::InferEngineCreationParams& params) { - std::string data_path = params.get_path(); - if (access(data_path.c_str(), F_OK) == -1) { - LOG(ERROR) << "create paddle predictor failed, path not exits: " - << data_path; - return -1; - } - - Config config; - config.SetModel(data_path); - config.DisableGpu(); - config.SwitchSpecifyInputNames(true); - config.SetCpuMathLibraryNumThreads(1); - - if (params.enable_memory_optimization()) { - config.EnableMemoryOptim(); - } - - if (params.enable_ir_optimization()) { - config.SwitchIrOptim(true); - } else { - config.SwitchIrOptim(false); - } - - AutoLock lock(GlobalPaddleCreateMutex::instance()); - _core = CreatePredictor(config); - if (NULL == _core.get()) { - LOG(ERROR) << "create paddle predictor failed, path: " << data_path; - return -1; - } - - VLOG(2) << "create paddle predictor sucess, path: " << data_path; - return 0; - } -}; - -class Parameter { - public: - Parameter() : _row(0), _col(0), _params(NULL) {} - ~Parameter() { - VLOG(2) << "before destroy Parameter, file_name[" << _file_name << "]"; - destroy(); - } - - int init(int row, int col, const char* file_name) { - destroy(); - _file_name = file_name; - _row = row; - _col = col; - _params = reinterpret_cast(malloc(_row * _col * sizeof(float))); - if (_params == NULL) { - LOG(ERROR) << "Load " << _file_name << " malloc error."; - return -1; - } - VLOG(2) << "Load parameter file[" << _file_name << "] success."; - return 0; - } - - void destroy() { - _row = 0; - _col = 0; - if (_params != NULL) { - free(_params); - _params = NULL; - } - } - - int load() { - if (_params == NULL || _row <= 0 || _col <= 0) { - LOG(ERROR) << "load parameter error [not inited]."; - return -1; - } - - FILE* fs = fopen(_file_name.c_str(), "rb"); - if (fs == NULL) { - LOG(ERROR) << "load " << _file_name << " fopen error."; - return -1; - } - static const uint32_t MODEL_FILE_HEAD_LEN = 16; - char head[MODEL_FILE_HEAD_LEN] = {0}; - if (fread(head, 1, MODEL_FILE_HEAD_LEN, fs) != MODEL_FILE_HEAD_LEN) { - destroy(); - LOG(ERROR) << "Load " << _file_name << " read head error."; - if (fs != NULL) { - fclose(fs); - fs = NULL; - } - return -1; - } - - uint32_t matrix_size = _row * _col; - if (matrix_size == fread(_params, sizeof(float), matrix_size, fs)) { - if (fs != NULL) { - fclose(fs); - fs = NULL; - } - VLOG(2) << "load " << _file_name << " read ok."; - return 0; - } else { - LOG(ERROR) << "load " << _file_name << " read error."; - destroy(); - if (fs != NULL) { - fclose(fs); - fs = NULL; - } - return -1; - } - return 0; - } - - public: - std::string _file_name; - int _row; - int _col; - float* _params; -}; - -class FluidCpuAnalysisEncryptCore : public FluidFamilyCore { - public: - void ReadBinaryFile(const std::string& filename, std::string* contents) { - std::ifstream fin(filename, std::ios::in | std::ios::binary); - fin.seekg(0, std::ios::end); - contents->clear(); - contents->resize(fin.tellg()); - fin.seekg(0, std::ios::beg); - fin.read(&(contents->at(0)), contents->size()); - fin.close(); - } - - int create(const predictor::InferEngineCreationParams& params) { - std::string data_path = params.get_path(); - if (access(data_path.c_str(), F_OK) == -1) { - LOG(ERROR) << "create paddle predictor failed, path note exits: " - << data_path; - return -1; - } - - std::string model_buffer, params_buffer, key_buffer; - ReadBinaryFile(data_path + "encrypt_model", &model_buffer); - ReadBinaryFile(data_path + "encrypt_params", ¶ms_buffer); - ReadBinaryFile(data_path + "key", &key_buffer); - - VLOG(2) << "prepare for encryption model"; - - auto cipher = paddle::MakeCipher(""); - std::string real_model_buffer = cipher->Decrypt(model_buffer, key_buffer); - std::string real_params_buffer = cipher->Decrypt(params_buffer, key_buffer); - - Config analysis_config; - // paddle::AnalysisConfig analysis_config; - analysis_config.SetModelBuffer(&real_model_buffer[0], - real_model_buffer.size(), - &real_params_buffer[0], - real_params_buffer.size()); - analysis_config.DisableGpu(); - analysis_config.SetCpuMathLibraryNumThreads(1); - if (params.enable_memory_optimization()) { - analysis_config.EnableMemoryOptim(); - } - analysis_config.SwitchSpecifyInputNames(true); - AutoLock lock(GlobalPaddleCreateMutex::instance()); - VLOG(2) << "decrypt model file sucess"; - _core = CreatePredictor(analysis_config); - if (NULL == _core.get()) { - LOG(ERROR) << "create paddle predictor failed, path: " << data_path; - return -1; - } - VLOG(2) << "create paddle predictor sucess, path: " << data_path; - return 0; - } -}; - -} // namespace fluid_cpu -} // namespace paddle_serving -} // namespace baidu diff --git a/paddle_inference/inferencer-fluid-cpu/src/fluid_cpu_engine.cpp b/paddle_inference/inferencer-fluid-cpu/src/fluid_cpu_engine.cpp deleted file mode 100644 index f8cf24ef..00000000 --- a/paddle_inference/inferencer-fluid-cpu/src/fluid_cpu_engine.cpp +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "paddle_inference/inferencer-fluid-cpu/include/fluid_cpu_engine.h" -#include "core/predictor/framework/factory.h" - -namespace baidu { -namespace paddle_serving { -namespace fluid_cpu { - -REGIST_FACTORY_OBJECT_IMPL_WITH_NAME( - ::baidu::paddle_serving::predictor::FluidInferEngine, - ::baidu::paddle_serving::predictor::InferEngine, - "FLUID_CPU_ANALYSIS"); - -REGIST_FACTORY_OBJECT_IMPL_WITH_NAME( - ::baidu::paddle_serving::predictor::FluidInferEngine< - FluidCpuAnalysisDirCore>, - ::baidu::paddle_serving::predictor::InferEngine, - "FLUID_CPU_ANALYSIS_DIR"); - -#if 1 -REGIST_FACTORY_OBJECT_IMPL_WITH_NAME( - ::baidu::paddle_serving::predictor::FluidInferEngine< - FluidCpuAnalysisEncryptCore>, - ::baidu::paddle_serving::predictor::InferEngine, - "FLUID_CPU_ANALYSIS_ENCRYPT"); -#endif -} // namespace fluid_cpu -} // namespace paddle_serving -} // namespace baidu diff --git a/paddle_inference/inferencer-fluid-gpu/CMakeLists.txt b/paddle_inference/inferencer-fluid-gpu/CMakeLists.txt deleted file mode 100644 index 6ba3ddd6..00000000 --- a/paddle_inference/inferencer-fluid-gpu/CMakeLists.txt +++ /dev/null @@ -1,11 +0,0 @@ -FILE(GLOB fluid_gpu_engine_srcs ${CMAKE_CURRENT_LIST_DIR}/src/*.cpp) -add_library(fluid_gpu_engine ${fluid_gpu_engine_srcs}) -target_include_directories(fluid_gpu_engine PUBLIC - ${CMAKE_BINARY_DIR}/Paddle/fluid_install_dir/) - -add_dependencies(fluid_gpu_engine pdserving extern_paddle configure) -target_link_libraries(fluid_gpu_engine pdserving paddle_fluid iomp5 mklml_intel -lpthread -lcrypto -lm -lrt -lssl -ldl -lz) - -install(TARGETS fluid_gpu_engine - ARCHIVE DESTINATION ${PADDLE_SERVING_INSTALL_DIR}/lib - ) diff --git a/paddle_inference/inferencer-fluid-gpu/include/fluid_gpu_engine.h b/paddle_inference/inferencer-fluid-gpu/include/fluid_gpu_engine.h deleted file mode 100644 index d3f63f72..00000000 --- a/paddle_inference/inferencer-fluid-gpu/include/fluid_gpu_engine.h +++ /dev/null @@ -1,342 +0,0 @@ -// Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#pragma once - -#include -#include -#include -#include -#include -#include -#include -#include "core/configure/include/configure_parser.h" -#include "core/configure/inferencer_configure.pb.h" -#include "core/predictor/framework/infer.h" -#include "paddle_inference_api.h" // NOLINT - -DECLARE_int32(gpuid); - -namespace baidu { -namespace paddle_serving { -namespace fluid_gpu { - -using configure::SigmoidConf; - -class AutoLock { - public: - explicit AutoLock(pthread_mutex_t& mutex) : _mut(mutex) { - pthread_mutex_lock(&mutex); - } - - ~AutoLock() { pthread_mutex_unlock(&_mut); } - - private: - pthread_mutex_t& _mut; -}; - -class GlobalPaddleCreateMutex { - public: - pthread_mutex_t& mutex() { return _mut; } - - static pthread_mutex_t& instance() { - static GlobalPaddleCreateMutex gmutex; - return gmutex.mutex(); - } - - private: - GlobalPaddleCreateMutex() { pthread_mutex_init(&_mut, NULL); } - - pthread_mutex_t _mut; -}; - -using paddle_infer::Config; -using paddle_infer::Predictor; -using paddle_infer::Tensor; -using paddle_infer::CreatePredictor; - -// data interface -class FluidFamilyCore { - public: - virtual ~FluidFamilyCore() {} - virtual std::vector GetInputNames() { - return _core->GetInputNames(); - } - - virtual std::unique_ptr GetInputHandle(const std::string& name) { - return _core->GetInputHandle(name); - } - - virtual std::vector GetOutputNames() { - return _core->GetOutputNames(); - } - - virtual std::unique_ptr GetOutputHandle(const std::string& name) { - return _core->GetOutputHandle(name); - } - - virtual bool Run() { - if (!_core->Run()) { - LOG(ERROR) << "Failed call Run with paddle predictor"; - return false; - } - return true; - } - - virtual int create(const predictor::InferEngineCreationParams& params) = 0; - - virtual int clone(void* origin_core) { - if (origin_core == NULL) { - LOG(ERROR) << "origin paddle Predictor is null."; - return -1; - } - Predictor* p_predictor = (Predictor*)origin_core; - _core = p_predictor->Clone(); - if (_core.get() == NULL) { - LOG(ERROR) << "fail to clone paddle predictor: " << origin_core; - return -1; - } - return 0; - } - - virtual void* get() { return _core.get(); } - - protected: - std::shared_ptr _core; -}; - -// infer interface -class FluidGpuAnalysisCore : public FluidFamilyCore { - public: - int create(const predictor::InferEngineCreationParams& params) { - std::string data_path = params.get_path(); - if (access(data_path.c_str(), F_OK) == -1) { - LOG(ERROR) << "create paddle predictor failed, path not exits: " - << data_path; - return -1; - } - - Config config; - config.SetParamsFile(data_path + "/__params__"); - config.SetProgFile(data_path + "/__model__"); - config.EnableUseGpu(100, FLAGS_gpuid); - config.SetCpuMathLibraryNumThreads(1); - - if (params.enable_memory_optimization()) { - config.EnableMemoryOptim(); - } - - config.SwitchSpecifyInputNames(true); - AutoLock lock(GlobalPaddleCreateMutex::instance()); - _core = CreatePredictor(config); - if (NULL == _core.get()) { - LOG(ERROR) << "create paddle predictor failed, path: " << data_path; - return -1; - } - - VLOG(2) << "create paddle predictor sucess, path: " << data_path; - return 0; - } -}; - -class FluidGpuAnalysisDirCore : public FluidFamilyCore { - public: - int create(const predictor::InferEngineCreationParams& params) { - std::string data_path = params.get_path(); - if (access(data_path.c_str(), F_OK) == -1) { - LOG(ERROR) << "create paddle predictor failed, path not exits: " - << data_path; - return -1; - } - - Config config; - config.SetModel(data_path); - config.EnableUseGpu(1500, FLAGS_gpuid); - config.SwitchSpecifyInputNames(true); - config.SetCpuMathLibraryNumThreads(1); - - if (params.enable_memory_optimization()) { - config.EnableMemoryOptim(); - } - int max_batch = 32; - int min_subgraph_size = 3; - if (params.use_trt()) { - config.EnableTensorRtEngine(1 << 20, - max_batch, - min_subgraph_size, - Config::Precision::kFloat32, - false, - false); - LOG(INFO) << "create TensorRT predictor"; - } else { - if (params.enable_memory_optimization()) { - config.EnableMemoryOptim(); - } - - if (params.enable_ir_optimization()) { - config.SwitchIrOptim(true); - } else { - config.SwitchIrOptim(false); - } - } - AutoLock lock(GlobalPaddleCreateMutex::instance()); - _core = CreatePredictor(config); - if (NULL == _core.get()) { - LOG(ERROR) << "create paddle predictor failed, path: " << data_path; - return -1; - } - - VLOG(2) << "create paddle predictor sucess, path: " << data_path; - return 0; - } -}; - -class Parameter { - public: - Parameter() : _row(0), _col(0), _params(NULL) {} - ~Parameter() { - LOG(INFO) << "before destroy Parameter, file_name[" << _file_name << "]"; - destroy(); - } - - int init(int row, int col, const char* file_name) { - destroy(); - _file_name = file_name; - _row = row; - _col = col; - _params = reinterpret_cast(malloc(_row * _col * sizeof(float))); - if (_params == NULL) { - LOG(ERROR) << "Load " << _file_name << " malloc error."; - return -1; - } - VLOG(2) << "Load parameter file[" << _file_name << "] success."; - return 0; - } - - void destroy() { - _row = 0; - _col = 0; - if (_params != NULL) { - free(_params); - _params = NULL; - } - } - - int load() { - if (_params == NULL || _row <= 0 || _col <= 0) { - LOG(ERROR) << "load parameter error [not inited]."; - return -1; - } - - FILE* fs = fopen(_file_name.c_str(), "rb"); - if (fs == NULL) { - LOG(ERROR) << "load " << _file_name << " fopen error."; - return -1; - } - static const uint32_t MODEL_FILE_HEAD_LEN = 16; - char head[MODEL_FILE_HEAD_LEN] = {0}; - if (fread(head, 1, MODEL_FILE_HEAD_LEN, fs) != MODEL_FILE_HEAD_LEN) { - destroy(); - LOG(ERROR) << "Load " << _file_name << " read head error."; - if (fs != NULL) { - fclose(fs); - fs = NULL; - } - return -1; - } - - uint32_t matrix_size = _row * _col; - if (matrix_size == fread(_params, sizeof(float), matrix_size, fs)) { - if (fs != NULL) { - fclose(fs); - fs = NULL; - } - LOG(INFO) << "load " << _file_name << " read ok."; - return 0; - } else { - LOG(ERROR) << "load " << _file_name << " read error."; - destroy(); - if (fs != NULL) { - fclose(fs); - fs = NULL; - } - return -1; - } - return 0; - } - - public: - std::string _file_name; - int _row; - int _col; - float* _params; -}; - -class FluidGpuAnalysisEncryptCore : public FluidFamilyCore { - public: - void ReadBinaryFile(const std::string& filename, std::string* contents) { - std::ifstream fin(filename, std::ios::in | std::ios::binary); - fin.seekg(0, std::ios::end); - contents->clear(); - contents->resize(fin.tellg()); - fin.seekg(0, std::ios::beg); - fin.read(&(contents->at(0)), contents->size()); - fin.close(); - } - - int create(const predictor::InferEngineCreationParams& params) { - std::string data_path = params.get_path(); - if (access(data_path.c_str(), F_OK) == -1) { - LOG(ERROR) << "create paddle predictor failed, path note exits: " - << data_path; - return -1; - } - - std::string model_buffer, params_buffer, key_buffer; - ReadBinaryFile(data_path + "encrypt_model", &model_buffer); - ReadBinaryFile(data_path + "encrypt_params", ¶ms_buffer); - ReadBinaryFile(data_path + "key", &key_buffer); - - VLOG(2) << "prepare for encryption model"; - - auto cipher = paddle::MakeCipher(""); - std::string real_model_buffer = cipher->Decrypt(model_buffer, key_buffer); - std::string real_params_buffer = cipher->Decrypt(params_buffer, key_buffer); - - Config analysis_config; - analysis_config.SetModelBuffer(&real_model_buffer[0], - real_model_buffer.size(), - &real_params_buffer[0], - real_params_buffer.size()); - analysis_config.EnableUseGpu(100, FLAGS_gpuid); - analysis_config.SetCpuMathLibraryNumThreads(1); - if (params.enable_memory_optimization()) { - analysis_config.EnableMemoryOptim(); - } - analysis_config.SwitchSpecifyInputNames(true); - AutoLock lock(GlobalPaddleCreateMutex::instance()); - VLOG(2) << "decrypt model file sucess"; - _core = CreatePredictor(analysis_config); - if (NULL == _core.get()) { - LOG(ERROR) << "create paddle predictor failed, path: " << data_path; - return -1; - } - VLOG(2) << "create paddle predictor sucess, path: " << data_path; - return 0; - } -}; - -} // namespace fluid_gpu -} // namespace paddle_serving -} // namespace baidu diff --git a/paddle_inference/inferencer-fluid-gpu/src/fluid_gpu_engine.cpp b/paddle_inference/inferencer-fluid-gpu/src/fluid_gpu_engine.cpp deleted file mode 100644 index 613b8343..00000000 --- a/paddle_inference/inferencer-fluid-gpu/src/fluid_gpu_engine.cpp +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "paddle_inference/inferencer-fluid-gpu/include/fluid_gpu_engine.h" -#include "core/predictor/framework/factory.h" - -DEFINE_int32(gpuid, 0, "GPU device id to use"); - -namespace baidu { -namespace paddle_serving { -namespace fluid_gpu { - -REGIST_FACTORY_OBJECT_IMPL_WITH_NAME( - ::baidu::paddle_serving::predictor::FluidInferEngine, - ::baidu::paddle_serving::predictor::InferEngine, - "FLUID_GPU_ANALYSIS"); - -REGIST_FACTORY_OBJECT_IMPL_WITH_NAME( - ::baidu::paddle_serving::predictor::FluidInferEngine< - FluidGpuAnalysisDirCore>, - ::baidu::paddle_serving::predictor::InferEngine, - "FLUID_GPU_ANALYSIS_DIR"); -REGIST_FACTORY_OBJECT_IMPL_WITH_NAME( - ::baidu::paddle_serving::predictor::FluidInferEngine< - FluidGpuAnalysisEncryptCore>, - ::baidu::paddle_serving::predictor::InferEngine, - "FLUID_GPU_ANALYSIS_ENCRPT") - -} // namespace fluid_gpu -} // namespace paddle_serving -} // namespace baidu diff --git a/paddle_inference/paddle/include/paddle_engine.h b/paddle_inference/paddle/include/paddle_engine.h index c92641f6..71cc9fc2 100644 --- a/paddle_inference/paddle/include/paddle_engine.h +++ b/paddle_inference/paddle/include/paddle_engine.h @@ -22,6 +22,7 @@ #include "core/configure/include/configure_parser.h" #include "core/configure/inferencer_configure.pb.h" #include "core/predictor/framework/infer.h" +#include "core/predictor/common/utils.h" #include "paddle_inference_api.h" // NOLINT namespace baidu { @@ -29,12 +30,16 @@ namespace paddle_serving { namespace inference { using paddle_infer::Config; +using paddle_infer::PrecisionType; using paddle_infer::Predictor; using paddle_infer::Tensor; using paddle_infer::CreatePredictor; +DECLARE_int32(gpuid); + const static int max_batch = 32; const static int min_subgraph_size = 3; + // Engine Base class PaddleEngineBase { public: @@ -100,7 +105,7 @@ class PaddleInferenceEngine : public PaddleEngineBase { // todo, auto config(zhangjun) if(engine_conf.has_combined_model()) { if(!engine_conf.combined_model()) { - config.SetModel(model_path) + config.SetModel(model_path); } else { config.SetParamsFile(model_path + "/__params__"); config.SetProgFile(model_path + "/__model__"); @@ -127,11 +132,11 @@ class PaddleInferenceEngine : public PaddleEngineBase { LOG(INFO) << "create TensorRT predictor"; } - if (engine_conf.has_lite() && engine_conf.use_lite()) { + if (engine_conf.has_use_lite() && engine_conf.use_lite()) { config.EnableLiteEngine(PrecisionType::kFloat32, true); } - if (engine_conf.has_xpu() && engine_conf.use_xpu()) { + if (engine_conf.has_use_xpu() && engine_conf.use_xpu()) { // 2 MB l3 cache config.EnableXpu(2 * 1024 * 1024); } @@ -150,14 +155,14 @@ class PaddleInferenceEngine : public PaddleEngineBase { //analysis_config.SetModelBuffer(); } - AutoLock lock(GlobalPaddleCreateMutex::instance()); + predictor::AutoLock lock(predictor::GlobalCreateMutex::instance()); _predictor = CreatePredictor(config); if (NULL == _predictor.get()) { - LOG(ERROR) << "create paddle predictor failed, path: " << data_path; + LOG(ERROR) << "create paddle predictor failed, path: " << model_path; return -1; } - VLOG(2) << "create paddle predictor sucess, path: " << data_path; + VLOG(2) << "create paddle predictor sucess, path: " << model_path; return 0; } }; diff --git a/paddle_inference/paddle/src/paddle_engine.cpp b/paddle_inference/paddle/src/paddle_engine.cpp index c9ae46eb..94ed4b9a 100644 --- a/paddle_inference/paddle/src/paddle_engine.cpp +++ b/paddle_inference/paddle/src/paddle_engine.cpp @@ -19,6 +19,8 @@ namespace baidu { namespace paddle_serving { namespace inference { +DEFINE_int32(gpuid, 0, "GPU device id to use"); + REGIST_FACTORY_OBJECT_IMPL_WITH_NAME( ::baidu::paddle_serving::predictor::FluidInferEngine, ::baidu::paddle_serving::predictor::InferEngine, -- GitLab