diff --git a/cube/cube-builder/CMakeLists.txt b/cube/cube-builder/CMakeLists.txt index 2048e24f1b852ff57fb67403679b13d337c87f36..1802465e3aaeb1885ae5d3396a1e5ac076228b37 100755 --- a/cube/cube-builder/CMakeLists.txt +++ b/cube/cube-builder/CMakeLists.txt @@ -20,11 +20,6 @@ set(CMAKE_CXX_STANDARD_REQUIRED ON) include_directories(SYSTEM ${CMAKE_CURRENT_LIST_DIR}/include) include_directories(SYSTEM ${CMAKE_CURRENT_BINARY_DIR}/../) -find_library(CURL_LIB NAMES curl) -if (NOT CURL_LIB) - message(FATAL_ERROR "Fail to find curl") -endif() - add_executable(cube-builder src/main.cpp include/cube-builder/util.h src/util.cpp src/builder_job.cpp include/cube-builder/builder_job.h include/cube-builder/define.h src/seqfile_reader.cpp include/cube-builder/seqfile_reader.h include/cube-builder/raw_reader.h include/cube-builder/vtext.h src/crovl_builder_increment.cpp include/cube-builder/crovl_builder_increment.h src/curl_simple.cpp include/cube-builder/curl_simple.h) add_dependencies(cube-builder jsoncpp) @@ -33,6 +28,7 @@ set(DYNAMIC_LIB gflags jsoncpp brpc + -lcurl -lssl -lcrypto ${CURL_LIB} diff --git a/doc/INSTALL.md b/doc/INSTALL.md index d17df500e09d1a2a497953f59706e6bd28f72886..655d034764832cd9370fe1f6110928a9785fa646 100644 --- a/doc/INSTALL.md +++ b/doc/INSTALL.md @@ -10,7 +10,9 @@ C++编译器 (验证过的版本：GCC 4.8.2/5.4.0) python (验证过的版本：2.7) -Go编译器 (验证过的版本：1.9.2) +Go编译器 (验证过的版本：1.9.2/1.12.0) + +openssl & openssl-devel ## 编译 diff --git a/inferencer-fluid-cpu/CMakeLists.txt b/inferencer-fluid-cpu/CMakeLists.txt index 9f892988d4c5ec8b04cbaf030087105bc84c040c..b58c0b720c1b7138de286ec18f34e2646ffa1301 100644 --- a/inferencer-fluid-cpu/CMakeLists.txt +++ b/inferencer-fluid-cpu/CMakeLists.txt @@ -2,7 +2,7 @@ FILE(GLOB fluid_cpu_engine_srcs ${CMAKE_CURRENT_LIST_DIR}/src/*.cpp) add_library(fluid_cpu_engine ${fluid_cpu_engine_srcs}) target_include_directories(fluid_cpu_engine PUBLIC ${CMAKE_BINARY_DIR}/Paddle/fluid_install_dir/) -add_dependencies(fluid_cpu_engine pdserving extern_paddle configure) +add_dependencies(fluid_cpu_engine pdserving extern_paddle configure kvdb) target_link_libraries(fluid_cpu_engine pdserving paddle_fluid iomp5 mklml_intel -lpthread -lcrypto -lm -lrt -lssl -ldl -lz) install(TARGETS fluid_cpu_engine diff --git a/inferencer-fluid-gpu/CMakeLists.txt b/inferencer-fluid-gpu/CMakeLists.txt index 725da85b45ca1070badf5343f340e49dce6b936f..6c980d6d94ef99585cf956072d8181d0d98cda42 100644 --- a/inferencer-fluid-gpu/CMakeLists.txt +++ b/inferencer-fluid-gpu/CMakeLists.txt @@ -2,7 +2,7 @@ FILE(GLOB fluid_gpu_engine_srcs ${CMAKE_CURRENT_LIST_DIR}/src/*.cpp) add_library(fluid_gpu_engine ${fluid_gpu_engine_srcs}) target_include_directories(fluid_gpu_engine PUBLIC ${CMAKE_BINARY_DIR}/Paddle/fluid_install_dir/) -add_dependencies(fluid_gpu_engine pdserving extern_paddle configure) +add_dependencies(fluid_gpu_engine pdserving extern_paddle configure kvdb) target_link_libraries(fluid_gpu_engine pdserving paddle_fluid iomp5 mklml_intel -lpthread -lcrypto -lm -lrt -lssl -ldl -lz) install(TARGETS fluid_gpu_engine diff --git a/kvdb/CMakeLists.txt b/kvdb/CMakeLists.txt index 51470669e0d69fa909c15890c2e52bfc4bc9a100..8eb2ee42cdbe1ff3982af9d375de0d9d2434f15b 100644 --- a/kvdb/CMakeLists.txt +++ b/kvdb/CMakeLists.txt @@ -9,6 +9,7 @@ set(SRC_LIST ${CMAKE_CURRENT_LIST_DIR}/src/test_rocksdb.cpp ${CMAKE_CURRENT_LIST_DIR}/src/gtest_kvdb.cpp) add_library(kvdb ${SRC_LIST}) +add_dependencies(kvdb rocksdb) install(TARGETS kvdb ARCHIVE DESTINATION ${PADDLE_SERVING_INSTALL_DIR}/lib/) add_executable(kvdb_test ${SRC_LIST}) diff --git a/kvdb/include/kvdb/kvdb_impl.h b/kvdb/include/kvdb/kvdb_impl.h index 8d8b146edd28fcd9e6a5ecb7a89e306c56a6839e..8dde028b6bddd8a32b9d2d4fd36eae90508a4e3c 100644 --- a/kvdb/include/kvdb/kvdb_impl.h +++ b/kvdb/include/kvdb/kvdb_impl.h @@ -45,6 +45,7 @@ class AbstractKVDB { virtual void SetDBName(std::string) = 0; virtual void Set(std::string key, std::string value) = 0; virtual std::string Get(std::string key) = 0; + virtual void Close() = 0; virtual ~AbstractKVDB() = 0; }; diff --git a/kvdb/include/kvdb/paddle_rocksdb.h b/kvdb/include/kvdb/paddle_rocksdb.h index 7be8d2be5048cbfcce311b4549f7e51a8eeea307..f11e11d1164eec6bc2f7f073a39bca7ff442c756 100644 --- a/kvdb/include/kvdb/paddle_rocksdb.h +++ b/kvdb/include/kvdb/paddle_rocksdb.h @@ -32,7 +32,7 @@ class RocksDBWrapper { void SetDBName(std::string db_name); static std::shared_ptr RocksDBWrapperFactory( std::string db_name = "SparseMatrix"); - + void Close(); private: rocksdb::DB *db_; std::string db_name_; diff --git a/kvdb/include/kvdb/rocksdb_impl.h b/kvdb/include/kvdb/rocksdb_impl.h index c6cef97ce95f0473642962753a60e98b40eeec58..84aa1313511d382952d9cfaa36dec18fc7adc06d 100644 --- a/kvdb/include/kvdb/rocksdb_impl.h +++ b/kvdb/include/kvdb/rocksdb_impl.h @@ -21,6 +21,7 @@ class RocksKVDB : public AbstractKVDB { void SetDBName(std::string); void Set(std::string key, std::string value); std::string Get(std::string key); + void Close(); ~RocksKVDB(); private: diff --git a/kvdb/src/gtest_kvdb.cpp b/kvdb/src/gtest_kvdb.cpp index 69113d70d9dbc5910e9074ca05d9e3b5e7b77776..a13bbd1bfb2ff4719a5f5f60431005a7beed1f5b 100644 --- a/kvdb/src/gtest_kvdb.cpp +++ b/kvdb/src/gtest_kvdb.cpp @@ -54,6 +54,7 @@ TEST_F(KVDBTest, AbstractKVDB_Unit_Test) { std::string val = kvdb->Get(std::to_string(i)); ASSERT_EQ(val, std::to_string(i * 2)); } + kvdb->Close(); } TEST_F(KVDBTest, FileReader_Unit_Test) { @@ -82,43 +83,6 @@ TEST_F(KVDBTest, FileReader_Unit_Test) { ASSERT_NE(timestamp_2, timestamp_3); } #include -TEST_F(KVDBTest, ParamDict_Unit_Test) { - std::string test_in_filename = "abs_dict_reader_test_in.txt"; - param_dict->SetFileReaderLst({test_in_filename}); - param_dict->SetReader([](std::string text) { - auto split = [](const std::string& s, - std::vector& sv, - const char* delim = " ") { - sv.clear(); - char* buffer = new char[s.size() + 1]; - std::copy(s.begin(), s.end(), buffer); - char* p = strtok(buffer, delim); - do { - sv.push_back(p); - } while ((p = strtok(NULL, delim))); - return; - }; - std::vector text_split; - split(text, text_split, " "); - std::string key = text_split[0]; - text_split.erase(text_split.begin()); - return make_pair(key, text_split); - }); - param_dict->CreateKVDB(); - GenerateTestIn(test_in_filename); - - param_dict->UpdateBaseModel(); - - std::this_thread::sleep_for(std::chrono::seconds(2)); - - std::vector test_vec = param_dict->GetSparseValue("1", ""); - - ASSERT_LT(fabs(test_vec[0] - 1.0), 1e-2); - - UpdateTestIn(test_in_filename); - param_dict->UpdateDeltaModel(); -} - void GenerateTestIn(std::string filename) { std::ifstream in_file(filename); if (in_file.good()) { diff --git a/kvdb/src/mock_param_dict_impl.cpp b/kvdb/src/mock_param_dict_impl.cpp index b85941c2d96f3fa3fc34cabd328adf6b5084c388..efb28bab90df796610258762d13e15c044b448d8 100644 --- a/kvdb/src/mock_param_dict_impl.cpp +++ b/kvdb/src/mock_param_dict_impl.cpp @@ -140,4 +140,5 @@ void ParamDict::CreateKVDB() { this->back_db->CreateDB(); } -ParamDict::~ParamDict() {} +ParamDict::~ParamDict() { +} diff --git a/kvdb/src/paddle_rocksdb.cpp b/kvdb/src/paddle_rocksdb.cpp index 4314b91c91c35d990c07eea1379e9e9f27ed7ea0..cea91a98ee2dd691874ff18cc0841b180a5059cc 100644 --- a/kvdb/src/paddle_rocksdb.cpp +++ b/kvdb/src/paddle_rocksdb.cpp @@ -48,6 +48,14 @@ void RocksDBWrapper::SetDBName(std::string db_name) { this->db_name_ = db_name; } +void RocksDBWrapper::Close() { + if (db_ != nullptr) { + db_->Close(); + delete(db_); + db_ = nullptr; + } +} + std::shared_ptr RocksDBWrapper::RocksDBWrapperFactory( std::string db_name) { return std::make_shared(db_name); diff --git a/kvdb/src/rockskvdb_impl.cpp b/kvdb/src/rockskvdb_impl.cpp index d6d4746804efb489169e32c2edb4e5fff3a718be..7fb3a9d252622ac71f7a6389086a5f6426575193 100644 --- a/kvdb/src/rockskvdb_impl.cpp +++ b/kvdb/src/rockskvdb_impl.cpp @@ -32,6 +32,12 @@ void RocksKVDB::Set(std::string key, std::string value) { return; } +void RocksKVDB::Close() { + this->db_->Close(); +} + std::string RocksKVDB::Get(std::string key) { return this->db_->Get(key); } -RocksKVDB::~RocksKVDB() {} +RocksKVDB::~RocksKVDB() { + this->db_->Close(); +}