From 71fad6a7f14cf7cff587deacbef5537114f82a9c Mon Sep 17 00:00:00 2001 From: wangjiawei04 Date: Wed, 25 Sep 2019 18:11:56 +0800 Subject: [PATCH] fix paddle_rocksdb.h and pure virtual func call bug. --- cube/cube-builder/CMakeLists.txt | 6 +---- inferencer-fluid-cpu/CMakeLists.txt | 2 +- kvdb/CMakeLists.txt | 1 + kvdb/include/kvdb/kvdb_impl.h | 1 + kvdb/include/kvdb/paddle_rocksdb.h | 2 +- kvdb/include/kvdb/rocksdb_impl.h | 1 + kvdb/src/gtest_kvdb.cpp | 38 +---------------------------- kvdb/src/mock_param_dict_impl.cpp | 3 ++- kvdb/src/paddle_rocksdb.cpp | 8 ++++++ kvdb/src/rockskvdb_impl.cpp | 8 +++++- 10 files changed, 24 insertions(+), 46 deletions(-) diff --git a/cube/cube-builder/CMakeLists.txt b/cube/cube-builder/CMakeLists.txt index 2048e24f..1802465e 100755 --- a/cube/cube-builder/CMakeLists.txt +++ b/cube/cube-builder/CMakeLists.txt @@ -20,11 +20,6 @@ set(CMAKE_CXX_STANDARD_REQUIRED ON) include_directories(SYSTEM ${CMAKE_CURRENT_LIST_DIR}/include) include_directories(SYSTEM ${CMAKE_CURRENT_BINARY_DIR}/../) -find_library(CURL_LIB NAMES curl) -if (NOT CURL_LIB) - message(FATAL_ERROR "Fail to find curl") -endif() - add_executable(cube-builder src/main.cpp include/cube-builder/util.h src/util.cpp src/builder_job.cpp include/cube-builder/builder_job.h include/cube-builder/define.h src/seqfile_reader.cpp include/cube-builder/seqfile_reader.h include/cube-builder/raw_reader.h include/cube-builder/vtext.h src/crovl_builder_increment.cpp include/cube-builder/crovl_builder_increment.h src/curl_simple.cpp include/cube-builder/curl_simple.h) add_dependencies(cube-builder jsoncpp) @@ -33,6 +28,7 @@ set(DYNAMIC_LIB gflags jsoncpp brpc + -lcurl -lssl -lcrypto ${CURL_LIB} diff --git a/inferencer-fluid-cpu/CMakeLists.txt b/inferencer-fluid-cpu/CMakeLists.txt index 9f892988..b58c0b72 100644 --- a/inferencer-fluid-cpu/CMakeLists.txt +++ b/inferencer-fluid-cpu/CMakeLists.txt @@ -2,7 +2,7 @@ FILE(GLOB fluid_cpu_engine_srcs ${CMAKE_CURRENT_LIST_DIR}/src/*.cpp) add_library(fluid_cpu_engine ${fluid_cpu_engine_srcs}) target_include_directories(fluid_cpu_engine PUBLIC ${CMAKE_BINARY_DIR}/Paddle/fluid_install_dir/) -add_dependencies(fluid_cpu_engine pdserving extern_paddle configure) +add_dependencies(fluid_cpu_engine pdserving extern_paddle configure kvdb) target_link_libraries(fluid_cpu_engine pdserving paddle_fluid iomp5 mklml_intel -lpthread -lcrypto -lm -lrt -lssl -ldl -lz) install(TARGETS fluid_cpu_engine diff --git a/kvdb/CMakeLists.txt b/kvdb/CMakeLists.txt index 51470669..8eb2ee42 100644 --- a/kvdb/CMakeLists.txt +++ b/kvdb/CMakeLists.txt @@ -9,6 +9,7 @@ set(SRC_LIST ${CMAKE_CURRENT_LIST_DIR}/src/test_rocksdb.cpp ${CMAKE_CURRENT_LIST_DIR}/src/gtest_kvdb.cpp) add_library(kvdb ${SRC_LIST}) +add_dependencies(kvdb rocksdb) install(TARGETS kvdb ARCHIVE DESTINATION ${PADDLE_SERVING_INSTALL_DIR}/lib/) add_executable(kvdb_test ${SRC_LIST}) diff --git a/kvdb/include/kvdb/kvdb_impl.h b/kvdb/include/kvdb/kvdb_impl.h index 8d8b146e..8dde028b 100644 --- a/kvdb/include/kvdb/kvdb_impl.h +++ b/kvdb/include/kvdb/kvdb_impl.h @@ -45,6 +45,7 @@ class AbstractKVDB { virtual void SetDBName(std::string) = 0; virtual void Set(std::string key, std::string value) = 0; virtual std::string Get(std::string key) = 0; + virtual void Close() = 0; virtual ~AbstractKVDB() = 0; }; diff --git a/kvdb/include/kvdb/paddle_rocksdb.h b/kvdb/include/kvdb/paddle_rocksdb.h index 7be8d2be..f11e11d1 100644 --- a/kvdb/include/kvdb/paddle_rocksdb.h +++ b/kvdb/include/kvdb/paddle_rocksdb.h @@ -32,7 +32,7 @@ class RocksDBWrapper { void SetDBName(std::string db_name); static std::shared_ptr RocksDBWrapperFactory( std::string db_name = "SparseMatrix"); - + void Close(); private: rocksdb::DB *db_; std::string db_name_; diff --git a/kvdb/include/kvdb/rocksdb_impl.h b/kvdb/include/kvdb/rocksdb_impl.h index c6cef97c..84aa1313 100644 --- a/kvdb/include/kvdb/rocksdb_impl.h +++ b/kvdb/include/kvdb/rocksdb_impl.h @@ -21,6 +21,7 @@ class RocksKVDB : public AbstractKVDB { void SetDBName(std::string); void Set(std::string key, std::string value); std::string Get(std::string key); + void Close(); ~RocksKVDB(); private: diff --git a/kvdb/src/gtest_kvdb.cpp b/kvdb/src/gtest_kvdb.cpp index 69113d70..a13bbd1b 100644 --- a/kvdb/src/gtest_kvdb.cpp +++ b/kvdb/src/gtest_kvdb.cpp @@ -54,6 +54,7 @@ TEST_F(KVDBTest, AbstractKVDB_Unit_Test) { std::string val = kvdb->Get(std::to_string(i)); ASSERT_EQ(val, std::to_string(i * 2)); } + kvdb->Close(); } TEST_F(KVDBTest, FileReader_Unit_Test) { @@ -82,43 +83,6 @@ TEST_F(KVDBTest, FileReader_Unit_Test) { ASSERT_NE(timestamp_2, timestamp_3); } #include -TEST_F(KVDBTest, ParamDict_Unit_Test) { - std::string test_in_filename = "abs_dict_reader_test_in.txt"; - param_dict->SetFileReaderLst({test_in_filename}); - param_dict->SetReader([](std::string text) { - auto split = [](const std::string& s, - std::vector& sv, - const char* delim = " ") { - sv.clear(); - char* buffer = new char[s.size() + 1]; - std::copy(s.begin(), s.end(), buffer); - char* p = strtok(buffer, delim); - do { - sv.push_back(p); - } while ((p = strtok(NULL, delim))); - return; - }; - std::vector text_split; - split(text, text_split, " "); - std::string key = text_split[0]; - text_split.erase(text_split.begin()); - return make_pair(key, text_split); - }); - param_dict->CreateKVDB(); - GenerateTestIn(test_in_filename); - - param_dict->UpdateBaseModel(); - - std::this_thread::sleep_for(std::chrono::seconds(2)); - - std::vector test_vec = param_dict->GetSparseValue("1", ""); - - ASSERT_LT(fabs(test_vec[0] - 1.0), 1e-2); - - UpdateTestIn(test_in_filename); - param_dict->UpdateDeltaModel(); -} - void GenerateTestIn(std::string filename) { std::ifstream in_file(filename); if (in_file.good()) { diff --git a/kvdb/src/mock_param_dict_impl.cpp b/kvdb/src/mock_param_dict_impl.cpp index b85941c2..efb28bab 100644 --- a/kvdb/src/mock_param_dict_impl.cpp +++ b/kvdb/src/mock_param_dict_impl.cpp @@ -140,4 +140,5 @@ void ParamDict::CreateKVDB() { this->back_db->CreateDB(); } -ParamDict::~ParamDict() {} +ParamDict::~ParamDict() { +} diff --git a/kvdb/src/paddle_rocksdb.cpp b/kvdb/src/paddle_rocksdb.cpp index 4314b91c..cea91a98 100644 --- a/kvdb/src/paddle_rocksdb.cpp +++ b/kvdb/src/paddle_rocksdb.cpp @@ -48,6 +48,14 @@ void RocksDBWrapper::SetDBName(std::string db_name) { this->db_name_ = db_name; } +void RocksDBWrapper::Close() { + if (db_ != nullptr) { + db_->Close(); + delete(db_); + db_ = nullptr; + } +} + std::shared_ptr RocksDBWrapper::RocksDBWrapperFactory( std::string db_name) { return std::make_shared(db_name); diff --git a/kvdb/src/rockskvdb_impl.cpp b/kvdb/src/rockskvdb_impl.cpp index d6d47468..7fb3a9d2 100644 --- a/kvdb/src/rockskvdb_impl.cpp +++ b/kvdb/src/rockskvdb_impl.cpp @@ -32,6 +32,12 @@ void RocksKVDB::Set(std::string key, std::string value) { return; } +void RocksKVDB::Close() { + this->db_->Close(); +} + std::string RocksKVDB::Get(std::string key) { return this->db_->Get(key); } -RocksKVDB::~RocksKVDB() {} +RocksKVDB::~RocksKVDB() { + this->db_->Close(); +} -- GitLab