diff --git a/demo-serving/CMakeLists.txt b/demo-serving/CMakeLists.txt index 54dabee4b04821fa7bf3186fe03fdc119b07b81b..5b9f564ebf71afef9a331687eea484a53b0b7cfb 100644 --- a/demo-serving/CMakeLists.txt +++ b/demo-serving/CMakeLists.txt @@ -32,7 +32,7 @@ if(WITH_GPU) endif() target_link_libraries(serving opencv_imgcodecs ${opencv_depend_libs} -Wl,--whole-archive fluid_cpu_engine - -Wl,--no-whole-archive pdserving paddle_fluid ${paddle_depend_libs} rocksdb_impl rocksdb + -Wl,--no-whole-archive pdserving paddle_fluid ${paddle_depend_libs} kvdb rocksdb ${MKLML_LIB} ${MKLML_IOMP_LIB} -lpthread -lcrypto -lm -lrt -lssl -ldl -lz -lbz2) install(TARGETS serving diff --git a/kvdb/CMakeLists.txt b/kvdb/CMakeLists.txt index 77a22e023fa138fc9dcd31df3548da00e0bc13fe..51470669e0d69fa909c15890c2e52bfc4bc9a100 100644 --- a/kvdb/CMakeLists.txt +++ b/kvdb/CMakeLists.txt @@ -8,8 +8,8 @@ set(SRC_LIST ${CMAKE_CURRENT_LIST_DIR}/src/test_rocksdb.cpp ${CMAKE_CURRENT_LIST_DIR}/src/paddle_rocksdb.cpp ${CMAKE_CURRENT_LIST_DIR}/src/gtest_kvdb.cpp) -add_library(rocksdb_impl ${SRC_LIST}) -install(TARGETS rocksdb_impl ARCHIVE DESTINATION ${PADDLE_SERVING_INSTALL_DIR}/lib/) +add_library(kvdb ${SRC_LIST}) +install(TARGETS kvdb ARCHIVE DESTINATION ${PADDLE_SERVING_INSTALL_DIR}/lib/) add_executable(kvdb_test ${SRC_LIST}) add_dependencies(kvdb_test rocksdb) diff --git a/kvdb/include/kvdb/kvdb_impl.h b/kvdb/include/kvdb/kvdb_impl.h index 54995822e59b24f2cea8da845d07d41ade8f83e4..215cd8e07762e99688f02eed0bd5ffc3a18a0f3e 100644 --- a/kvdb/include/kvdb/kvdb_impl.h +++ b/kvdb/include/kvdb/kvdb_impl.h @@ -45,7 +45,7 @@ public: virtual std::string GetMD5() = 0; virtual bool CheckDiff() = 0; virtual std::chrono::system_clock::time_point GetTimeStamp() = 0; - virtual void Read(std::vector&) = 0; + virtual void Read(AbstractParamDict*) = 0; virtual ~AbstractDictReader() = 0; protected: std::string filename_; diff --git a/kvdb/include/kvdb/mock_kvdb_impl.h b/kvdb/include/kvdb/rocksdb_impl.h similarity index 90% rename from kvdb/include/kvdb/mock_kvdb_impl.h rename to kvdb/include/kvdb/rocksdb_impl.h index d996614666ebe0ba4fe5aab9c495c7eda24810b9..890c5f86cab6c41b916359fbecbe36cf800e4d75 100644 --- a/kvdb/include/kvdb/mock_kvdb_impl.h +++ b/kvdb/include/kvdb/rocksdb_impl.h @@ -30,18 +30,18 @@ public: static int db_count; }; -class MockDictReader : public AbstractDictReader{ +class RocksDBDictReader : public AbstractDictReader{ public: std::string GetFileName(); void SetFileName(std::string); std::string GetMD5(); bool CheckDiff(); std::chrono::system_clock::time_point GetTimeStamp(); - void Read(std::vector&); - ~MockDictReader(); + void Read(AbstractParamDict*); + ~RocksDBDictReader(); }; -class MockParamDict : public AbstractParamDict{ +class RocksDBParamDict : public AbstractParamDict{ public: std::vector GetDictReaderLst(); void SetDictReaderLst(std::vector); @@ -59,7 +59,7 @@ public: void SetKVDB(std::pair); void CreateKVDB(); - ~MockParamDict(); + ~RocksDBParamDict(); }; diff --git a/kvdb/src/gtest_db_func.cpp b/kvdb/src/gtest_db_func.cpp index 4b3056a5782580c05a0af7edaa4a35930a4fa5be..f987c960b7165ebfc4b47fcfb1d669da59db00e8 100644 --- a/kvdb/src/gtest_db_func.cpp +++ b/kvdb/src/gtest_db_func.cpp @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "kvdb/mock_kvdb_impl.h" +#include "kvdb/rocksdb_impl.h" #include "kvdb/kvdb_impl.h" #include "kvdb/paddle_rocksdb.h" #include diff --git a/kvdb/src/gtest_db_thread.cpp b/kvdb/src/gtest_db_thread.cpp index 7db0fdbfcebc8cb507e44caf6a4d29acf4cde820..2e8a7f5bbe46f732cdcdd336a728a4b6be5bbedb 100644 --- a/kvdb/src/gtest_db_thread.cpp +++ b/kvdb/src/gtest_db_thread.cpp @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "kvdb/mock_kvdb_impl.h" +#include "kvdb/rocksdb_impl.h" #include "kvdb/kvdb_impl.h" #include "kvdb/paddle_rocksdb.h" #include diff --git a/kvdb/src/gtest_kvdb.cpp b/kvdb/src/gtest_kvdb.cpp index 99a5146f86095c2857e47cfa725b1d0a5ead5a54..b0070a82f958b1d5fa87b3c3563c882f85719501 100644 --- a/kvdb/src/gtest_kvdb.cpp +++ b/kvdb/src/gtest_kvdb.cpp @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "kvdb/mock_kvdb_impl.h" +#include "kvdb/rocksdb_impl.h" #include "kvdb/kvdb_impl.h" #include "kvdb/paddle_rocksdb.h" #include @@ -28,8 +28,8 @@ protected: static void SetUpTestCase() { kvdb = std::make_shared(); - dict_reader = std::make_shared(); - param_dict = std::make_shared(); + dict_reader = std::make_shared(); + param_dict = std::make_shared(); } static AbsKVDBPtr kvdb; @@ -81,7 +81,7 @@ TEST_F(KVDBTest, AbstractDictReader_Unit_Test) { ASSERT_NE(timestamp_2, timestamp_3); } #include -TEST_F(KVDBTest, MockParamDict_Unit_Test) { +TEST_F(KVDBTest, RocksDBParamDict_Unit_Test) { std::string test_in_filename = "abs_dict_reader_test_in.txt"; param_dict->SetDictReaderLst({dict_reader}); param_dict->CreateKVDB(); diff --git a/kvdb/src/mock_param_dict_impl.cpp b/kvdb/src/mock_param_dict_impl.cpp index 8f77c881d8c4130426c37f85674bab2763130d79..07f5b7b0304fb835f47d178c52529d3619059310 100644 --- a/kvdb/src/mock_param_dict_impl.cpp +++ b/kvdb/src/mock_param_dict_impl.cpp @@ -12,22 +12,22 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "kvdb/mock_kvdb_impl.h" +#include "kvdb/rocksdb_impl.h" #include #include #include #include -std::string MockDictReader::GetFileName() { +std::string RocksDBDictReader::GetFileName() { return this->filename_; } -void MockDictReader::SetFileName(std::string filename) { +void RocksDBDictReader::SetFileName(std::string filename) { this->filename_ = filename; this->last_md5_val_ = this->GetMD5(); this->time_stamp_ = std::chrono::system_clock::now(); } -std::string MockDictReader::GetMD5() { +std::string RocksDBDictReader::GetMD5() { auto getCmdOut = [] (std::string cmd) { std::string data; FILE *stream = nullptr; @@ -50,40 +50,40 @@ std::string MockDictReader::GetMD5() { return md5val; } -bool MockDictReader::CheckDiff() { +bool RocksDBDictReader::CheckDiff() { return this->GetMD5() == this->last_md5_val_; } -std::chrono::system_clock::time_point MockDictReader::GetTimeStamp() { -//TODO: Implement Get Time Stamp of dict file +std::chrono::system_clock::time_point RocksDBDictReader::GetTimeStamp() { return this->time_stamp_; } -void MockDictReader::Read(std::vector& res) { +void RocksDBDictReader::Read(AbstractParamDict* param_dict) { std::string line; std::ifstream infile(this->filename_); if (infile.is_open()) { while (getline(infile, line)) { - res.push_back(line); + //TODO: Write String Parse Here + // param_dict->InsertSparseValue(); } } infile.close(); } -MockDictReader::~MockDictReader() { +RocksDBDictReader::~RocksDBDictReader() { //TODO: I imageine nothing to do here } -std::vector MockParamDict::GetDictReaderLst() { +std::vector RocksDBParamDict::GetDictReaderLst() { return this->dict_reader_lst_; } -void MockParamDict::SetDictReaderLst(std::vector lst) { +void RocksDBParamDict::SetDictReaderLst(std::vector lst) { this->dict_reader_lst_ = lst; } -std::vector MockParamDict::GetSparseValue(std::string feasign, std::string slot) { +std::vector RocksDBParamDict::GetSparseValue(std::string feasign, std::string slot) { auto BytesToFloat = [](uint8_t* byteArray){ return *((float*)byteArray); }; @@ -100,15 +100,15 @@ std::vector MockParamDict::GetSparseValue(std::string feasign, std::strin return value; } -std::vector MockParamDict::GetSparseValue(int64_t feasign, int64_t slot) { +std::vector RocksDBParamDict::GetSparseValue(int64_t feasign, int64_t slot) { return this->GetSparseValue(std::to_string(feasign), std::to_string(slot)); } -bool MockParamDict::InsertSparseValue(int64_t feasign, int64_t slot, const std::vector& values) { +bool RocksDBParamDict::InsertSparseValue(int64_t feasign, int64_t slot, const std::vector& values) { return this->InsertSparseValue(std::to_string(feasign), std::to_string(slot), values); } -bool MockParamDict::InsertSparseValue(std::string feasign, std::string slot, const std::vector& values) { +bool RocksDBParamDict::InsertSparseValue(std::string feasign, std::string slot, const std::vector& values) { auto FloatToBytes = [](float fvalue, uint8_t *arr){ unsigned char *pf = nullptr; unsigned char *px = nullptr; @@ -136,12 +136,12 @@ bool MockParamDict::InsertSparseValue(std::string feasign, std::string slot, con return true; } -void MockParamDict::UpdateBaseModel() { +void RocksDBParamDict::UpdateBaseModel() { std::thread t([&] () { for (AbsDictReaderPtr dict_reader: this->dict_reader_lst_) { if (dict_reader->CheckDiff()) { std::vector strs; - dict_reader->Read(strs); + dict_reader->Read(this); for (const std::string& str: strs) { std::vector arr; std::istringstream in(str); @@ -162,27 +162,27 @@ void MockParamDict::UpdateBaseModel() { } -void MockParamDict::UpdateDeltaModel() { +void RocksDBParamDict::UpdateDeltaModel() { UpdateBaseModel(); } -std::pair MockParamDict::GetKVDB() { +std::pair RocksDBParamDict::GetKVDB() { return {front_db, back_db}; } -void MockParamDict::SetKVDB(std::pair kvdbs) { +void RocksDBParamDict::SetKVDB(std::pair kvdbs) { this->front_db = kvdbs.first; this->back_db = kvdbs.second; } -void MockParamDict::CreateKVDB() { +void RocksDBParamDict::CreateKVDB() { this->front_db = std::make_shared(); this->back_db = std::make_shared(); this->front_db->CreateDB(); this->back_db->CreateDB(); } -MockParamDict::~MockParamDict() { +RocksDBParamDict::~RocksDBParamDict() { } diff --git a/kvdb/src/rockskvdb_impl.cpp b/kvdb/src/rockskvdb_impl.cpp index 6c545d31c81cca42b28b492b9035ef3069294862..08371abf337878414c1617aad5bbb2ac63aa0a11 100644 --- a/kvdb/src/rockskvdb_impl.cpp +++ b/kvdb/src/rockskvdb_impl.cpp @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "kvdb/mock_kvdb_impl.h" +#include "kvdb/rocksdb_impl.h" int RocksKVDB::db_count; void RocksKVDB::CreateDB() { diff --git a/kvdb/src/test_rocksdb.cpp b/kvdb/src/test_rocksdb.cpp index 1a9cc9143a32c24c774b577b6c4a304c8e2d7276..ca1d714ac412e63d0f1c1e5e0b1ac0c75efc618a 100644 --- a/kvdb/src/test_rocksdb.cpp +++ b/kvdb/src/test_rocksdb.cpp @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "kvdb/mock_kvdb_impl.h" +#include "kvdb/rocksdb_impl.h" #include "kvdb/paddle_rocksdb.h" #include void test_rockskvdb() {