diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e718b32cb6c48d11e73600509a17db107f438708..d8112837dc9627bc2e501940b8e97c89e97c45ff 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -42,12 +42,6 @@ repos: entry: bash ./tools/codestyle/pylint_pre_commit.hook language: system files: \.(py)$ -- repo: https://github.com/PaddlePaddle/pre-commit-golang - sha: 8337620115c25ff8333f1b1a493bd031049bd7c0 - hooks: - - id: go-fmt - types: - - go - repo: local hooks: - id: copyright_checker diff --git a/cmake/external/opencv.cmake b/cmake/external/opencv.cmake index 07b0afdb0da8aef053bbc847e6e1556bab61308c..0744120f2fb99a256d783ab52153335fdf953032 100644 --- a/cmake/external/opencv.cmake +++ b/cmake/external/opencv.cmake @@ -38,6 +38,7 @@ ExternalProject_Add( -DCMAKE_INSTALL_PREFIX=${OPENCV_INSTALL_DIR} -DCMAKE_INSTALL_LIBDIR=${OPENCV_INSTALL_DIR}/lib -DCMAKE_POSITION_INDEPENDENT_CODE=ON + -DWITH_GTK=OFF -DBUILD_TESTS=OFF -DBUILD_PERF_TESTS=OFF -DCMAKE_BUILD_TYPE=${THIRD_PARTY_BUILD_TYPE} diff --git a/cmake/external/redis++.cmake b/cmake/external/redis++.cmake deleted file mode 100644 index 1809035f5cacc4507dade593a58b274b322698d7..0000000000000000000000000000000000000000 --- a/cmake/external/redis++.cmake +++ /dev/null @@ -1,45 +0,0 @@ -# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -INCLUDE(ExternalProject) - -SET(REDISCLIENT_SOURCES_DIR ${THIRD_PARTY_PATH}/redis++) -SET(REDISCLIENT_INSTALL_DIR ${THIRD_PARTY_PATH}/install/redis++) -SET(REDISCLIENT_INCLUDE_DIR "${REDISCLIENT_INSTALL_DIR}/include" CACHE PATH "redis++ include directory." FORCE) -SET(REDISCLIENT_LIBRARIES "${REDISCLIENT_INSTALL_DIR}/lib/libredis++.a" CACHE FILEPATH "redis++ library." FORCE) -INCLUDE_DIRECTORIES(${REDISCLIENT_INCLUDE_DIR}) - -ExternalProject_Add( - extern_redis++ - ${EXTERNAL_PROJECT_LOG_ARGS} - PREFIX ${REDISCLIENT_SOURCES_DIR} - GIT_REPOSITORY "https://github.com/sewenew/redis-plus-plus" - GIT_TAG master - UPDATE_COMMAND "" - CONFIGURE_COMMAND "" - BUILD_COMMAND cmake . && CXXFLAGS=-fPIC make -j ${NUM_OF_PROCESSOR} static - INSTALL_COMMAND mkdir -p ${REDISCLIENT_INSTALL_DIR}/lib/ - && cp ${REDISCLIENT_SOURCES_DIR}/src/extern_redis++/lib/libredis++.a ${REDISCLIENT_LIBRARIES} - && cp -r ${REDISCLIENT_SOURCES_DIR}/src/extern_redis++/src/sw/redis++/ ${REDISCLIENT_INSTALL_DIR}/include/redis++/ - BUILD_IN_SOURCE 1 -) - -ADD_DEPENDENCIES(extern_redis++ snappy) - -ADD_LIBRARY(redis++ STATIC IMPORTED GLOBAL) -SET_PROPERTY(TARGET redis++ PROPERTY IMPORTED_LOCATION ${ROCKSDB_LIBRARIES}) -ADD_DEPENDENCIES(redis++ extern_redis++) - -LIST(APPEND external_project_dependencies redis++) - diff --git a/demo-client/src/echo_kvdb.cpp b/demo-client/src/echo_kvdb.cpp index ae53f8838bee596e63fc3c6bbc7007fe5cf7a0ea..8f6dda22d664c102ccc2432c1dfa1abbc9a84be1 100644 --- a/demo-client/src/echo_kvdb.cpp +++ b/demo-client/src/echo_kvdb.cpp @@ -16,12 +16,12 @@ #include #include +#include #include #include "sdk-cpp/builtin_format.pb.h" #include "sdk-cpp/echo_kvdb_service.pb.h" #include "sdk-cpp/include/common.h" #include "sdk-cpp/include/predictor_sdk.h" -#include using baidu::paddle_serving::sdk_cpp::Predictor; using baidu::paddle_serving::sdk_cpp::PredictorApi; using baidu::paddle_serving::predictor::echo_kvdb_service::Request; @@ -48,11 +48,11 @@ void print_res(const Request& req, uint64_t elapse_ms) { LOG(INFO) << "Receive Response size: " << res.ress_size(); for (size_t i = 0; i < res.ress_size(); i++) { - KVDBRes val = res.ress(i); - LOG(INFO) << "Receive value from demo-server: " << val.value(); -} - LOG(INFO) << "Succ call predictor[echo_kvdb_service], the tag is: " << route_tag - << ", elapse_ms: " << elapse_ms; + KVDBRes val = res.ress(i); + LOG(INFO) << "Receive value from demo-server: " << val.value(); + } + LOG(INFO) << "Succ call predictor[echo_kvdb_service], the tag is: " + << route_tag << ", elapse_ms: " << elapse_ms; } int main(int argc, char** argv) { @@ -100,7 +100,7 @@ int main(int argc, char** argv) { while (true) { if (global_key > 10000) { - break; + break; } timeval start; gettimeofday(&start, NULL); diff --git a/demo-serving/op/kvdb_echo_op.cpp b/demo-serving/op/kvdb_echo_op.cpp index 1c4d013a815a9708fae0384873952d625ecb32be..844401012e3d25cab11197f10d2e8601c10c5a25 100644 --- a/demo-serving/op/kvdb_echo_op.cpp +++ b/demo-serving/op/kvdb_echo_op.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. -// +// // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -23,36 +23,34 @@ using baidu::paddle_serving::predictor::format::KVDBRes; using baidu::paddle_serving::predictor::echo_kvdb_service::Request; using baidu::paddle_serving::predictor::echo_kvdb_service::Response; -int KVDBEchoOp::inference() { - debug(); -} +int KVDBEchoOp::inference() { debug(); } int KVDBEchoOp::debug() { - //TODO: implement DEBUG mode - baidu::paddle_serving::predictor::Resource& resource = baidu::paddle_serving::predictor::Resource::instance(); - std::shared_ptr db = resource.getDB(); - const Request* req = dynamic_cast(get_request_message()); - Response* res = mutable_data(); - LOG(INFO) << "Receive request in KVDB echo service: " << req->ShortDebugString(); - for (size_t i = 0; i < req->reqs_size(); i++) { - auto kvdbreq = req->reqs(i); - std::string op = kvdbreq.op(); - std::string key = kvdbreq.key(); - std::string val = kvdbreq.value(); - if (op == "SET") { - db->Put(key, val); - KVDBRes* kvdb_value_res = res->mutable_ress()->Add(); - kvdb_value_res -> set_value("OK"); - } else if (op == "GET") { - std::string getvalue = db->Get(key); - KVDBRes* kvdb_value_res = res->mutable_ress()->Add(); - kvdb_value_res -> set_value(getvalue); - } + // TODO: implement DEBUG mode + baidu::paddle_serving::predictor::Resource& resource = + baidu::paddle_serving::predictor::Resource::instance(); + std::shared_ptr db = resource.getDB(); + const Request* req = dynamic_cast(get_request_message()); + Response* res = mutable_data(); + LOG(INFO) << "Receive request in KVDB echo service: " + << req->ShortDebugString(); + for (size_t i = 0; i < req->reqs_size(); i++) { + auto kvdbreq = req->reqs(i); + std::string op = kvdbreq.op(); + std::string key = kvdbreq.key(); + std::string val = kvdbreq.value(); + if (op == "SET") { + db->Put(key, val); + KVDBRes* kvdb_value_res = res->mutable_ress()->Add(); + kvdb_value_res->set_value("OK"); + } else if (op == "GET") { + std::string getvalue = db->Get(key); + KVDBRes* kvdb_value_res = res->mutable_ress()->Add(); + kvdb_value_res->set_value(getvalue); } - return 0; + } + return 0; } - - DEFINE_OP(KVDBEchoOp); } } diff --git a/demo-serving/op/kvdb_echo_op.h b/demo-serving/op/kvdb_echo_op.h index 7c7502a11716fc3cf64e059481127f09f8bbb0ea..f78ce05857f5ed74a02210ba32feef73752f1580 100644 --- a/demo-serving/op/kvdb_echo_op.h +++ b/demo-serving/op/kvdb_echo_op.h @@ -14,23 +14,24 @@ #pragma once #include "demo-serving/echo_kvdb_service.pb.h" -#include "predictor/framework/resource.h" +#include "kvdb/paddle_rocksdb.h" #include "predictor/common/inner_common.h" #include "predictor/framework/channel.h" #include "predictor/framework/op_repository.h" +#include "predictor/framework/resource.h" #include "predictor/op/op.h" -#include "kvdb/paddle_rocksdb.h" namespace baidu { namespace paddle_serving { namespace predictor { -class KVDBEchoOp: public OpWithChannel { - public: - DECLARE_OP(KVDBEchoOp); - int inference(); - int debug(); - +class KVDBEchoOp + : public OpWithChannel< + baidu::paddle_serving::predictor::echo_kvdb_service::Response> { + public: + DECLARE_OP(KVDBEchoOp); + int inference(); + int debug(); }; } // namespace predictor } // namespace paddle_serving diff --git a/demo-serving/op/reader_op.cpp b/demo-serving/op/reader_op.cpp index c57e15191423ab5b6ce6ee428e230b4cf2e227d5..85492b80603d00b65e195ba1e4d8249d2945fe9e 100644 --- a/demo-serving/op/reader_op.cpp +++ b/demo-serving/op/reader_op.cpp @@ -64,9 +64,10 @@ int ReaderOp::inference() { size_t dense_capacity = 3 * resize.width * resize.height; size_t len = dense_capacity * sizeof(float) * sample_size; - // Allocate buffer in PaddleTensor, so that buffer will be managed by the Tensor + // Allocate buffer in PaddleTensor, so that buffer will be managed by the + // Tensor in_tensor.data.Resize(len); - float *data = reinterpret_cast(in_tensor.data.data()); + float* data = reinterpret_cast(in_tensor.data.data()); if (in_tensor.data.data() == NULL) { LOG(ERROR) << "Failed create temp float array, " << "size=" << dense_capacity * sample_size * sizeof(float); diff --git a/demo-serving/proto/echo_kvdb_service.proto b/demo-serving/proto/echo_kvdb_service.proto index 58ffb21bd870185bc84df566fb57b58a63f59c49..d7ef6fc3a7928d1b0e8633f34a21778be50dd7dd 100644 --- a/demo-serving/proto/echo_kvdb_service.proto +++ b/demo-serving/proto/echo_kvdb_service.proto @@ -20,15 +20,15 @@ package baidu.paddle_serving.predictor.echo_kvdb_service; option cc_generic_services = true; message Request { - repeated baidu.paddle_serving.predictor.format.KVDBReq reqs = 1; + repeated baidu.paddle_serving.predictor.format.KVDBReq reqs = 1; }; message Response { - repeated baidu.paddle_serving.predictor.format.KVDBRes ress = 1; + repeated baidu.paddle_serving.predictor.format.KVDBRes ress = 1; }; service EchoKVDBService { - rpc inference(Request) returns (Response); - rpc debug(Request) returns (Response); - option (pds.options).generate_impl = true; + rpc inference(Request) returns (Response); + rpc debug(Request) returns (Response); + option (pds.options).generate_impl = true; }; diff --git a/kvdb/include/kvdb/kvdb_impl.h b/kvdb/include/kvdb/kvdb_impl.h index b610120afc15314a46134a88db1c5979ba6b1287..8d8b146edd28fcd9e6a5ecb7a89e306c56a6839e 100644 --- a/kvdb/include/kvdb/kvdb_impl.h +++ b/kvdb/include/kvdb/kvdb_impl.h @@ -1,3 +1,16 @@ +// Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. // Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. // @@ -14,10 +27,10 @@ // limitations under the License. #pragma once -#include -#include -#include #include +#include +#include +#include class AbstractKVDB; class FileReader; class ParamDict; @@ -27,104 +40,102 @@ typedef std::shared_ptr FileReaderPtr; typedef std::shared_ptr ParamDictPtr; class AbstractKVDB { -public: - virtual void CreateDB() = 0; - virtual void SetDBName(std::string) = 0; - virtual void Set(std::string key, std::string value) = 0; - virtual std::string Get(std::string key) = 0; - virtual ~AbstractKVDB() = 0; + public: + virtual void CreateDB() = 0; + virtual void SetDBName(std::string) = 0; + virtual void Set(std::string key, std::string value) = 0; + virtual std::string Get(std::string key) = 0; + virtual ~AbstractKVDB() = 0; }; -// TODO: Implement RedisKVDB -//class RedisKVDB; - class FileReader { -public: - inline virtual std::string GetFileName() { - return this->filename_; - } - - inline virtual void SetFileName(std::string filename) { - this->filename_ = filename; - this->last_md5_val_ = this->GetMD5(); - this->time_stamp_ = std::chrono::system_clock::now(); - } - - inline virtual std::string GetMD5() { - auto getCmdOut = [] (std::string cmd) { - std::string data; - FILE *stream = nullptr; - const int max_buffer = 256; - char buffer[max_buffer]; - cmd.append(" 2>&1"); - stream = popen(cmd.c_str(), "r"); - if (stream) { - if (fgets(buffer, max_buffer, stream) != NULL) { - data.append(buffer); - } - } - return data; - }; - std::string cmd = "md5sum " + this->filename_; - //TODO: throw exception if error occurs during execution of shell command - std::string md5val = getCmdOut(cmd); - this->time_stamp_ = md5val == this->last_md5_val_? this->time_stamp_: std::chrono::system_clock::now(); - this->last_md5_val_ = md5val; - return md5val; - } - - inline virtual bool CheckDiff() { - return this->GetMD5() == this->last_md5_val_; - } - - inline virtual std::chrono::system_clock::time_point GetTimeStamp() { - return this->time_stamp_; - } - - inline virtual ~FileReader() {}; -protected: - std::string filename_; - std::string last_md5_val_; - std::chrono::system_clock::time_point time_stamp_; + public: + inline virtual std::string GetFileName() { return this->filename_; } + + inline virtual void SetFileName(std::string filename) { + this->filename_ = filename; + this->last_md5_val_ = this->GetMD5(); + this->time_stamp_ = std::chrono::system_clock::now(); + } + + inline virtual std::string GetMD5() { + auto getCmdOut = [](std::string cmd) { + std::string data; + FILE *stream = nullptr; + const int max_buffer = 256; + char buffer[max_buffer]; + cmd.append(" 2>&1"); + stream = popen(cmd.c_str(), "r"); + if (stream) { + if (fgets(buffer, max_buffer, stream) != NULL) { + data.append(buffer); + } + } + return data; + }; + std::string cmd = "md5sum " + this->filename_; + // TODO: throw exception if error occurs during execution of shell command + std::string md5val = getCmdOut(cmd); + this->time_stamp_ = md5val == this->last_md5_val_ + ? this->time_stamp_ + : std::chrono::system_clock::now(); + this->last_md5_val_ = md5val; + return md5val; + } + + inline virtual bool CheckDiff() { + return this->GetMD5() == this->last_md5_val_; + } + + inline virtual std::chrono::system_clock::time_point GetTimeStamp() { + return this->time_stamp_; + } + + inline virtual ~FileReader(){}; + + private: + std::string filename_; + std::string last_md5_val_; + std::chrono::system_clock::time_point time_stamp_; }; - class ParamDict { - typedef std::string Key; - typedef std::vector Value; -public: - virtual std::vector GetDictReaderLst(); - virtual void SetFileReaderLst(std::vector lst); - - virtual std::vector GetSparseValue(int64_t, int64_t); - virtual std::vector GetSparseValue(std::string, std::string); - - virtual bool InsertSparseValue(int64_t, int64_t, const std::vector&); - virtual bool InsertSparseValue(std::string, std::string, const std::vector&); - - virtual void SetReader(std::function(std::string)>); - virtual void UpdateBaseModel(); - virtual void UpdateDeltaModel(); - - virtual std::pair GetKVDB(); - virtual void SetKVDB(std::pair); - virtual void CreateKVDB(); - - virtual ~ParamDict(); -protected: - std::function(std::string)> read_func_; - std::vector file_reader_lst_; - AbsKVDBPtr front_db, back_db; -}; + typedef std::string Key; + typedef std::vector Value; + public: + virtual std::vector GetDictReaderLst(); + virtual void SetFileReaderLst(std::vector lst); + virtual std::vector GetSparseValue(int64_t, int64_t); + virtual std::vector GetSparseValue(std::string, std::string); -class ParamDictMgr { -public: - void UpdateAll(); - void InsertParamDict(std::string, ParamDictPtr); + virtual bool InsertSparseValue(int64_t, int64_t, const std::vector &); + virtual bool InsertSparseValue(std::string, + std::string, + const std::vector &); + + virtual void SetReader(std::function(std::string)>); + virtual void UpdateBaseModel(); + virtual void UpdateDeltaModel(); + + virtual std::pair GetKVDB(); + virtual void SetKVDB(std::pair); + virtual void CreateKVDB(); + + virtual ~ParamDict(); -protected: - std::unordered_map ParamDictMap; + private: + std::function(std::string)> read_func_; + std::vector file_reader_lst_; + AbsKVDBPtr front_db, back_db; }; +class ParamDictMgr { + public: + void UpdateAll(); + void InsertParamDict(std::string, ParamDictPtr); + + private: + std::unordered_map ParamDictMap; +}; diff --git a/kvdb/include/kvdb/paddle_rocksdb.h b/kvdb/include/kvdb/paddle_rocksdb.h index 5ae1a9dd4a061f5dc57ad0d0c30762f7e2a1d3a0..7be8d2be5048cbfcce311b4549f7e51a8eeea307 100644 --- a/kvdb/include/kvdb/paddle_rocksdb.h +++ b/kvdb/include/kvdb/paddle_rocksdb.h @@ -15,26 +15,25 @@ #pragma once #include #include +#include "rocksdb/compaction_filter.h" #include "rocksdb/db.h" +#include "rocksdb/filter_policy.h" #include "rocksdb/options.h" #include "rocksdb/slice.h" #include "rocksdb/sst_file_writer.h" #include "rocksdb/table.h" -#include "rocksdb/compaction_filter.h" -#include "rocksdb/filter_policy.h" - class RocksDBWrapper { -public: - RocksDBWrapper(std::string db_name); - std::string Get(std::string key); + public: + RocksDBWrapper(std::string db_name); + std::string Get(std::string key); - bool Put(std::string key, std::string value); - void SetDBName(std::string db_name); - static std::shared_ptr RocksDBWrapperFactory(std::string db_name = "SparseMatrix"); + bool Put(std::string key, std::string value); + void SetDBName(std::string db_name); + static std::shared_ptr RocksDBWrapperFactory( + std::string db_name = "SparseMatrix"); -protected: - rocksdb::DB *db_; - std::string db_name_; + private: + rocksdb::DB *db_; + std::string db_name_; }; - diff --git a/kvdb/include/kvdb/rocksdb_impl.h b/kvdb/include/kvdb/rocksdb_impl.h index 71adb23a5175d4b4542f71947f415b9313db74f9..c6cef97ce95f0473642962753a60e98b40eeec58 100644 --- a/kvdb/include/kvdb/rocksdb_impl.h +++ b/kvdb/include/kvdb/rocksdb_impl.h @@ -13,23 +13,19 @@ // limitations under the License. #pragma once -#include "kvdb_impl.h" -#include "paddle_rocksdb.h" -class RocksKVDB: public AbstractKVDB { -public: - void CreateDB(); - void SetDBName(std::string); - void Set(std::string key, std::string value); - std::string Get(std::string key); - ~RocksKVDB(); +#include "kvdb/kvdb_impl.h" +#include "kvdb/paddle_rocksdb.h" +class RocksKVDB : public AbstractKVDB { + public: + void CreateDB(); + void SetDBName(std::string); + void Set(std::string key, std::string value); + std::string Get(std::string key); + ~RocksKVDB(); -protected: - std::shared_ptr db_; + private: + std::shared_ptr db_; -public: - static int db_count; + public: + static int db_count; }; - - - - diff --git a/kvdb/src/gtest_db_func.cpp b/kvdb/src/gtest_db_func.cpp index f987c960b7165ebfc4b47fcfb1d669da59db00e8..aaf679bfa72a83646a8abae98e4068a26331a487 100644 --- a/kvdb/src/gtest_db_func.cpp +++ b/kvdb/src/gtest_db_func.cpp @@ -12,65 +12,56 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "kvdb/rocksdb_impl.h" -#include "kvdb/kvdb_impl.h" -#include "kvdb/paddle_rocksdb.h" #include -#include +#include #include #include -#include +#include #include +#include "kvdb/kvdb_impl.h" +#include "kvdb/paddle_rocksdb.h" +#include "kvdb/rocksdb_impl.h" class KVDBTest : public ::testing::Test { -protected: - void SetUp() override{ - - } - - static void SetUpTestCase() { - } - + protected: + void SetUp() override {} + static void SetUpTestCase() {} }; int my_argc; char** my_argv; std::vector StringSplit(std::string str, char split) { - std::vector strs; - std::istringstream f(str); - std:: string s; - while (getline(f, s, split)) { - strs.push_back(s); - } - return strs; + std::vector strs; + std::istringstream f(str); + std::string s; + while (getline(f, s, split)) { + strs.push_back(s); + } + return strs; } TEST_F(KVDBTest, AbstractKVDB_Func_Test) { - AbsKVDBPtr kvdb = std::make_shared(); - kvdb->CreateDB(); - std::string set_list = "setlist.txt"; - std::string get_list = "getlist.txt"; - std::ifstream set_file(set_list); - std::ifstream get_file(get_list); - for (std::string line; getline(set_file, line); ) - { - std::vector strs = StringSplit (line, ' '); - kvdb->Set(strs[0], strs[1]); - } + AbsKVDBPtr kvdb = std::make_shared(); + kvdb->CreateDB(); + std::string set_list = "setlist.txt"; + std::string get_list = "getlist.txt"; + std::ifstream set_file(set_list); + std::ifstream get_file(get_list); + for (std::string line; getline(set_file, line);) { + std::vector strs = StringSplit(line, ' '); + kvdb->Set(strs[0], strs[1]); + } - for (std::string line; getline(get_file, line); ) { - std::vector strs = StringSplit(line, ' '); - std::string val = kvdb->Get(strs[0]); - ASSERT_EQ(val, strs[1]); - } + for (std::string line; getline(get_file, line);) { + std::vector strs = StringSplit(line, ' '); + std::string val = kvdb->Get(strs[0]); + ASSERT_EQ(val, strs[1]); + } } - - int main(int argc, char** argv) { - my_argc = argc; - my_argv = argv; - ::testing::InitGoogleTest(&argc, argv); - return RUN_ALL_TESTS(); + my_argc = argc; + my_argv = argv; + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); } - diff --git a/kvdb/src/gtest_db_thread.cpp b/kvdb/src/gtest_db_thread.cpp index 2e8a7f5bbe46f732cdcdd336a728a4b6be5bbedb..cfc5807726c10c36760b04a92883cc97937c66ba 100644 --- a/kvdb/src/gtest_db_thread.cpp +++ b/kvdb/src/gtest_db_thread.cpp @@ -12,63 +12,59 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "kvdb/rocksdb_impl.h" -#include "kvdb/kvdb_impl.h" -#include "kvdb/paddle_rocksdb.h" #include -#include -#include #include +#include +#include #include +#include "kvdb/kvdb_impl.h" +#include "kvdb/paddle_rocksdb.h" +#include "kvdb/rocksdb_impl.h" class KVDBTest : public ::testing::Test { -protected: - void SetUp() override{ - - } - - static void SetUpTestCase() { + protected: + void SetUp() override {} - } + static void SetUpTestCase() {} }; + int my_argc; char** my_argv; - void db_thread_test(AbsKVDBPtr kvdb, int size) { - for (int i = 0; i < size; i++) { - kvdb->Set(std::to_string(i), std::to_string(i)); - kvdb->Get(std::to_string(i)); - } + for (int i = 0; i < size; i++) { + kvdb->Set(std::to_string(i), std::to_string(i)); + kvdb->Get(std::to_string(i)); + } } TEST_F(KVDBTest, AbstractKVDB_Thread_Test) { - if (my_argc != 3) { - std::cerr << "illegal input! should be db_thread ${num_of_thread} ${num_of_ops_each_thread}" << std::endl; - return; - } - int num_of_thread = atoi(my_argv[1]); - int nums_of_ops_each_thread = atoi(my_argv[2]); - std::vector kvdbptrs; - for (int i= 0; i < num_of_thread; i++) { - kvdbptrs.push_back(std::make_shared()); - kvdbptrs[i]->CreateDB(); - } - std::vector tarr; - for (int i = 0; i< num_of_thread; i++) { - tarr.push_back(std::thread(db_thread_test, kvdbptrs[i], nums_of_ops_each_thread)); - } - for (int i = 0; i< num_of_thread; i++) { - tarr[i].join(); - } + if (my_argc != 3) { + std::cerr << "illegal input! should be db_thread ${num_of_thread} " + "${num_of_ops_each_thread}" + << std::endl; return; + } + int num_of_thread = atoi(my_argv[1]); + int nums_of_ops_each_thread = atoi(my_argv[2]); + std::vector kvdbptrs; + for (int i = 0; i < num_of_thread; i++) { + kvdbptrs.push_back(std::make_shared()); + kvdbptrs[i]->CreateDB(); + } + std::vector tarr; + for (int i = 0; i < num_of_thread; i++) { + tarr.push_back( + std::thread(db_thread_test, kvdbptrs[i], nums_of_ops_each_thread)); + } + for (int i = 0; i < num_of_thread; i++) { + tarr[i].join(); + } + return; } - - int main(int argc, char** argv) { - my_argc = argc; - my_argv = argv; - ::testing::InitGoogleTest(&argc, argv); - return RUN_ALL_TESTS(); + my_argc = argc; + my_argv = argv; + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); } - diff --git a/kvdb/src/gtest_kvdb.cpp b/kvdb/src/gtest_kvdb.cpp index 4e7d0aca5ae778a17ed5a4040f6424ee96579c47..69113d70d9dbc5910e9074ca05d9e3b5e7b77776 100644 --- a/kvdb/src/gtest_kvdb.cpp +++ b/kvdb/src/gtest_kvdb.cpp @@ -12,32 +12,29 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "kvdb/rocksdb_impl.h" -#include "kvdb/kvdb_impl.h" -#include "kvdb/paddle_rocksdb.h" #include +#include +#include #include #include -#include -#include #include +#include "kvdb/kvdb_impl.h" +#include "kvdb/paddle_rocksdb.h" +#include "kvdb/rocksdb_impl.h" class KVDBTest : public ::testing::Test { -protected: - void SetUp() override{ - - } - - static void SetUpTestCase() { - kvdb = std::make_shared(); - dict_reader = std::make_shared(); - param_dict = std::make_shared(); - } - - static AbsKVDBPtr kvdb; - static FileReaderPtr dict_reader; - static ParamDictPtr param_dict; - static ParamDictMgr dict_mgr; - + protected: + void SetUp() override {} + + static void SetUpTestCase() { + kvdb = std::make_shared(); + dict_reader = std::make_shared(); + param_dict = std::make_shared(); + } + + static AbsKVDBPtr kvdb; + static FileReaderPtr dict_reader; + static ParamDictPtr param_dict; + static ParamDictMgr dict_mgr; }; AbsKVDBPtr KVDBTest::kvdb; FileReaderPtr KVDBTest::dict_reader; @@ -48,116 +45,117 @@ void GenerateTestIn(std::string); void UpdateTestIn(std::string); TEST_F(KVDBTest, AbstractKVDB_Unit_Test) { - kvdb->CreateDB(); - kvdb->SetDBName("test_kvdb"); - for (int i = 0; i < 100; i++) { - kvdb->Set(std::to_string(i), std::to_string(i * 2)); - } - for (int i = 0; i < 100; i++) { - std::string val = kvdb->Get(std::to_string(i)); - ASSERT_EQ(val, std::to_string(i * 2)); - } + kvdb->CreateDB(); + kvdb->SetDBName("test_kvdb"); + for (int i = 0; i < 100; i++) { + kvdb->Set(std::to_string(i), std::to_string(i * 2)); + } + for (int i = 0; i < 100; i++) { + std::string val = kvdb->Get(std::to_string(i)); + ASSERT_EQ(val, std::to_string(i * 2)); + } } TEST_F(KVDBTest, FileReader_Unit_Test) { - std::string test_in_filename = "abs_dict_reader_test_in.txt"; - GenerateTestIn(test_in_filename); - dict_reader->SetFileName(test_in_filename); - - std::string md5_1 = dict_reader->GetMD5(); - std::chrono::system_clock::time_point timestamp_1 = dict_reader->GetTimeStamp(); - - std::string md5_2 = dict_reader->GetMD5(); - std::chrono::system_clock::time_point timestamp_2 = dict_reader->GetTimeStamp(); - - ASSERT_EQ(md5_1, md5_2); - ASSERT_EQ(timestamp_1, timestamp_2); - - UpdateTestIn(test_in_filename); - - std::string md5_3 = dict_reader->GetMD5(); - std::chrono::system_clock::time_point timestamp_3 = dict_reader->GetTimeStamp(); - - ASSERT_NE(md5_2, md5_3); - ASSERT_NE(timestamp_2, timestamp_3); + std::string test_in_filename = "abs_dict_reader_test_in.txt"; + GenerateTestIn(test_in_filename); + dict_reader->SetFileName(test_in_filename); + + std::string md5_1 = dict_reader->GetMD5(); + std::chrono::system_clock::time_point timestamp_1 = + dict_reader->GetTimeStamp(); + + std::string md5_2 = dict_reader->GetMD5(); + std::chrono::system_clock::time_point timestamp_2 = + dict_reader->GetTimeStamp(); + + ASSERT_EQ(md5_1, md5_2); + ASSERT_EQ(timestamp_1, timestamp_2); + + UpdateTestIn(test_in_filename); + + std::string md5_3 = dict_reader->GetMD5(); + std::chrono::system_clock::time_point timestamp_3 = + dict_reader->GetTimeStamp(); + + ASSERT_NE(md5_2, md5_3); + ASSERT_NE(timestamp_2, timestamp_3); } #include TEST_F(KVDBTest, ParamDict_Unit_Test) { - std::string test_in_filename = "abs_dict_reader_test_in.txt"; - param_dict->SetFileReaderLst({test_in_filename}); - param_dict->SetReader( - [] (std::string text) { - auto split = [](const std::string& s, - std::vector& sv, - const char* delim = " ") { - sv.clear(); - char* buffer = new char[s.size() + 1]; - std::copy(s.begin(), s.end(), buffer); - char* p = strtok(buffer, delim); - do { - sv.push_back(p); - } while ((p = strtok(NULL, delim))); - return; - }; - std::vector text_split; - split(text, text_split, " "); - std::string key = text_split[0]; - text_split.erase(text_split.begin()); - return make_pair(key, text_split); - }); - param_dict->CreateKVDB(); - GenerateTestIn(test_in_filename); - - param_dict->UpdateBaseModel(); - - std::this_thread::sleep_for(std::chrono::seconds(2)); - - std::vector test_vec = param_dict->GetSparseValue("1", ""); - - ASSERT_LT(fabs(test_vec[0] - 1.0), 1e-2); - - UpdateTestIn(test_in_filename); - param_dict->UpdateDeltaModel(); + std::string test_in_filename = "abs_dict_reader_test_in.txt"; + param_dict->SetFileReaderLst({test_in_filename}); + param_dict->SetReader([](std::string text) { + auto split = [](const std::string& s, + std::vector& sv, + const char* delim = " ") { + sv.clear(); + char* buffer = new char[s.size() + 1]; + std::copy(s.begin(), s.end(), buffer); + char* p = strtok(buffer, delim); + do { + sv.push_back(p); + } while ((p = strtok(NULL, delim))); + return; + }; + std::vector text_split; + split(text, text_split, " "); + std::string key = text_split[0]; + text_split.erase(text_split.begin()); + return make_pair(key, text_split); + }); + param_dict->CreateKVDB(); + GenerateTestIn(test_in_filename); + + param_dict->UpdateBaseModel(); + + std::this_thread::sleep_for(std::chrono::seconds(2)); + + std::vector test_vec = param_dict->GetSparseValue("1", ""); + + ASSERT_LT(fabs(test_vec[0] - 1.0), 1e-2); + + UpdateTestIn(test_in_filename); + param_dict->UpdateDeltaModel(); } void GenerateTestIn(std::string filename) { - std::ifstream in_file(filename); - if (in_file.good()) { - in_file.close(); - std::string cmd = "rm -rf "+ filename; - system(cmd.c_str()); + std::ifstream in_file(filename); + if (in_file.good()) { + in_file.close(); + std::string cmd = "rm -rf " + filename; + system(cmd.c_str()); + } + std::ofstream out_file(filename); + for (size_t i = 0; i < 100000; i++) { + out_file << i << " " << i << " "; + for (size_t j = 0; j < 3; j++) { + out_file << i << " "; } - std::ofstream out_file(filename); - for (size_t i = 0; i < 100000; i++) { - out_file << i << " " << i << " "; - for (size_t j = 0; j < 3; j++) { - out_file << i << " "; - } - out_file << std::endl; - } - out_file.close(); + out_file << std::endl; + } + out_file.close(); } void UpdateTestIn(std::string filename) { - std::ifstream in_file(filename); - if (in_file.good()) { - in_file.close(); - std::string cmd = "rm -rf " + filename; - system(cmd.c_str()); - } - std::ofstream out_file(filename); - for (size_t i = 0; i < 10000; i++) { - out_file << i << " " << i << " "; - for (size_t j = 0; j < 3; j++) { - out_file << i + 1 << " "; - } - out_file << std::endl; + std::ifstream in_file(filename); + if (in_file.good()) { + in_file.close(); + std::string cmd = "rm -rf " + filename; + system(cmd.c_str()); + } + std::ofstream out_file(filename); + for (size_t i = 0; i < 10000; i++) { + out_file << i << " " << i << " "; + for (size_t j = 0; j < 3; j++) { + out_file << i + 1 << " "; } - out_file.close(); + out_file << std::endl; + } + out_file.close(); } int main(int argc, char** argv) { - ::testing::InitGoogleTest(&argc, argv); - return RUN_ALL_TESTS(); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); } - diff --git a/kvdb/src/mock_param_dict_impl.cpp b/kvdb/src/mock_param_dict_impl.cpp index e1763b2b6bd1b668a3a2d3baa25ccdbbd63e95fe..b85941c2d96f3fa3fc34cabd328adf6b5084c388 100644 --- a/kvdb/src/mock_param_dict_impl.cpp +++ b/kvdb/src/mock_param_dict_impl.cpp @@ -12,139 +12,132 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "kvdb/rocksdb_impl.h" -#include -#include -#include #include +#include +#include #include +#include +#include "kvdb/rocksdb_impl.h" std::vector ParamDict::GetDictReaderLst() { - return this->file_reader_lst_; + return this->file_reader_lst_; } void ParamDict::SetFileReaderLst(std::vector lst) { - for (size_t i = 0; i < lst.size(); i++) { - FileReaderPtr fr = std::make_shared(); - fr->SetFileName(lst[i]); - this->file_reader_lst_.push_back(fr); - } + for (size_t i = 0; i < lst.size(); i++) { + FileReaderPtr fr = std::make_shared(); + fr->SetFileName(lst[i]); + this->file_reader_lst_.push_back(fr); + } } -std::vector ParamDict::GetSparseValue(std::string feasign, std::string slot) { - auto BytesToFloat = [](uint8_t* byteArray){ - return *((float*)byteArray); - }; - //TODO: the concatation of feasign and slot is TBD. - std::string result = front_db->Get(feasign + slot); - std::vector value; - if (result == "NOT_FOUND") - return value; - uint8_t* raw_values_ptr = reinterpret_cast(&result[0]); - for (size_t i = 0; i < result.size(); i += 4) { - float temp = BytesToFloat(raw_values_ptr + i); - value.push_back(temp); - } - return value; +std::vector ParamDict::GetSparseValue(std::string feasign, + std::string slot) { + auto BytesToFloat = [](uint8_t* byte_array) { return *((float*)byte_array); }; + // TODO: the concatation of feasign and slot is TBD. + std::string result = front_db->Get(feasign + slot); + std::vector value; + if (result == "NOT_FOUND") return value; + uint8_t* raw_values_ptr = reinterpret_cast(&result[0]); + for (size_t i = 0; i < result.size(); i += sizeof(float)) { + float temp = BytesToFloat(raw_values_ptr + i); + value.push_back(temp); + } + return value; } -void ParamDict::SetReader(std::function(std::string)> func) { - read_func_ = func; +void ParamDict::SetReader( + std::function(std::string)> func) { + read_func_ = func; } std::vector ParamDict::GetSparseValue(int64_t feasign, int64_t slot) { - return this->GetSparseValue(std::to_string(feasign), std::to_string(slot)); + return this->GetSparseValue(std::to_string(feasign), std::to_string(slot)); } -bool ParamDict::InsertSparseValue(int64_t feasign, int64_t slot, const std::vector& values) { - return this->InsertSparseValue(std::to_string(feasign), std::to_string(slot), values); +bool ParamDict::InsertSparseValue(int64_t feasign, + int64_t slot, + const std::vector& values) { + return this->InsertSparseValue( + std::to_string(feasign), std::to_string(slot), values); } -bool ParamDict::InsertSparseValue(std::string feasign, std::string slot, const std::vector& values) { - auto FloatToBytes = [](float fvalue, uint8_t *arr){ - unsigned char *pf = nullptr; - unsigned char *px = nullptr; - unsigned char i = 0; - pf =(unsigned char *)&fvalue; - px = arr; - for (i = 0; i < 4; i++) - { - *(px+i)=*(pf+i); - } - }; - - std::string key = feasign + slot; - uint8_t* values_ptr = new uint8_t[values.size() * 4]; - std::string value; - for (size_t i = 0; i < values.size(); i++) { - FloatToBytes(values[i], values_ptr + 4 * i); +bool ParamDict::InsertSparseValue(std::string feasign, + std::string slot, + const std::vector& values) { + auto FloatToBytes = [](float fvalue, uint8_t* arr) { + unsigned char* pf = nullptr; + unsigned char* px = nullptr; + unsigned char i = 0; + pf = (unsigned char*)&fvalue; + px = arr; + for (i = 0; i < sizeof(float); i++) { + *(px + i) = *(pf + i); } - char* raw_values_ptr = reinterpret_cast(values_ptr); - for (size_t i = 0; i < values.size()*4; i++) { - value.push_back(raw_values_ptr[i]); - } - back_db->Set(key, value); -//TODO: change stateless to stateful - return true; + }; + + std::string key = feasign + slot; + uint8_t* values_ptr = new uint8_t[values.size() * sizeof(float)]; + std::string value; + for (size_t i = 0; i < values.size(); i++) { + FloatToBytes(values[i], values_ptr + sizeof(float) * i); + } + char* raw_values_ptr = reinterpret_cast(values_ptr); + for (size_t i = 0; i < values.size() * sizeof(float); i++) { + value.push_back(raw_values_ptr[i]); + } + back_db->Set(key, value); + // TODO: change stateless to stateful + return true; } void ParamDict::UpdateBaseModel() { - auto is_number = [] (const std::string& s) - { - return !s.empty() && std::find_if(s.begin(), - s.end(), [](char c) { return !std::isdigit(c); }) == s.end(); - }; - std::thread t([&] () { - for (FileReaderPtr file_reader: this->file_reader_lst_) { - std::string line; - std::ifstream infile(file_reader->GetFileName()); - if (infile.is_open()) { - while (getline(infile, line)) { - std::pair kvpair = read_func_(line); - std::vector nums; - for (size_t i = 0; i < kvpair.second.size(); i++) { - if (is_number(kvpair.second[i])) { - nums.push_back(std::stof(kvpair.second[i])); - } - } - this->InsertSparseValue(kvpair.first, "", nums); - } + auto is_number = [](const std::string& s) { + return !s.empty() && std::find_if(s.begin(), s.end(), [](char c) { + return !std::isdigit(c); + }) == s.end(); + }; + std::thread t([&]() { + for (FileReaderPtr file_reader : this->file_reader_lst_) { + std::string line; + std::ifstream infile(file_reader->GetFileName()); + if (infile.is_open()) { + while (getline(infile, line)) { + std::pair kvpair = read_func_(line); + std::vector nums; + for (size_t i = 0; i < kvpair.second.size(); i++) { + if (is_number(kvpair.second[i])) { + nums.push_back(std::stof(kvpair.second[i])); } - infile.close(); + } + this->InsertSparseValue(kvpair.first, "", nums); } - AbsKVDBPtr temp = front_db; - front_db = back_db; - back_db = temp; - }); - t.detach(); + } + infile.close(); + } + AbsKVDBPtr temp = front_db; + front_db = back_db; + back_db = temp; + }); + t.detach(); } +void ParamDict::UpdateDeltaModel() { UpdateBaseModel(); } -void ParamDict::UpdateDeltaModel() { - UpdateBaseModel(); -} - -std::pair ParamDict::GetKVDB() { - return {front_db, back_db}; +std::pair ParamDict::GetKVDB() { + return {front_db, back_db}; } void ParamDict::SetKVDB(std::pair kvdbs) { - this->front_db = kvdbs.first; - this->back_db = kvdbs.second; + this->front_db = kvdbs.first; + this->back_db = kvdbs.second; } void ParamDict::CreateKVDB() { - this->front_db = std::make_shared(); - this->back_db = std::make_shared(); - this->front_db->CreateDB(); - this->back_db->CreateDB(); + this->front_db = std::make_shared(); + this->back_db = std::make_shared(); + this->front_db->CreateDB(); + this->back_db->CreateDB(); } -ParamDict::~ParamDict() { - -} - - - - - +ParamDict::~ParamDict() {} diff --git a/kvdb/src/paddle_rocksdb.cpp b/kvdb/src/paddle_rocksdb.cpp index 55728ba65285b04ed09708ffc1de179522d3f235..4314b91c91c35d990c07eea1379e9e9f27ed7ea0 100644 --- a/kvdb/src/paddle_rocksdb.cpp +++ b/kvdb/src/paddle_rocksdb.cpp @@ -15,39 +15,40 @@ #include "kvdb/paddle_rocksdb.h" RocksDBWrapper::RocksDBWrapper(std::string db_name) { - rocksdb::Options options; - options.create_if_missing = true; - db_name_ = db_name; - db_ = nullptr; - rocksdb::Status s = rocksdb::DB::Open(options, db_name, &db_); - return; + rocksdb::Options options; + options.create_if_missing = true; + db_name_ = db_name; + db_ = nullptr; + rocksdb::Status s = rocksdb::DB::Open(options, db_name, &db_); + return; } std::string RocksDBWrapper::Get(std::string key) { - rocksdb::ReadOptions options; - options.verify_checksums = true; - std::string result; - rocksdb::Status s = db_->Get(options, key, &result); - if (s.IsNotFound()) { - result = "NOT_FOUND"; - } - return result; + rocksdb::ReadOptions options; + options.verify_checksums = true; + std::string result; + rocksdb::Status s = db_->Get(options, key, &result); + if (s.IsNotFound()) { + result = "NOT_FOUND"; + } + return result; } bool RocksDBWrapper::Put(std::string key, std::string value) { - rocksdb::WriteOptions options; - rocksdb::Status s = db_->Put(options, key, value); - if (s.ok()) { - return true; - } else { - return false; - } + rocksdb::WriteOptions options; + rocksdb::Status s = db_->Put(options, key, value); + if (s.ok()) { + return true; + } else { + return false; + } } void RocksDBWrapper::SetDBName(std::string db_name) { - this->db_name_ = db_name; + this->db_name_ = db_name; } -std::shared_ptr RocksDBWrapper::RocksDBWrapperFactory(std::string db_name) { - return std::make_shared(db_name); +std::shared_ptr RocksDBWrapper::RocksDBWrapperFactory( + std::string db_name) { + return std::make_shared(db_name); } diff --git a/kvdb/src/param_dict_mgr_impl.cpp b/kvdb/src/param_dict_mgr_impl.cpp index 67fd8744fe8dfb0ebe028650506432e5f5ac3186..37dbd99e3a3d83ec37957c95f5fa5351dfe1205b 100644 --- a/kvdb/src/param_dict_mgr_impl.cpp +++ b/kvdb/src/param_dict_mgr_impl.cpp @@ -15,14 +15,14 @@ #include "kvdb/kvdb_impl.h" void ParamDictMgr::UpdateAll() { - for (auto it = this->ParamDictMap.begin(); it!= this->ParamDictMap.end(); ++it) { - it->second->UpdateBaseModel(); - } - + for (auto it = this->ParamDictMap.begin(); it != this->ParamDictMap.end(); + ++it) { + it->second->UpdateBaseModel(); + } } void ParamDictMgr::InsertParamDict(std::string key, ParamDictPtr value) { - this->ParamDictMap.insert(std::make_pair(key, value)); + this->ParamDictMap.insert(std::make_pair(key, value)); } AbstractKVDB::~AbstractKVDB() {} diff --git a/kvdb/src/rockskvdb_impl.cpp b/kvdb/src/rockskvdb_impl.cpp index 08371abf337878414c1617aad5bbb2ac63aa0a11..d6d4746804efb489169e32c2edb4e5fff3a718be 100644 --- a/kvdb/src/rockskvdb_impl.cpp +++ b/kvdb/src/rockskvdb_impl.cpp @@ -16,29 +16,22 @@ int RocksKVDB::db_count; void RocksKVDB::CreateDB() { - this->db_ = RocksDBWrapper::RocksDBWrapperFactory("RocksDB_" + std::to_string(RocksKVDB::db_count)); - RocksKVDB::db_count ++; - return; + this->db_ = RocksDBWrapper::RocksDBWrapperFactory( + "RocksDB_" + std::to_string(RocksKVDB::db_count)); + RocksKVDB::db_count++; + return; } void RocksKVDB::SetDBName(std::string db_name) { - this->db_->SetDBName(db_name); - return; + this->db_->SetDBName(db_name); + return; } void RocksKVDB::Set(std::string key, std::string value) { - this->db_->Put(key, value); - return; + this->db_->Put(key, value); + return; } -std::string RocksKVDB::Get(std::string key) { - return this->db_->Get(key); -} - -RocksKVDB::~RocksKVDB() { - -} - - - +std::string RocksKVDB::Get(std::string key) { return this->db_->Get(key); } +RocksKVDB::~RocksKVDB() {} diff --git a/kvdb/src/test_rocksdb.cpp b/kvdb/src/test_rocksdb.cpp index ca1d714ac412e63d0f1c1e5e0b1ac0c75efc618a..300c8a9deac3b08d2a3b2cabf0ee8056dd0d6767 100644 --- a/kvdb/src/test_rocksdb.cpp +++ b/kvdb/src/test_rocksdb.cpp @@ -12,37 +12,34 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "kvdb/rocksdb_impl.h" -#include "kvdb/paddle_rocksdb.h" #include +#include "kvdb/paddle_rocksdb.h" +#include "kvdb/rocksdb_impl.h" void test_rockskvdb() { - RocksKVDB db; - db.CreateDB(); - db.SetDBName("Sparse Matrix"); - db.Set("1", "One"); - std::cout << db.Get("1") << std::endl; - return ; + RocksKVDB db; + db.CreateDB(); + db.SetDBName("Sparse Matrix"); + db.Set("1", "One"); + std::cout << db.Get("1") << std::endl; + return; } void test_rocksdbwrapper() { - std::shared_ptr db = RocksDBWrapper::RocksDBWrapperFactory("TEST"); - for (size_t i = 0; i < 1000; i++) { - db->Put(std::to_string(i), std::to_string(i * 2)); - } - for (size_t i = 0; i < 1000; i++) { - std::string res = db->Get(std::to_string(i)); - std::cout << res << " "; - } - std::cout << std::endl; + std::shared_ptr db = + RocksDBWrapper::RocksDBWrapperFactory("TEST"); + for (size_t i = 0; i < 1000; i++) { + db->Put(std::to_string(i), std::to_string(i * 2)); + } + for (size_t i = 0; i < 1000; i++) { + std::string res = db->Get(std::to_string(i)); + std::cout << res << " "; + } + std::cout << std::endl; } #ifdef RAW_TEST int main() { - test_rockskvdb(); - test_rocksdbwrapper(); + test_rockskvdb(); + test_rocksdbwrapper(); } #endif - - - - diff --git a/pdcodegen/src/pdcodegen.cpp b/pdcodegen/src/pdcodegen.cpp index ba22738a234afe4df72784d45f6404b70183fb3e..1896caeb044f86c7a87897bbb22dbc8d17f1ac2d 100644 --- a/pdcodegen/src/pdcodegen.cpp +++ b/pdcodegen/src/pdcodegen.cpp @@ -484,7 +484,8 @@ class PdsCodeGenerator : public CodeGenerator { "response);\n" "}\n" "tt.stop();\n" - "if (ret.flags != baidu::rpc::SKIP_SUB_CHANNEL && ret.method != NULL) {\n" + "if (ret.flags != baidu::rpc::SKIP_SUB_CHANNEL && ret.method != " + "NULL) {\n" " _stub_handler->update_latency(tt.u_elapsed(), \"pack_map\");\n" "}\n" "return ret;\n"); @@ -498,7 +499,8 @@ class PdsCodeGenerator : public CodeGenerator { //////////////////////////////////////////////////////////////// printer->Print( - "class $name$_StubResponseMerger : public baidu::rpc::ResponseMerger {\n" + "class $name$_StubResponseMerger : public baidu::rpc::ResponseMerger " + "{\n" "private:\n" " uint32_t _package_size;\n" " baidu::paddle_serving::sdk_cpp::Stub* _stub_handler;\n" @@ -600,7 +602,8 @@ class PdsCodeGenerator : public CodeGenerator { "}\n"); "LOG(INFO) \n" " << \"[default] Succ map, channel_index: \" << channel_index;\n"; - printer->Print("return baidu::rpc::SubCall(method, request, cur_res, 0);\n"); + printer->Print( + "return baidu::rpc::SubCall(method, request, cur_res, 0);\n"); return true; } bool generate_paddle_serving_stub_default_merger( diff --git a/predictor/CMakeLists.txt b/predictor/CMakeLists.txt index 652211448f01c830ee956406cdd2ba07c91df635..766052103313bc69b6d1ba0edf66194df08da3b4 100644 --- a/predictor/CMakeLists.txt +++ b/predictor/CMakeLists.txt @@ -37,4 +37,3 @@ install(FILES ${CMAKE_CURRENT_LIST_DIR}/mempool/mempool.h DESTINATION ${PADDLE_SERVING_INSTALL_DIR}/include/predictor/mempool) install(FILES ${CMAKE_CURRENT_LIST_DIR}/op/op.h DESTINATION ${PADDLE_SERVING_INSTALL_DIR}/include/predictor/op) - diff --git a/predictor/framework/memory.cpp b/predictor/framework/memory.cpp index 78a4643468eeca9f1da925a67f1e1330b89927a3..28b73c166efc7b5d2073263ca6b841e78aa124f8 100644 --- a/predictor/framework/memory.cpp +++ b/predictor/framework/memory.cpp @@ -20,11 +20,10 @@ namespace paddle_serving { namespace predictor { struct MempoolRegion { - MempoolRegion(im::fugue::memory::Region *region, - im::Mempool *mempool) : - _region(region), _mempool(mempool){} - im::fugue::memory::Region *region() {return _region;} - im::Mempool *mempool() {return _mempool;} + MempoolRegion(im::fugue::memory::Region* region, im::Mempool* mempool) + : _region(region), _mempool(mempool) {} + im::fugue::memory::Region* region() { return _region; } + im::Mempool* mempool() { return _mempool; } im::fugue::memory::Region* _region; im::Mempool* _mempool; @@ -54,10 +53,10 @@ int MempoolWrapper::initialize() { } int MempoolWrapper::thread_initialize() { - im::fugue::memory::Region *region = new im::fugue::memory::Region(); + im::fugue::memory::Region* region = new im::fugue::memory::Region(); region->init(); im::Mempool* mempool = new (std::nothrow) im::Mempool(region); - MempoolRegion *mempool_region = new MempoolRegion(region, mempool); + MempoolRegion* mempool_region = new MempoolRegion(region, mempool); if (mempool == NULL) { LOG(ERROR) << "Failed create thread mempool"; return -1; @@ -76,7 +75,8 @@ int MempoolWrapper::thread_initialize() { } int MempoolWrapper::thread_clear() { - MempoolRegion* mempool_region = (MempoolRegion*)THREAD_GETSPECIFIC(_bspec_key); + MempoolRegion* mempool_region = + (MempoolRegion*)THREAD_GETSPECIFIC(_bspec_key); if (mempool_region == NULL) { LOG(WARNING) << "THREAD_GETSPECIFIC() returned NULL"; return -1; @@ -91,7 +91,8 @@ int MempoolWrapper::thread_clear() { } void* MempoolWrapper::malloc(size_t size) { - MempoolRegion* mempool_region = (MempoolRegion*)THREAD_GETSPECIFIC(_bspec_key); + MempoolRegion* mempool_region = + (MempoolRegion*)THREAD_GETSPECIFIC(_bspec_key); if (mempool_region == NULL) { LOG(WARNING) << "THREAD_GETSPECIFIC() returned NULL"; return NULL; diff --git a/predictor/framework/memory.h b/predictor/framework/memory.h index c096bc2c6b2d96ff2bd93daa804bac007cef6295..4f60eb2ef5e3036f0d07637a97874d68bec23ac7 100644 --- a/predictor/framework/memory.h +++ b/predictor/framework/memory.h @@ -39,7 +39,7 @@ class MempoolWrapper { void* malloc(size_t size); private: - //im::fugue::memory::Region _region; + // im::fugue::memory::Region _region; THREAD_KEY_T _bspec_key; }; diff --git a/predictor/framework/predictor_metric.h b/predictor/framework/predictor_metric.h index 92fb057e41dfa2cc5934d404dff268bc7cff7198..e8b54f87194577daf850c8a9a4d8ba8e99e57898 100644 --- a/predictor/framework/predictor_metric.h +++ b/predictor/framework/predictor_metric.h @@ -24,7 +24,7 @@ #include // BAIDU_SCOPED_LOCK #endif -#include // bvar +#include // bvar #include #ifdef BCLOUD diff --git a/predictor/framework/resource.cpp b/predictor/framework/resource.cpp index 9b545c9150a4e4f4ed3f2a548bdfe1e288348730..a2ed40cb4fdde9778c23755e2e1dddc0624c15e2 100644 --- a/predictor/framework/resource.cpp +++ b/predictor/framework/resource.cpp @@ -36,13 +36,9 @@ DynamicResource::DynamicResource() {} DynamicResource::~DynamicResource() {} -int DynamicResource::initialize() { - return 0; -} +int DynamicResource::initialize() { return 0; } -std::shared_ptr Resource::getDB() { - return db; -} +std::shared_ptr Resource::getDB() { return db; } int DynamicResource::clear() { return 0; } @@ -86,7 +82,7 @@ int Resource::initialize(const std::string& path, const std::string& file) { LOG(ERROR) << "unable to create tls_bthread_key of thrd_data"; return -1; } - //init rocksDB instance + // init rocksDB instance if (db.get() == nullptr) { db = RocksDBWrapper::RocksDBWrapperFactory("kvdb"); } diff --git a/predictor/framework/resource.h b/predictor/framework/resource.h index 767b71d40c1d4478ab2ac2d2e32aedba0335621b..a9dde0ab0342655a979e759d09fdaaa1bf13c631 100644 --- a/predictor/framework/resource.h +++ b/predictor/framework/resource.h @@ -14,9 +14,9 @@ #pragma once #include +#include "kvdb/paddle_rocksdb.h" #include "predictor/common/inner_common.h" #include "predictor/framework/memory.h" -#include "kvdb/paddle_rocksdb.h" namespace baidu { namespace paddle_serving { @@ -31,7 +31,6 @@ struct DynamicResource { int initialize(); int clear(); - }; class Resource { @@ -65,7 +64,7 @@ class Resource { private: int thread_finalize() { return 0; } std::shared_ptr db; - + THREAD_KEY_T _tls_bspec_key; }; diff --git a/predictor/framework/server.cpp b/predictor/framework/server.cpp index a96f63f2847cf2518b5a8e907c161b75f92a125d..4545d4a07cc1d990dad30df245bf77a4f2c212e6 100644 --- a/predictor/framework/server.cpp +++ b/predictor/framework/server.cpp @@ -16,7 +16,7 @@ #ifdef BCLOUD #include // NovaServiceAdaptor #include // NsheadMcpackAdaptor -#include // PublicPbrpcServiceAdaptor +#include // PublicPbrpcServiceAdaptor #else #include // NovaServiceAdaptor #include // NsheadMcpackAdaptor diff --git a/predictor/proto/builtin_format.proto b/predictor/proto/builtin_format.proto index c2abb4fe882809e57ec5901bfebeccf23383564b..6b92a9e6a1d84962d52e4471c7d6d8cecf2fa53e 100644 --- a/predictor/proto/builtin_format.proto +++ b/predictor/proto/builtin_format.proto @@ -17,16 +17,11 @@ package baidu.paddle_serving.predictor.format; // echo kvdb formant message KVDBReq { - required string op = 1; - required string key = 2; - optional string value = 3; - + required string op = 1; + required string key = 2; + optional string value = 3; }; -message KVDBRes{ - required string value = 2; -}; - - +message KVDBRes { required string value = 2; }; // dense format message DenseInstance { repeated float features = 1; }; diff --git a/predictor/src/pdserving.cpp b/predictor/src/pdserving.cpp index 3fc1eed7b632b68be1c5c9f39db63f260319d48c..46c9b46841ba6892cc2d598aa0a567f287c7bde7 100644 --- a/predictor/src/pdserving.cpp +++ b/predictor/src/pdserving.cpp @@ -143,7 +143,8 @@ int main(int argc, char** argv) { std::string filename(argv[0]); filename = filename.substr(filename.find_last_of('/') + 1); - settings.log_file = strdup((std::string("./log/") + filename + ".log").c_str()); + settings.log_file = + strdup((std::string("./log/") + filename + ".log").c_str()); settings.delete_old = logging::DELETE_OLD_LOG_FILE; logging::InitLogging(settings); diff --git a/sdk-cpp/include/common.h b/sdk-cpp/include/common.h index a29d5f104f55b1dc3c3bc7eb54a2655fdbf9f61b..19a37650c5e037044598da506f2c8576a283b3e1 100644 --- a/sdk-cpp/include/common.h +++ b/sdk-cpp/include/common.h @@ -31,11 +31,11 @@ #include "baidu/rpc/channel.h" #include "baidu/rpc/parallel_channel.h" #include "baidu/rpc/traceprintf.h" -#include "bthread.h" -#include "base/logging.h" #include "base/comlog_sink.h" +#include "base/logging.h" #include "base/object_pool.h" #include "base/time.h" +#include "bthread.h" #else #include "brpc/channel.h" #include "brpc/parallel_channel.h" diff --git a/sdk-cpp/proto/echo_kvdb_service.proto b/sdk-cpp/proto/echo_kvdb_service.proto index 332e5c4ad9d7a710fdcc571059d616457866f593..2c57953f51c48ee05af4b3fa1dbaf03bb3c6bb22 100644 --- a/sdk-cpp/proto/echo_kvdb_service.proto +++ b/sdk-cpp/proto/echo_kvdb_service.proto @@ -20,15 +20,15 @@ package baidu.paddle_serving.predictor.echo_kvdb_service; option cc_generic_services = true; message Request { - repeated baidu.paddle_serving.predictor.format.KVDBReq reqs = 1; + repeated baidu.paddle_serving.predictor.format.KVDBReq reqs = 1; }; message Response { - repeated baidu.paddle_serving.predictor.format.KVDBRes ress = 1; + repeated baidu.paddle_serving.predictor.format.KVDBRes ress = 1; }; service EchoKVDBService { - rpc inference(Request) returns (Response); - rpc debug(Request) returns (Response); - option (pds.options).generate_stub = true; + rpc inference(Request) returns (Response); + rpc debug(Request) returns (Response); + option (pds.options).generate_stub = true; };