From 56291b218d2070b615c4b61151e9fb890bf2918b Mon Sep 17 00:00:00 2001 From: groot Date: Mon, 13 May 2019 15:17:05 +0800 Subject: [PATCH] use rocksdb column family Former-commit-id: 8cbc5d91e02e6bbc0eff1e329b4c76687d2c1f74 --- cpp/conf/server_config.yaml | 16 +++--- cpp/conf/vecwise_engine_log.conf | 12 ++-- cpp/src/server/VecIdMapper.cpp | 81 +++++++++++++++++++++------ cpp/src/server/VecIdMapper.h | 33 +++++------ cpp/unittest/server/idmapper_test.cpp | 10 +++- 5 files changed, 101 insertions(+), 51 deletions(-) diff --git a/cpp/conf/server_config.yaml b/cpp/conf/server_config.yaml index 790bca4b..d9116183 100644 --- a/cpp/conf/server_config.yaml +++ b/cpp/conf/server_config.yaml @@ -1,11 +1,11 @@ server_config: - address: 0.0.0.0 + address: 127.0.0.1 port: 33001 transfer_protocol: json #optional: binary, compact, json, debug server_mode: thread_pool #optional: simple, thread_pool db_config: - db_path: /var/vecwise + db_path: /tmp/vecwise db_backend_url: http://127.0.0.1 db_flush_interval: 5 #unit: second idmapper_max_open_file: 128 @@ -13,7 +13,7 @@ db_config: log_config: global: format: "%datetime | %level | %logger | %msg" - filename: "/var/vecwise/logs/vecwise_engine-%datetime{%h:%m}-global.log" + filename: "/tmp/vecwise/logs/vecwise_engine-%datetime{%h:%m}-global.log" enabled: true to_file: true to_standard_output: true @@ -21,22 +21,22 @@ log_config: performance_tracking: false max_log_file_size: 2097152 # throw log files away after 2mb debug: - filename: "/var/vecwise/logs/vecwise_engine-%datetime{%h:%m}-debug.log" + filename: "/tmp/vecwise/logs/vecwise_engine-%datetime{%h:%m}-debug.log" enabled: true warning: - filename: "/var/vecwise/logs/vecwise_engine-%datetime{%h:%m}-warning.log" + filename: "/tmp/vecwise/logs/vecwise_engine-%datetime{%h:%m}-warning.log" trace: - filename: "/var/vecwise/logs/vecwise_engine-%datetime{%h:%m}-trace.log" + filename: "/tmp/vecwise/logs/vecwise_engine-%datetime{%h:%m}-trace.log" verbose: format: "%datetime{%d/%m/%y} | %level-%vlevel | %msg" to_file: false to_standard_output: true error: enabled: false - filename: "/var/vecwise/logs/vecwise_engine-%datetime{%h:%m}-error.log" + filename: "/tmp/vecwise/logs/vecwise_engine-%datetime{%h:%m}-error.log" fatal: enabled: false - filename: "/var/vecwise/logs/vecwise_engine-%datetime{%h:%m}-fatal.log" + filename: "/tmp/vecwise/logs/vecwise_engine-%datetime{%h:%m}-fatal.log" cache_config: cpu_cache_capacity: 16 # unit: GB diff --git a/cpp/conf/vecwise_engine_log.conf b/cpp/conf/vecwise_engine_log.conf index d0164b41..4935b2fd 100644 --- a/cpp/conf/vecwise_engine_log.conf +++ b/cpp/conf/vecwise_engine_log.conf @@ -1,6 +1,6 @@ * GLOBAL: FORMAT = "%datetime | %level | %logger | %msg" - FILENAME = "/var/vecwise/logs/vecwise_engine-%datetime{%H:%m}-global.log" + FILENAME = "/tmp/vecwise/logs/vecwise_engine-%datetime{%H:%m}-global.log" ENABLED = true TO_FILE = true TO_STANDARD_OUTPUT = true @@ -8,12 +8,12 @@ PERFORMANCE_TRACKING = false MAX_LOG_FILE_SIZE = 2097152 ## Throw log files away after 2MB * DEBUG: - FILENAME = "/var/vecwise/logs/vecwise_engine-%datetime{%H:%m}-debug.log" + FILENAME = "/tmp/vecwise/logs/vecwise_engine-%datetime{%H:%m}-debug.log" ENABLED = true * WARNING: - FILENAME = "/var/vecwise/logs/vecwise_engine-%datetime{%H:%m}-warning.log" + FILENAME = "/tmp/vecwise/logs/vecwise_engine-%datetime{%H:%m}-warning.log" * TRACE: - FILENAME = "/var/vecwise/logs/vecwise_engine-%datetime{%H:%m}-trace.log" + FILENAME = "/tmp/vecwise/logs/vecwise_engine-%datetime{%H:%m}-trace.log" * VERBOSE: FORMAT = "%datetime{%d/%M/%y} | %level-%vlevel | %msg" TO_FILE = false @@ -21,7 +21,7 @@ ## Error logs * ERROR: ENABLED = false - FILENAME = "/var/vecwise/logs/vecwise_engine-%datetime{%H:%m}-error.log" + FILENAME = "/tmp/vecwise/logs/vecwise_engine-%datetime{%H:%m}-error.log" * FATAL: ENABLED = false - FILENAME = "/var/vecwise/logs/vecwise_engine-%datetime{%H:%m}-fatal.log" \ No newline at end of file + FILENAME = "/tmp/vecwise/logs/vecwise_engine-%datetime{%H:%m}-fatal.log" \ No newline at end of file diff --git a/cpp/src/server/VecIdMapper.cpp b/cpp/src/server/VecIdMapper.cpp index 2a50d4db..b0dfcc6d 100644 --- a/cpp/src/server/VecIdMapper.cpp +++ b/cpp/src/server/VecIdMapper.cpp @@ -13,6 +13,8 @@ #include "rocksdb/slice.h" #include "rocksdb/options.h" +#include + namespace zilliz { namespace vecwise { namespace server { @@ -36,26 +38,33 @@ SimpleIdMapper::~SimpleIdMapper() { } -ServerError SimpleIdMapper::Put(const std::string& nid, const std::string& sid) { - ids_[nid] = sid; +//not thread-safe +ServerError SimpleIdMapper::Put(const std::string& nid, const std::string& sid, const std::string& group) { + ID_MAPPING& mapping = id_groups_[group]; + mapping[nid] = sid; return SERVER_SUCCESS; } -ServerError SimpleIdMapper::Put(const std::vector& nid, const std::vector& sid) { +//not thread-safe +ServerError SimpleIdMapper::Put(const std::vector& nid, const std::vector& sid, const std::string& group) { if(nid.size() != sid.size()) { return SERVER_INVALID_ARGUMENT; } + ID_MAPPING& mapping = id_groups_[group]; for(size_t i = 0; i < nid.size(); i++) { - ids_[nid[i]] = sid[i]; + mapping[nid[i]] = sid[i]; } return SERVER_SUCCESS; } -ServerError SimpleIdMapper::Get(const std::string& nid, std::string& sid) const { - auto iter = ids_.find(nid); - if(iter == ids_.end()) { +//not thread-safe +ServerError SimpleIdMapper::Get(const std::string& nid, std::string& sid, const std::string& group) const { + ID_MAPPING& mapping = id_groups_[group]; + + auto iter = mapping.find(nid); + if(iter == mapping.end()) { return SERVER_INVALID_ARGUMENT; } @@ -64,13 +73,16 @@ ServerError SimpleIdMapper::Get(const std::string& nid, std::string& sid) const return SERVER_SUCCESS; } -ServerError SimpleIdMapper::Get(const std::vector& nid, std::vector& sid) const { +//not thread-safe +ServerError SimpleIdMapper::Get(const std::vector& nid, std::vector& sid, const std::string& group) const { sid.clear(); + ID_MAPPING& mapping = id_groups_[group]; + ServerError err = SERVER_SUCCESS; for(size_t i = 0; i < nid.size(); i++) { - auto iter = ids_.find(nid[i]); - if(iter == ids_.end()) { + auto iter = mapping.find(nid[i]); + if(iter == mapping.end()) { sid.push_back(""); SERVER_LOG_ERROR << "ID mapper failed to find id: " << nid[i]; err = SERVER_INVALID_ARGUMENT; @@ -83,8 +95,10 @@ ServerError SimpleIdMapper::Get(const std::vector& nid, std::vector return err; } -ServerError SimpleIdMapper::Delete(const std::string& nid) { - ids_.erase(nid); +//not thread-safe +ServerError SimpleIdMapper::Delete(const std::string& nid, const std::string& group) { + ID_MAPPING& mapping = id_groups_[group]; + mapping.erase(nid); return SERVER_SUCCESS; } @@ -104,12 +118,43 @@ RocksIdMapper::RocksIdMapper() { options.create_if_missing = true; options.max_open_files = config.GetInt32Value(CONFIG_DB_IDMAPPER_MAX_FILE, 128); + //load column families + std::vector column_names; + rocksdb::Status s = rocksdb::DB::ListColumnFamilies(options, db_path, &column_names); + if (!s.ok()) { + SERVER_LOG_ERROR << "ID mapper failed to initialize:" << s.ToString(); + } + + std::vector column_families; + for(auto& column_name : column_names) { + rocksdb::ColumnFamilyDescriptor desc; + desc.name = column_name; + column_families.emplace_back(desc); + } + // open DB - rocksdb::Status s = rocksdb::DB::Open(options, db_path, &db_); + std::vector column_handles; + s = rocksdb::DB::Open(options, db_path, column_families, &column_handles, &db_); if(!s.ok()) { SERVER_LOG_ERROR << "ID mapper failed to initialize:" << s.ToString(); db_ = nullptr; } + + try { + rocksdb::ColumnFamilyHandle *cf; + s = db_->CreateColumnFamily(rocksdb::ColumnFamilyOptions(), "222", &cf); + if (!s.ok()) { + SERVER_LOG_ERROR << "ID mapper failed to initialize:" << s.ToString(); + } + + std::vector column_families; + s = db_->ListColumnFamilies(options, db_path, &column_families); + if (!s.ok()) { + SERVER_LOG_ERROR << "ID mapper failed to initialize:" << s.ToString(); + } + } catch(std::exception& ex) { + std::cout << ex.what() << std::endl; + } } RocksIdMapper::~RocksIdMapper() { if(db_) { @@ -118,7 +163,7 @@ RocksIdMapper::~RocksIdMapper() { } } -ServerError RocksIdMapper::Put(const std::string& nid, const std::string& sid) { +ServerError RocksIdMapper::Put(const std::string& nid, const std::string& sid, const std::string& group) { if(db_ == nullptr) { return SERVER_NULL_POINTER; } @@ -134,7 +179,7 @@ ServerError RocksIdMapper::Put(const std::string& nid, const std::string& sid) { return SERVER_SUCCESS; } -ServerError RocksIdMapper::Put(const std::vector& nid, const std::vector& sid) { +ServerError RocksIdMapper::Put(const std::vector& nid, const std::vector& sid, const std::string& group) { if(nid.size() != sid.size()) { return SERVER_INVALID_ARGUMENT; } @@ -150,7 +195,7 @@ ServerError RocksIdMapper::Put(const std::vector& nid, const std::v return err; } -ServerError RocksIdMapper::Get(const std::string& nid, std::string& sid) const { +ServerError RocksIdMapper::Get(const std::string& nid, std::string& sid, const std::string& group) const { if(db_ == nullptr) { return SERVER_NULL_POINTER; } @@ -165,7 +210,7 @@ ServerError RocksIdMapper::Get(const std::string& nid, std::string& sid) const { return SERVER_SUCCESS; } -ServerError RocksIdMapper::Get(const std::vector& nid, std::vector& sid) const { +ServerError RocksIdMapper::Get(const std::vector& nid, std::vector& sid, const std::string& group) const { sid.clear(); ServerError err = SERVER_SUCCESS; @@ -185,7 +230,7 @@ ServerError RocksIdMapper::Get(const std::vector& nid, std::vector< return err; } -ServerError RocksIdMapper::Delete(const std::string& nid) { +ServerError RocksIdMapper::Delete(const std::string& nid, const std::string& group) { if(db_ == nullptr) { return SERVER_NULL_POINTER; } diff --git a/cpp/src/server/VecIdMapper.h b/cpp/src/server/VecIdMapper.h index b376c126..216218b5 100644 --- a/cpp/src/server/VecIdMapper.h +++ b/cpp/src/server/VecIdMapper.h @@ -25,14 +25,14 @@ public: virtual ~IVecIdMapper(){} - virtual ServerError Put(const std::string& nid, const std::string& sid) = 0; - virtual ServerError Put(const std::vector& nid, const std::vector& sid) = 0; + virtual ServerError Put(const std::string& nid, const std::string& sid, const std::string& group = "") = 0; + virtual ServerError Put(const std::vector& nid, const std::vector& sid, const std::string& group = "") = 0; - virtual ServerError Get(const std::string& nid, std::string& sid) const = 0; + virtual ServerError Get(const std::string& nid, std::string& sid, const std::string& group = "") const = 0; //NOTE: the 'sid' will be cleared at begin of the function - virtual ServerError Get(const std::vector& nid, std::vector& sid) const = 0; + virtual ServerError Get(const std::vector& nid, std::vector& sid, const std::string& group = "") const = 0; - virtual ServerError Delete(const std::string& nid) = 0; + virtual ServerError Delete(const std::string& nid, const std::string& group = "") = 0; }; class SimpleIdMapper : public IVecIdMapper{ @@ -40,16 +40,17 @@ public: SimpleIdMapper(); ~SimpleIdMapper(); - ServerError Put(const std::string& nid, const std::string& sid) override; - ServerError Put(const std::vector& nid, const std::vector& sid) override; + ServerError Put(const std::string& nid, const std::string& sid, const std::string& group = "") override; + ServerError Put(const std::vector& nid, const std::vector& sid, const std::string& group = "") override; - ServerError Get(const std::string& nid, std::string& sid) const override; - ServerError Get(const std::vector& nid, std::vector& sid) const override; + ServerError Get(const std::string& nid, std::string& sid, const std::string& group = "") const override; + ServerError Get(const std::vector& nid, std::vector& sid, const std::string& group = "") const override; - ServerError Delete(const std::string& nid) override; + ServerError Delete(const std::string& nid, const std::string& group = "") override; private: - std::unordered_map ids_; + using ID_MAPPING = std::unordered_map; + mutable std::unordered_map id_groups_; }; class RocksIdMapper : public IVecIdMapper{ @@ -57,13 +58,13 @@ public: RocksIdMapper(); ~RocksIdMapper(); - ServerError Put(const std::string& nid, const std::string& sid) override; - ServerError Put(const std::vector& nid, const std::vector& sid) override; + ServerError Put(const std::string& nid, const std::string& sid, const std::string& group = "") override; + ServerError Put(const std::vector& nid, const std::vector& sid, const std::string& group = "") override; - ServerError Get(const std::string& nid, std::string& sid) const override; - ServerError Get(const std::vector& nid, std::vector& sid) const override; + ServerError Get(const std::string& nid, std::string& sid, const std::string& group = "") const override; + ServerError Get(const std::vector& nid, std::vector& sid, const std::string& group = "") const override; - ServerError Delete(const std::string& nid) override; + ServerError Delete(const std::string& nid, const std::string& group = "") override; private: rocksdb::DB* db_; diff --git a/cpp/unittest/server/idmapper_test.cpp b/cpp/unittest/server/idmapper_test.cpp index 80bbb2e4..d0831c82 100644 --- a/cpp/unittest/server/idmapper_test.cpp +++ b/cpp/unittest/server/idmapper_test.cpp @@ -44,9 +44,13 @@ TEST(IdMapperTest, IDMAPPER_TEST) { nid.clear(); sid.clear(); const int64_t count = 1000000; - for(int64_t i = 0; i < count; i++) { - nid.push_back(std::to_string(i+100000)); - sid.push_back("val_" + std::to_string(i)); + { + server::TimeRecorder rc("prepare id data"); + for (int64_t i = 0; i < count; i++) { + nid.push_back(std::to_string(i + 100000)); + sid.push_back("val_" + std::to_string(i)); + } + rc.Record("done!"); } { -- GitLab