From 149c2459586eb20fadcd39f93932dc3dd13411da Mon Sep 17 00:00:00 2001 From: groot Date: Sun, 28 Apr 2019 14:48:05 +0800 Subject: [PATCH] add rocksdb option Former-commit-id: cebc9569d8dca44c360ac68510427f1447105ddc --- cpp/conf/server_config.yaml | 3 +++ cpp/src/server/ServerConfig.h | 9 ++++++--- cpp/src/server/VecIdMapper.cpp | 5 +++-- cpp/src/server/VecServiceTask.cpp | 8 ++++---- cpp/test_client/src/ClientTest.cpp | 4 ++-- 5 files changed, 18 insertions(+), 11 deletions(-) diff --git a/cpp/conf/server_config.yaml b/cpp/conf/server_config.yaml index 1df05868..d9116183 100644 --- a/cpp/conf/server_config.yaml +++ b/cpp/conf/server_config.yaml @@ -3,9 +3,12 @@ server_config: port: 33001 transfer_protocol: json #optional: binary, compact, json, debug server_mode: thread_pool #optional: simple, thread_pool + +db_config: db_path: /tmp/vecwise db_backend_url: http://127.0.0.1 db_flush_interval: 5 #unit: second + idmapper_max_open_file: 128 log_config: global: diff --git a/cpp/src/server/ServerConfig.h b/cpp/src/server/ServerConfig.h index fbb95b83..b8c5ae8c 100644 --- a/cpp/src/server/ServerConfig.h +++ b/cpp/src/server/ServerConfig.h @@ -19,9 +19,12 @@ static const std::string CONFIG_SERVER_ADDRESS = "address"; static const std::string CONFIG_SERVER_PORT = "port"; static const std::string CONFIG_SERVER_PROTOCOL = "transfer_protocol"; static const std::string CONFIG_SERVER_MODE = "server_mode"; -static const std::string CONFIG_SERVER_DB_URL = "db_backend_url"; -static const std::string CONFIG_SERVER_DB_PATH = "db_path"; -static const std::string CONFIG_SERVER_DB_FLUSH_INTERVAL = "db_flush_interval"; + +static const std::string CONFIG_DB = "db_config"; +static const std::string CONFIG_DB_URL = "db_backend_url"; +static const std::string CONFIG_DB_PATH = "db_path"; +static const std::string CONFIG_DB_FLUSH_INTERVAL = "db_flush_interval"; +static const std::string CONFIG_DB_IDMAPPER_MAX_FILE = "idmapper_max_open_file"; static const std::string CONFIG_LOG = "log_config"; diff --git a/cpp/src/server/VecIdMapper.cpp b/cpp/src/server/VecIdMapper.cpp index bf894cb7..2a50d4db 100644 --- a/cpp/src/server/VecIdMapper.cpp +++ b/cpp/src/server/VecIdMapper.cpp @@ -91,8 +91,8 @@ ServerError SimpleIdMapper::Delete(const std::string& nid) { //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// RocksIdMapper::RocksIdMapper() { - ConfigNode& config = ServerConfig::GetInstance().GetConfig(CONFIG_SERVER); - std::string db_path = config.GetValue(CONFIG_SERVER_DB_PATH); + ConfigNode& config = ServerConfig::GetInstance().GetConfig(CONFIG_DB); + std::string db_path = config.GetValue(CONFIG_DB_PATH); db_path += "/id_mapping"; CommonUtil::CreateDirectory(db_path); @@ -102,6 +102,7 @@ RocksIdMapper::RocksIdMapper() { options.OptimizeLevelStyleCompaction(); // create the DB if it's not already present options.create_if_missing = true; + options.max_open_files = config.GetInt32Value(CONFIG_DB_IDMAPPER_MAX_FILE, 128); // open DB rocksdb::Status s = rocksdb::DB::Open(options, db_path, &db_); diff --git a/cpp/src/server/VecServiceTask.cpp b/cpp/src/server/VecServiceTask.cpp index c8d2907d..53ada256 100644 --- a/cpp/src/server/VecServiceTask.cpp +++ b/cpp/src/server/VecServiceTask.cpp @@ -24,10 +24,10 @@ namespace { public: DBWrapper() { zilliz::vecwise::engine::Options opt; - ConfigNode& config = ServerConfig::GetInstance().GetConfig(CONFIG_SERVER); - opt.meta.backend_uri = config.GetValue(CONFIG_SERVER_DB_URL); - std::string db_path = config.GetValue(CONFIG_SERVER_DB_PATH); - opt.memory_sync_interval = (uint16_t)config.GetInt32Value(CONFIG_SERVER_DB_FLUSH_INTERVAL, 10); + ConfigNode& config = ServerConfig::GetInstance().GetConfig(CONFIG_DB); + opt.meta.backend_uri = config.GetValue(CONFIG_DB_URL); + std::string db_path = config.GetValue(CONFIG_DB_PATH); + opt.memory_sync_interval = (uint16_t)config.GetInt32Value(CONFIG_DB_FLUSH_INTERVAL, 10); opt.meta.path = db_path + "/db"; CommonUtil::CreateDirectory(opt.meta.path); diff --git a/cpp/test_client/src/ClientTest.cpp b/cpp/test_client/src/ClientTest.cpp index 4216bb8e..59d717aa 100644 --- a/cpp/test_client/src/ClientTest.cpp +++ b/cpp/test_client/src/ClientTest.cpp @@ -41,8 +41,8 @@ namespace { int32_t GetFlushInterval() { server::ServerConfig& config = server::ServerConfig::GetInstance(); - server::ConfigNode server_config = config.GetConfig(server::CONFIG_SERVER); - return server_config.GetInt32Value(server::CONFIG_SERVER_DB_FLUSH_INTERVAL); + server::ConfigNode db_config = config.GetConfig(server::CONFIG_DB); + return db_config.GetInt32Value(server::CONFIG_DB_FLUSH_INTERVAL); } std::string GetGroupID() { -- GitLab