From 41d7500f61ff8139fa82d7cf5fccc56f9bceb368 Mon Sep 17 00:00:00 2001 From: Cai Yudong Date: Sat, 14 Dec 2019 15:11:02 +0800 Subject: [PATCH] support config cli test 2 (#764) * #665 support get/set config via CLI * #665 support get/set config via CLI * #665 add unittest for config CLI * #665 remove config_node_map_ * #665 remove config_node_map_ * #665 fix clang-format * #665 fix clang-format * #665 update changelog * #665 code clean * #665 update API interface * #665 update API interface * #665 handle server status * #665 handle unknown command * #665 update debug log * #665 rollback CmdRequest.cpp --- CHANGELOG.md | 1 + core/src/sdk/grpc/ClientProxy.cpp | 21 ++- core/src/sdk/grpc/ClientProxy.h | 6 + core/src/sdk/include/MilvusApi.h | 26 ++++ core/src/sdk/interface/ConnectionImpl.cpp | 9 ++ core/src/sdk/interface/ConnectionImpl.h | 6 + core/src/server/Config.cpp | 102 ++++++++++++++- core/src/server/Config.h | 12 +- core/unittest/server/test_config.cpp | 150 ++++++++++++++++++++++ 9 files changed, 329 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2ca5b236..8bc28d0c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ Please mark all change in change log and use the issue from GitHub ## Feature - \#343 - Add Opentracing +- \#665 - Support get/set config via CLI ## Improvement diff --git a/core/src/sdk/grpc/ClientProxy.cpp b/core/src/sdk/grpc/ClientProxy.cpp index c6a087cf..afa90c95 100644 --- a/core/src/sdk/grpc/ClientProxy.cpp +++ b/core/src/sdk/grpc/ClientProxy.cpp @@ -334,7 +334,7 @@ ClientProxy::ServerStatus() const { try { std::string dummy; - Status status = client_ptr_->Cmd(dummy, ""); + Status status = client_ptr_->Cmd(dummy, "status"); return "server alive"; } catch (std::exception& ex) { return "connection lost"; @@ -456,4 +456,23 @@ ClientProxy::DropPartition(const PartitionParam& partition_param) { } } +Status +ClientProxy::GetConfig(const std::string& node_name, std::string& value) const { + try { + return client_ptr_->Cmd(value, "get " + node_name); + } catch (std::exception& ex) { + return Status(StatusCode::UnknownError, "Fail to get config: " + node_name); + } +} + +Status +ClientProxy::SetConfig(const std::string& node_name, const std::string& value) const { + try { + std::string dummy; + return client_ptr_->Cmd(dummy, "set " + node_name + " " + value); + } catch (std::exception& ex) { + return Status(StatusCode::UnknownError, "Fail to set config: " + node_name); + } +} + } // namespace milvus diff --git a/core/src/sdk/grpc/ClientProxy.h b/core/src/sdk/grpc/ClientProxy.h index 0a0ea0b8..c19bd3a1 100644 --- a/core/src/sdk/grpc/ClientProxy.h +++ b/core/src/sdk/grpc/ClientProxy.h @@ -104,6 +104,12 @@ class ClientProxy : public Connection { Status DropPartition(const PartitionParam& partition_param) override; + Status + GetConfig(const std::string& node_name, std::string& value) const override; + + Status + SetConfig(const std::string& node_name, const std::string& value) const override; + private: std::shared_ptr<::grpc::Channel> channel_; diff --git a/core/src/sdk/include/MilvusApi.h b/core/src/sdk/include/MilvusApi.h index 4abdf313..61966695 100644 --- a/core/src/sdk/include/MilvusApi.h +++ b/core/src/sdk/include/MilvusApi.h @@ -444,6 +444,32 @@ class Connection { */ virtual Status DropPartition(const PartitionParam& param) = 0; + + /** + * @brief Get config method + * + * This method is used to set config. + * + * @param node_name, config node name. + * @param value, config value. + * + * @return Indicate if this operation is successful. + */ + virtual Status + GetConfig(const std::string& node_name, std::string& value) const = 0; + + /** + * @brief Set config method + * + * This method is used to set config. + * + * @param node_name, config node name. + * @param value, config value. + * + * @return Indicate if this operation is successful. + */ + virtual Status + SetConfig(const std::string& node_name, const std::string& value) const = 0; }; } // namespace milvus diff --git a/core/src/sdk/interface/ConnectionImpl.cpp b/core/src/sdk/interface/ConnectionImpl.cpp index 3b544c00..698c2c4e 100644 --- a/core/src/sdk/interface/ConnectionImpl.cpp +++ b/core/src/sdk/interface/ConnectionImpl.cpp @@ -161,4 +161,13 @@ ConnectionImpl::DropPartition(const PartitionParam& param) { return client_proxy_->DropPartition(param); } +Status +ConnectionImpl::GetConfig(const std::string& node_name, std::string& value) const { + return client_proxy_->GetConfig(node_name, value); +} + +Status +ConnectionImpl::SetConfig(const std::string& node_name, const std::string& value) const { + return client_proxy_->SetConfig(node_name, value); +} } // namespace milvus diff --git a/core/src/sdk/interface/ConnectionImpl.h b/core/src/sdk/interface/ConnectionImpl.h index e22f8d33..93eda046 100644 --- a/core/src/sdk/interface/ConnectionImpl.h +++ b/core/src/sdk/interface/ConnectionImpl.h @@ -106,6 +106,12 @@ class ConnectionImpl : public Connection { Status DropPartition(const PartitionParam& param) override; + Status + GetConfig(const std::string& node_name, std::string& value) const override; + + Status + SetConfig(const std::string& node_name, const std::string& value) const override; + private: std::shared_ptr client_proxy_; }; diff --git a/core/src/server/Config.cpp b/core/src/server/Config.cpp index c770ff7b..1e82b6a8 100644 --- a/core/src/server/Config.cpp +++ b/core/src/server/Config.cpp @@ -33,7 +33,7 @@ namespace milvus { namespace server { -constexpr uint64_t GB = 1UL << 30; +constexpr int64_t GB = 1UL << 30; static const std::unordered_map milvus_config_version_map({{"0.6.0", "0.1"}}); @@ -400,6 +400,92 @@ Config::PrintAll() { PrintConfigSection(CONFIG_GPU_RESOURCE); } +Status +Config::GetConfigCli(const std::string& parent_key, const std::string& child_key, std::string& value) { + if (!ConfigNodeValid(parent_key, child_key)) { + std::string str = "Config node invalid: " + parent_key + CONFIG_NODE_DELIMITER + child_key; + return Status(SERVER_UNEXPECTED_ERROR, str); + } + return GetConfigValueInMem(parent_key, child_key, value); +} + +Status +Config::SetConfigCli(const std::string& parent_key, const std::string& child_key, const std::string& value) { + if (!ConfigNodeValid(parent_key, child_key)) { + std::string str = "Config node invalid: " + parent_key + CONFIG_NODE_DELIMITER + child_key; + return Status(SERVER_UNEXPECTED_ERROR, str); + } + if (parent_key == CONFIG_SERVER) { + return Status(SERVER_UNSUPPORTED_ERROR, "Not support set server_config"); + } else if (parent_key == CONFIG_DB) { + return Status(SERVER_UNSUPPORTED_ERROR, "Not support set db_config"); + } else if (parent_key == CONFIG_METRIC) { + return Status(SERVER_UNSUPPORTED_ERROR, "Not support set metric_config"); + } else if (parent_key == CONFIG_CACHE) { + if (child_key == CONFIG_CACHE_CPU_CACHE_CAPACITY) { + return SetCacheConfigCpuCacheCapacity(value); + } else if (child_key == CONFIG_CACHE_CPU_CACHE_THRESHOLD) { + return SetCacheConfigCpuCacheThreshold(value); + } else if (child_key == CONFIG_CACHE_CACHE_INSERT_DATA) { + return SetCacheConfigCacheInsertData(value); + } + } else if (parent_key == CONFIG_ENGINE) { + if (child_key == CONFIG_ENGINE_USE_BLAS_THRESHOLD) { + return SetEngineConfigUseBlasThreshold(value); + } else if (child_key == CONFIG_ENGINE_OMP_THREAD_NUM) { + return SetEngineConfigOmpThreadNum(value); +#ifdef MILVUS_GPU_VERSION + } else if (child_key == CONFIG_ENGINE_GPU_SEARCH_THRESHOLD) { + return SetEngineConfigGpuSearchThreshold(value); +#endif + } +#ifdef MILVUS_GPU_VERSION + } else if (parent_key == CONFIG_GPU_RESOURCE) { + if (child_key == CONFIG_GPU_RESOURCE_ENABLE) { + return SetGpuResourceConfigEnable(value); + } else if (child_key == CONFIG_GPU_RESOURCE_CACHE_CAPACITY) { + return SetGpuResourceConfigCacheCapacity(value); + } else if (child_key == CONFIG_GPU_RESOURCE_CACHE_THRESHOLD) { + return SetGpuResourceConfigCacheThreshold(value); + } else if (child_key == CONFIG_GPU_RESOURCE_SEARCH_RESOURCES) { + return SetGpuResourceConfigSearchResources(value); + } else if (child_key == CONFIG_GPU_RESOURCE_BUILD_INDEX_RESOURCES) { + return SetGpuResourceConfigBuildIndexResources(value); + } +#endif + } else if (parent_key == CONFIG_TRACING) { + return Status(SERVER_UNSUPPORTED_ERROR, "Not support set tracing_config"); + } +} + +Status +Config::HandleConfigCli(std::string& result, const std::string& cmd) { + std::vector tokens; + std::vector nodes; + server::StringHelpFunctions::SplitStringByDelimeter(cmd, " ", tokens); + if (tokens[0] == "get") { + if (tokens.size() != 2) { + return Status(SERVER_UNEXPECTED_ERROR, "Invalid command: " + cmd); + } + server::StringHelpFunctions::SplitStringByDelimeter(tokens[1], CONFIG_NODE_DELIMITER, nodes); + if (nodes.size() != 2) { + return Status(SERVER_UNEXPECTED_ERROR, "Invalid command: " + cmd); + } + return GetConfigCli(nodes[0], nodes[1], result); + } else if (tokens[0] == "set") { + if (tokens.size() != 3) { + return Status(SERVER_UNEXPECTED_ERROR, "Invalid command: " + cmd); + } + server::StringHelpFunctions::SplitStringByDelimeter(tokens[1], CONFIG_NODE_DELIMITER, nodes); + if (nodes.size() != 2) { + return Status(SERVER_UNEXPECTED_ERROR, "Invalid command: " + cmd); + } + return SetConfigCli(nodes[0], nodes[1], tokens[2]); + } else { + return Status(SERVER_UNEXPECTED_ERROR, "Invalid command: " + cmd); + } +} + //////////////////////////////////////////////////////////////////////////////// Status Config::CheckConfigVersion(const std::string& value) { @@ -804,6 +890,17 @@ Config::GetConfigNode(const std::string& name) { return GetConfigRoot().GetChild(name); } +bool +Config::ConfigNodeValid(const std::string& parent_key, const std::string& child_key) { + if (config_map_.find(parent_key) == config_map_.end()) { + return false; + } + if (config_map_[parent_key].count(child_key) == 0) { + return false; + } + return true; +} + Status Config::GetConfigValueInMem(const std::string& parent_key, const std::string& child_key, std::string& value) { std::lock_guard lock(mutex_); @@ -815,10 +912,11 @@ Config::GetConfigValueInMem(const std::string& parent_key, const std::string& ch return Status(SERVER_UNEXPECTED_ERROR, "key not exist"); } -void +Status Config::SetConfigValueInMem(const std::string& parent_key, const std::string& child_key, const std::string& value) { std::lock_guard lock(mutex_); config_map_[parent_key][child_key] = value; + return Status::OK(); } //////////////////////////////////////////////////////////////////////////////// diff --git a/core/src/server/Config.h b/core/src/server/Config.h index 9939779e..9062070e 100644 --- a/core/src/server/Config.h +++ b/core/src/server/Config.h @@ -28,6 +28,7 @@ namespace milvus { namespace server { +static const char* CONFIG_NODE_DELIMITER = "."; static const char* CONFIG_VERSION = "version"; /* server config */ @@ -56,6 +57,7 @@ static const char* CONFIG_DB_ARCHIVE_DAYS_THRESHOLD_DEFAULT = "0"; static const char* CONFIG_DB_INSERT_BUFFER_SIZE = "insert_buffer_size"; static const char* CONFIG_DB_INSERT_BUFFER_SIZE_DEFAULT = "4"; static const char* CONFIG_DB_PRELOAD_TABLE = "preload_table"; +static const char* CONFIG_DB_PRELOAD_TABLE_DEFAULT = ""; /* cache config */ static const char* CONFIG_CACHE = "cache_config"; @@ -120,18 +122,26 @@ class Config { ResetDefaultConfig(); void PrintAll(); + Status + HandleConfigCli(std::string& result, const std::string& cmd); private: ConfigNode& GetConfigRoot(); ConfigNode& GetConfigNode(const std::string& name); + bool + ConfigNodeValid(const std::string& parent_key, const std::string& child_key); Status GetConfigValueInMem(const std::string& parent_key, const std::string& child_key, std::string& value); - void + Status SetConfigValueInMem(const std::string& parent_key, const std::string& child_key, const std::string& value); void PrintConfigSection(const std::string& config_node_name); + Status + GetConfigCli(const std::string& parent_key, const std::string& child_key, std::string& value); + Status + SetConfigCli(const std::string& parent_key, const std::string& child_key, const std::string& value); /////////////////////////////////////////////////////////////////////////// Status diff --git a/core/unittest/server/test_config.cpp b/core/unittest/server/test_config.cpp index 791876ee..a85cd7d9 100644 --- a/core/unittest/server/test_config.cpp +++ b/core/unittest/server/test_config.cpp @@ -35,6 +35,8 @@ static constexpr uint64_t GB = MB * 1024; } // namespace +namespace ms = milvus::server; + TEST_F(ConfigTest, CONFIG_TEST) { milvus::server::ConfigMgr* config_mgr = milvus::server::YamlConfigMgr::GetInstance(); @@ -307,6 +309,154 @@ TEST_F(ConfigTest, SERVER_CONFIG_VALID_TEST) { #endif } +std::string gen_get_command(const std::string& parent_node, const std::string& child_node) { + std::string cmd = "get " + parent_node + ms::CONFIG_NODE_DELIMITER + child_node; + return cmd; +} + +std::string gen_set_command(const std::string& parent_node, const std::string& child_node, const std::string& value) { + std::string cmd = "set " + parent_node + ms::CONFIG_NODE_DELIMITER + child_node + " " + value; + return cmd; +} + +TEST_F(ConfigTest, SERVER_CONFIG_CLI_TEST) { + std::string config_path(CONFIG_PATH); + milvus::server::Config& config = milvus::server::Config::GetInstance(); + milvus::Status s; + + std::string get_cmd, set_cmd; + std::string result, dummy; + + /* server config */ + std::string server_addr = "192.168.1.155"; + get_cmd = gen_get_command(ms::CONFIG_SERVER, ms::CONFIG_SERVER_ADDRESS); + set_cmd = gen_set_command(ms::CONFIG_SERVER, ms::CONFIG_SERVER_ADDRESS, server_addr); + s = config.HandleConfigCli(dummy, set_cmd); + ASSERT_FALSE(s.ok()); + s = config.HandleConfigCli(result, get_cmd); + ASSERT_TRUE(s.ok()); + + /* db config */ + std::string db_primary_path = "/home/zilliz"; + get_cmd = gen_get_command(ms::CONFIG_DB, ms::CONFIG_DB_PRIMARY_PATH); + set_cmd = gen_set_command(ms::CONFIG_DB, ms::CONFIG_DB_PRIMARY_PATH, db_primary_path); + s = config.HandleConfigCli(dummy, set_cmd); + ASSERT_FALSE(s.ok()); + s = config.HandleConfigCli(result, get_cmd); + ASSERT_TRUE(s.ok()); + + /* metric config */ + std::string metric_enable_monitor = "false"; + get_cmd = gen_get_command(ms::CONFIG_METRIC, ms::CONFIG_METRIC_ENABLE_MONITOR); + set_cmd = gen_set_command(ms::CONFIG_METRIC, ms::CONFIG_METRIC_ENABLE_MONITOR, metric_enable_monitor); + s = config.HandleConfigCli(dummy, set_cmd); + ASSERT_FALSE(s.ok()); + s = config.HandleConfigCli(result, get_cmd); + ASSERT_TRUE(s.ok()); + + /* cache config */ + std::string cache_cpu_cache_capacity = "5"; + get_cmd = gen_get_command(ms::CONFIG_CACHE, ms::CONFIG_CACHE_CPU_CACHE_CAPACITY); + set_cmd = gen_set_command(ms::CONFIG_CACHE, ms::CONFIG_CACHE_CPU_CACHE_CAPACITY, cache_cpu_cache_capacity); + s = config.HandleConfigCli(dummy, set_cmd); + ASSERT_TRUE(s.ok()); + s = config.HandleConfigCli(result, get_cmd); + ASSERT_TRUE(s.ok()); + ASSERT_TRUE(result == cache_cpu_cache_capacity); + + std::string cache_cpu_cache_threshold = "0.1"; + get_cmd = gen_get_command(ms::CONFIG_CACHE, ms::CONFIG_CACHE_CPU_CACHE_THRESHOLD); + set_cmd = gen_set_command(ms::CONFIG_CACHE, ms::CONFIG_CACHE_CPU_CACHE_THRESHOLD, cache_cpu_cache_threshold); + s = config.HandleConfigCli(dummy, set_cmd); + ASSERT_TRUE(s.ok()); + s = config.HandleConfigCli(result, get_cmd); + ASSERT_TRUE(result == cache_cpu_cache_threshold); + + std::string cache_insert_data = "true"; + get_cmd = gen_get_command(ms::CONFIG_CACHE, ms::CONFIG_CACHE_CACHE_INSERT_DATA); + set_cmd = gen_set_command(ms::CONFIG_CACHE, ms::CONFIG_CACHE_CACHE_INSERT_DATA, cache_insert_data); + s = config.HandleConfigCli(dummy, set_cmd); + ASSERT_TRUE(s.ok()); + s = config.HandleConfigCli(result, get_cmd); + ASSERT_TRUE(result == cache_insert_data); + + /* engine config */ + std::string engine_use_blas_threshold = "50"; + get_cmd = gen_get_command(ms::CONFIG_ENGINE, ms::CONFIG_ENGINE_USE_BLAS_THRESHOLD); + set_cmd = gen_set_command(ms::CONFIG_ENGINE, ms::CONFIG_ENGINE_USE_BLAS_THRESHOLD, engine_use_blas_threshold); + s = config.HandleConfigCli(dummy, set_cmd); + ASSERT_TRUE(s.ok()); + s = config.HandleConfigCli(result, get_cmd); + ASSERT_TRUE(s.ok()); + ASSERT_TRUE(result == engine_use_blas_threshold); + + std::string engine_omp_thread_num = "8"; + get_cmd = gen_get_command(ms::CONFIG_ENGINE, ms::CONFIG_ENGINE_OMP_THREAD_NUM); + set_cmd = gen_set_command(ms::CONFIG_ENGINE, ms::CONFIG_ENGINE_OMP_THREAD_NUM, engine_omp_thread_num); + s = config.HandleConfigCli(dummy, set_cmd); + ASSERT_TRUE(s.ok()); + s = config.HandleConfigCli(result, get_cmd); + ASSERT_TRUE(s.ok()); + ASSERT_TRUE(result == engine_omp_thread_num); + +#ifdef MILVUS_GPU_VERSION + std::string engine_gpu_search_threshold = "800"; + get_cmd = gen_get_command(ms::CONFIG_ENGINE, ms::CONFIG_ENGINE_GPU_SEARCH_THRESHOLD); + set_cmd = gen_set_command(ms::CONFIG_ENGINE, ms::CONFIG_ENGINE_GPU_SEARCH_THRESHOLD, engine_gpu_search_threshold); + s = config.HandleConfigCli(dummy, set_cmd); + ASSERT_TRUE(s.ok()); + s = config.HandleConfigCli(result, get_cmd); + ASSERT_TRUE(s.ok()); + ASSERT_TRUE(result == engine_gpu_search_threshold); + + /* gpu resource config */ + std::string resource_enable_gpu = "true"; + get_cmd = gen_get_command(ms::CONFIG_GPU_RESOURCE, ms::CONFIG_GPU_RESOURCE_ENABLE); + set_cmd = gen_set_command(ms::CONFIG_GPU_RESOURCE, ms::CONFIG_GPU_RESOURCE_ENABLE, resource_enable_gpu); + s = config.HandleConfigCli(dummy, set_cmd); + ASSERT_TRUE(s.ok()); + s = config.HandleConfigCli(result, get_cmd); + ASSERT_TRUE(s.ok()); + ASSERT_TRUE(result == resource_enable_gpu); + + std::string gpu_cache_capacity = "1"; + get_cmd = gen_get_command(ms::CONFIG_GPU_RESOURCE, ms::CONFIG_GPU_RESOURCE_CACHE_CAPACITY); + set_cmd = gen_set_command(ms::CONFIG_GPU_RESOURCE, ms::CONFIG_GPU_RESOURCE_CACHE_CAPACITY, gpu_cache_capacity); + s = config.HandleConfigCli(dummy, set_cmd); + ASSERT_TRUE(s.ok()); + s = config.HandleConfigCli(result, get_cmd); + ASSERT_TRUE(s.ok()); + ASSERT_TRUE(result == gpu_cache_capacity); + + std::string gpu_cache_threshold = "0.2"; + get_cmd = gen_get_command(ms::CONFIG_GPU_RESOURCE, ms::CONFIG_GPU_RESOURCE_CACHE_THRESHOLD); + set_cmd = gen_set_command(ms::CONFIG_GPU_RESOURCE, ms::CONFIG_GPU_RESOURCE_CACHE_THRESHOLD, gpu_cache_threshold); + s = config.HandleConfigCli(dummy, set_cmd); + ASSERT_TRUE(s.ok()); + s = config.HandleConfigCli(result, get_cmd); + ASSERT_TRUE(result == gpu_cache_threshold); + + std::string search_resources = "gpu0"; + get_cmd = gen_get_command(ms::CONFIG_GPU_RESOURCE, ms::CONFIG_GPU_RESOURCE_SEARCH_RESOURCES); + set_cmd = gen_set_command(ms::CONFIG_GPU_RESOURCE, ms::CONFIG_GPU_RESOURCE_SEARCH_RESOURCES, search_resources); + s = config.HandleConfigCli(dummy, set_cmd); + ASSERT_TRUE(s.ok()); + s = config.HandleConfigCli(result, get_cmd); + ASSERT_TRUE(s.ok()); + ASSERT_TRUE(result == search_resources); + + std::string build_index_resources = "gpu0"; + get_cmd = gen_get_command(ms::CONFIG_GPU_RESOURCE, ms::CONFIG_GPU_RESOURCE_BUILD_INDEX_RESOURCES); + set_cmd = + gen_set_command(ms::CONFIG_GPU_RESOURCE, ms::CONFIG_GPU_RESOURCE_BUILD_INDEX_RESOURCES, build_index_resources); + s = config.HandleConfigCli(dummy, set_cmd); + ASSERT_TRUE(s.ok()); + s = config.HandleConfigCli(result, get_cmd); + ASSERT_TRUE(s.ok()); + ASSERT_TRUE(result == build_index_resources); +#endif +} + TEST_F(ConfigTest, SERVER_CONFIG_INVALID_TEST) { std::string config_path(CONFIG_PATH); milvus::server::Config& config = milvus::server::Config::GetInstance(); -- GitLab