diff --git a/CHANGELOG.md b/CHANGELOG.md index 8bc28d0c6238063331a9ea2aab8da53312ddfe30..ee008d269a9f82675433e889aa157a6c466a42ab 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ Please mark all change in change log and use the issue from GitHub ## Feature - \#343 - Add Opentracing - \#665 - Support get/set config via CLI +- \#771 - Add server build commit info interface ## Improvement diff --git a/core/src/sdk/examples/simple/src/ClientTest.cpp b/core/src/sdk/examples/simple/src/ClientTest.cpp index a3837011953e1138e86226f445b17f727d8fdf01..557a7a6390c32dc31a49a6ea224f7ea8e6bf96a6 100644 --- a/core/src/sdk/examples/simple/src/ClientTest.cpp +++ b/core/src/sdk/examples/simple/src/ClientTest.cpp @@ -76,6 +76,23 @@ ClientTest::Test(const std::string& address, const std::string& port) { std::cout << "SDK version: " << version << std::endl; } + { // build commit + std::string build_commit_str; + std::vector params; + stat = conn->ProcessCommand(build_commit_str, "build_commit_id", params); + std::cout << "ProcessCommand status: " << stat.message() << std::endl; + std::cout << "Milvus build commit id: " << build_commit_str << std::endl; + } + + { // build commit + std::string config_str; + std::vector params; + params.push_back("*"); + stat = conn->ProcessCommand(config_str, "get_config", params); + std::cout << "ProcessCommand status: " << stat.message() << std::endl; + std::cout << "Milvus config json: " << config_str << std::endl; + } + { // show tables std::vector tables; stat = conn->ShowTables(tables); diff --git a/core/src/sdk/grpc/ClientProxy.cpp b/core/src/sdk/grpc/ClientProxy.cpp index afa90c955b6b6743dce723ce0a3189a416c169da..80a1d8034cceca8b5647583228c655d515a27924 100644 --- a/core/src/sdk/grpc/ClientProxy.cpp +++ b/core/src/sdk/grpc/ClientProxy.cpp @@ -457,21 +457,15 @@ ClientProxy::DropPartition(const PartitionParam& partition_param) { } Status -ClientProxy::GetConfig(const std::string& node_name, std::string& value) const { +ClientProxy::ProcessCommand(std::string& result, const std::string& cmd, const std::vector& params) const { try { - return client_ptr_->Cmd(value, "get " + node_name); - } catch (std::exception& ex) { - return Status(StatusCode::UnknownError, "Fail to get config: " + node_name); - } -} - -Status -ClientProxy::SetConfig(const std::string& node_name, const std::string& value) const { - try { - std::string dummy; - return client_ptr_->Cmd(dummy, "set " + node_name + " " + value); + std::string cmd_line = cmd; + for (const std::string& param : params) { + cmd_line += " " + param; + } + return client_ptr_->Cmd(result, cmd_line); } catch (std::exception& ex) { - return Status(StatusCode::UnknownError, "Fail to set config: " + node_name); + return Status(StatusCode::UnknownError, "Fail to process command: " + cmd); } } diff --git a/core/src/sdk/grpc/ClientProxy.h b/core/src/sdk/grpc/ClientProxy.h index c19bd3a12bb1791c24d22fa653fb416e62411c4a..5a38b5cf3b470e5dc6ddd55f7098f42f7f79c497 100644 --- a/core/src/sdk/grpc/ClientProxy.h +++ b/core/src/sdk/grpc/ClientProxy.h @@ -105,10 +105,7 @@ class ClientProxy : public Connection { DropPartition(const PartitionParam& partition_param) override; Status - GetConfig(const std::string& node_name, std::string& value) const override; - - Status - SetConfig(const std::string& node_name, const std::string& value) const override; + ProcessCommand(std::string& result, const std::string& cmd, const std::vector& params) const override; private: std::shared_ptr<::grpc::Channel> channel_; diff --git a/core/src/sdk/include/MilvusApi.h b/core/src/sdk/include/MilvusApi.h index 61966695d14550e6ca3d53506b630b68c4379a29..2cc2e03674f91343f4e0286abd699f6f8b97298e 100644 --- a/core/src/sdk/include/MilvusApi.h +++ b/core/src/sdk/include/MilvusApi.h @@ -446,30 +446,18 @@ class Connection { DropPartition(const PartitionParam& param) = 0; /** - * @brief Get config method + * @brief Process command * - * This method is used to set config. + * This method is used to process pre-defined commands. * - * @param node_name, config node name. - * @param value, config value. + * @param result, result string. + * @param cmd, command string. + * @param params, command parameters. * * @return Indicate if this operation is successful. */ virtual Status - GetConfig(const std::string& node_name, std::string& value) const = 0; - - /** - * @brief Set config method - * - * This method is used to set config. - * - * @param node_name, config node name. - * @param value, config value. - * - * @return Indicate if this operation is successful. - */ - virtual Status - SetConfig(const std::string& node_name, const std::string& value) const = 0; + ProcessCommand(std::string& result, const std::string& cmd, const std::vector& params) const = 0; }; } // namespace milvus diff --git a/core/src/sdk/interface/ConnectionImpl.cpp b/core/src/sdk/interface/ConnectionImpl.cpp index 698c2c4ea864fda84a1dec61dcc7cb1ab57c5ed2..200305c7c36761643688fb3ca174ea2c2988b956 100644 --- a/core/src/sdk/interface/ConnectionImpl.cpp +++ b/core/src/sdk/interface/ConnectionImpl.cpp @@ -162,12 +162,9 @@ ConnectionImpl::DropPartition(const PartitionParam& param) { } Status -ConnectionImpl::GetConfig(const std::string& node_name, std::string& value) const { - return client_proxy_->GetConfig(node_name, value); +ConnectionImpl::ProcessCommand(std::string& result, const std::string& cmd, + const std::vector& params) const { + return client_proxy_->ProcessCommand(result, cmd, params); } -Status -ConnectionImpl::SetConfig(const std::string& node_name, const std::string& value) const { - return client_proxy_->SetConfig(node_name, value); -} } // namespace milvus diff --git a/core/src/sdk/interface/ConnectionImpl.h b/core/src/sdk/interface/ConnectionImpl.h index 93eda04662221c294f9c26a875a6f391fe1ac110..4b42cb7f58307bd8cae7f77b0a5767b4836ccba1 100644 --- a/core/src/sdk/interface/ConnectionImpl.h +++ b/core/src/sdk/interface/ConnectionImpl.h @@ -107,10 +107,7 @@ class ConnectionImpl : public Connection { DropPartition(const PartitionParam& param) override; Status - GetConfig(const std::string& node_name, std::string& value) const override; - - Status - SetConfig(const std::string& node_name, const std::string& value) const override; + ProcessCommand(std::string& result, const std::string& cmd, const std::vector& params) const override; private: std::shared_ptr client_proxy_; diff --git a/core/src/server/Config.cpp b/core/src/server/Config.cpp index 1e82b6a86374cae122c32d86acce99d936bf08dc..13c90262a75777bc1f23382415e20124c398cfbe 100644 --- a/core/src/server/Config.cpp +++ b/core/src/server/Config.cpp @@ -26,6 +26,7 @@ #include "config/YamlConfigMgr.h" #include "server/Config.h" +#include "thirdparty/nlohmann/json.hpp" #include "utils/CommonUtil.h" #include "utils/StringHelpFunctions.h" #include "utils/ValidationUtil.h" @@ -380,28 +381,13 @@ Config::ResetDefaultConfig() { } void -Config::PrintConfigSection(const std::string& config_node_name) { - std::cout << std::endl; - std::cout << config_node_name << ":" << std::endl; - if (config_map_.find(config_node_name) != config_map_.end()) { - for (auto item : config_map_[config_node_name]) { - std::cout << item.first << ": " << item.second << std::endl; - } - } -} - -void -Config::PrintAll() { - PrintConfigSection(CONFIG_SERVER); - PrintConfigSection(CONFIG_DB); - PrintConfigSection(CONFIG_CACHE); - PrintConfigSection(CONFIG_METRIC); - PrintConfigSection(CONFIG_ENGINE); - PrintConfigSection(CONFIG_GPU_RESOURCE); +Config::GetConfigJsonStr(std::string& result) { + nlohmann::json config_json(config_map_); + result = config_json.dump(); } Status -Config::GetConfigCli(const std::string& parent_key, const std::string& child_key, std::string& value) { +Config::GetConfigCli(std::string& value, const std::string& parent_key, const std::string& child_key) { if (!ConfigNodeValid(parent_key, child_key)) { std::string str = "Config node invalid: " + parent_key + CONFIG_NODE_DELIMITER + child_key; return Status(SERVER_UNEXPECTED_ERROR, str); @@ -459,20 +445,25 @@ Config::SetConfigCli(const std::string& parent_key, const std::string& child_key } Status -Config::HandleConfigCli(std::string& result, const std::string& cmd) { +Config::ProcessConfigCli(std::string& result, const std::string& cmd) { std::vector tokens; std::vector nodes; server::StringHelpFunctions::SplitStringByDelimeter(cmd, " ", tokens); - if (tokens[0] == "get") { + if (tokens[0] == "get_config") { if (tokens.size() != 2) { return Status(SERVER_UNEXPECTED_ERROR, "Invalid command: " + cmd); } - server::StringHelpFunctions::SplitStringByDelimeter(tokens[1], CONFIG_NODE_DELIMITER, nodes); - if (nodes.size() != 2) { - return Status(SERVER_UNEXPECTED_ERROR, "Invalid command: " + cmd); + if (tokens[1] == "*") { + GetConfigJsonStr(result); + return Status::OK(); + } else { + server::StringHelpFunctions::SplitStringByDelimeter(tokens[1], CONFIG_NODE_DELIMITER, nodes); + if (nodes.size() != 2) { + return Status(SERVER_UNEXPECTED_ERROR, "Invalid command: " + cmd); + } + return GetConfigCli(result, nodes[0], nodes[1]); } - return GetConfigCli(nodes[0], nodes[1], result); - } else if (tokens[0] == "set") { + } else if (tokens[0] == "set_config") { if (tokens.size() != 3) { return Status(SERVER_UNEXPECTED_ERROR, "Invalid command: " + cmd); } diff --git a/core/src/server/Config.h b/core/src/server/Config.h index 9062070e793eba966fa9b1628b2ca3876c3df353..d35753b707e73de170648bef31a277067ef49af4 100644 --- a/core/src/server/Config.h +++ b/core/src/server/Config.h @@ -121,9 +121,9 @@ class Config { Status ResetDefaultConfig(); void - PrintAll(); + GetConfigJsonStr(std::string& result); Status - HandleConfigCli(std::string& result, const std::string& cmd); + ProcessConfigCli(std::string& result, const std::string& cmd); private: ConfigNode& @@ -136,10 +136,8 @@ class Config { GetConfigValueInMem(const std::string& parent_key, const std::string& child_key, std::string& value); Status SetConfigValueInMem(const std::string& parent_key, const std::string& child_key, const std::string& value); - void - PrintConfigSection(const std::string& config_node_name); Status - GetConfigCli(const std::string& parent_key, const std::string& child_key, std::string& value); + GetConfigCli(std::string& value, const std::string& parent_key, const std::string& child_key); Status SetConfigCli(const std::string& parent_key, const std::string& child_key, const std::string& value); diff --git a/core/src/server/delivery/request/CmdRequest.cpp b/core/src/server/delivery/request/CmdRequest.cpp index b2434076eb3b15547c828243c34db1dc2ab4daa4..4263f8715cb34b398ea59dae31f57026547395c6 100644 --- a/core/src/server/delivery/request/CmdRequest.cpp +++ b/core/src/server/delivery/request/CmdRequest.cpp @@ -52,9 +52,11 @@ CmdRequest::OnExecute() { #else result_ = "CPU"; #endif - } else if (cmd_.substr(0, 3) == "set" || cmd_.substr(0, 3) == "get") { + } else if (cmd_ == "build_commit_id") { + result_ = LAST_COMMIT_ID; + } else if (cmd_.substr(0, 10) == "set_config" || cmd_.substr(0, 10) == "get_config") { server::Config& config = server::Config::GetInstance(); - stat = config.HandleConfigCli(result_, cmd_); + stat = config.ProcessConfigCli(result_, cmd_); } else { result_ = "Unknown command"; } diff --git a/core/unittest/server/test_config.cpp b/core/unittest/server/test_config.cpp index 3c27572f1afac3b9a0384918b3162abfbfcae71a..49b14deb4902ba7ae48e1e1061c3e1aafef36e63 100644 --- a/core/unittest/server/test_config.cpp +++ b/core/unittest/server/test_config.cpp @@ -311,12 +311,12 @@ TEST_F(ConfigTest, SERVER_CONFIG_VALID_TEST) { } std::string gen_get_command(const std::string& parent_node, const std::string& child_node) { - std::string cmd = "get " + parent_node + ms::CONFIG_NODE_DELIMITER + child_node; + std::string cmd = "get_config " + parent_node + ms::CONFIG_NODE_DELIMITER + child_node; return cmd; } std::string gen_set_command(const std::string& parent_node, const std::string& child_node, const std::string& value) { - std::string cmd = "set " + parent_node + ms::CONFIG_NODE_DELIMITER + child_node + " " + value; + std::string cmd = "set_config " + parent_node + ms::CONFIG_NODE_DELIMITER + child_node + " " + value; return cmd; } @@ -328,75 +328,78 @@ TEST_F(ConfigTest, SERVER_CONFIG_CLI_TEST) { std::string get_cmd, set_cmd; std::string result, dummy; + s = config.ProcessConfigCli(result, "get_config *"); + ASSERT_TRUE(s.ok()); + /* server config */ std::string server_addr = "192.168.1.155"; get_cmd = gen_get_command(ms::CONFIG_SERVER, ms::CONFIG_SERVER_ADDRESS); set_cmd = gen_set_command(ms::CONFIG_SERVER, ms::CONFIG_SERVER_ADDRESS, server_addr); - s = config.HandleConfigCli(dummy, set_cmd); + s = config.ProcessConfigCli(dummy, set_cmd); ASSERT_FALSE(s.ok()); - s = config.HandleConfigCli(result, get_cmd); + s = config.ProcessConfigCli(result, get_cmd); ASSERT_TRUE(s.ok()); /* db config */ std::string db_primary_path = "/home/zilliz"; get_cmd = gen_get_command(ms::CONFIG_DB, ms::CONFIG_DB_PRIMARY_PATH); set_cmd = gen_set_command(ms::CONFIG_DB, ms::CONFIG_DB_PRIMARY_PATH, db_primary_path); - s = config.HandleConfigCli(dummy, set_cmd); + s = config.ProcessConfigCli(dummy, set_cmd); ASSERT_FALSE(s.ok()); - s = config.HandleConfigCli(result, get_cmd); + s = config.ProcessConfigCli(result, get_cmd); ASSERT_TRUE(s.ok()); /* metric config */ std::string metric_enable_monitor = "false"; get_cmd = gen_get_command(ms::CONFIG_METRIC, ms::CONFIG_METRIC_ENABLE_MONITOR); set_cmd = gen_set_command(ms::CONFIG_METRIC, ms::CONFIG_METRIC_ENABLE_MONITOR, metric_enable_monitor); - s = config.HandleConfigCli(dummy, set_cmd); + s = config.ProcessConfigCli(dummy, set_cmd); ASSERT_FALSE(s.ok()); - s = config.HandleConfigCli(result, get_cmd); + s = config.ProcessConfigCli(result, get_cmd); ASSERT_TRUE(s.ok()); /* cache config */ std::string cache_cpu_cache_capacity = "5"; get_cmd = gen_get_command(ms::CONFIG_CACHE, ms::CONFIG_CACHE_CPU_CACHE_CAPACITY); set_cmd = gen_set_command(ms::CONFIG_CACHE, ms::CONFIG_CACHE_CPU_CACHE_CAPACITY, cache_cpu_cache_capacity); - s = config.HandleConfigCli(dummy, set_cmd); + s = config.ProcessConfigCli(dummy, set_cmd); ASSERT_TRUE(s.ok()); - s = config.HandleConfigCli(result, get_cmd); + s = config.ProcessConfigCli(result, get_cmd); ASSERT_TRUE(s.ok()); ASSERT_TRUE(result == cache_cpu_cache_capacity); std::string cache_cpu_cache_threshold = "0.1"; get_cmd = gen_get_command(ms::CONFIG_CACHE, ms::CONFIG_CACHE_CPU_CACHE_THRESHOLD); set_cmd = gen_set_command(ms::CONFIG_CACHE, ms::CONFIG_CACHE_CPU_CACHE_THRESHOLD, cache_cpu_cache_threshold); - s = config.HandleConfigCli(dummy, set_cmd); + s = config.ProcessConfigCli(dummy, set_cmd); ASSERT_TRUE(s.ok()); - s = config.HandleConfigCli(result, get_cmd); + s = config.ProcessConfigCli(result, get_cmd); ASSERT_TRUE(result == cache_cpu_cache_threshold); std::string cache_insert_data = "true"; get_cmd = gen_get_command(ms::CONFIG_CACHE, ms::CONFIG_CACHE_CACHE_INSERT_DATA); set_cmd = gen_set_command(ms::CONFIG_CACHE, ms::CONFIG_CACHE_CACHE_INSERT_DATA, cache_insert_data); - s = config.HandleConfigCli(dummy, set_cmd); + s = config.ProcessConfigCli(dummy, set_cmd); ASSERT_TRUE(s.ok()); - s = config.HandleConfigCli(result, get_cmd); + s = config.ProcessConfigCli(result, get_cmd); ASSERT_TRUE(result == cache_insert_data); /* engine config */ std::string engine_use_blas_threshold = "50"; get_cmd = gen_get_command(ms::CONFIG_ENGINE, ms::CONFIG_ENGINE_USE_BLAS_THRESHOLD); set_cmd = gen_set_command(ms::CONFIG_ENGINE, ms::CONFIG_ENGINE_USE_BLAS_THRESHOLD, engine_use_blas_threshold); - s = config.HandleConfigCli(dummy, set_cmd); + s = config.ProcessConfigCli(dummy, set_cmd); ASSERT_TRUE(s.ok()); - s = config.HandleConfigCli(result, get_cmd); + s = config.ProcessConfigCli(result, get_cmd); ASSERT_TRUE(s.ok()); ASSERT_TRUE(result == engine_use_blas_threshold); std::string engine_omp_thread_num = "8"; get_cmd = gen_get_command(ms::CONFIG_ENGINE, ms::CONFIG_ENGINE_OMP_THREAD_NUM); set_cmd = gen_set_command(ms::CONFIG_ENGINE, ms::CONFIG_ENGINE_OMP_THREAD_NUM, engine_omp_thread_num); - s = config.HandleConfigCli(dummy, set_cmd); + s = config.ProcessConfigCli(dummy, set_cmd); ASSERT_TRUE(s.ok()); - s = config.HandleConfigCli(result, get_cmd); + s = config.ProcessConfigCli(result, get_cmd); ASSERT_TRUE(s.ok()); ASSERT_TRUE(result == engine_omp_thread_num); @@ -404,9 +407,9 @@ TEST_F(ConfigTest, SERVER_CONFIG_CLI_TEST) { std::string engine_gpu_search_threshold = "800"; get_cmd = gen_get_command(ms::CONFIG_ENGINE, ms::CONFIG_ENGINE_GPU_SEARCH_THRESHOLD); set_cmd = gen_set_command(ms::CONFIG_ENGINE, ms::CONFIG_ENGINE_GPU_SEARCH_THRESHOLD, engine_gpu_search_threshold); - s = config.HandleConfigCli(dummy, set_cmd); + s = config.ProcessConfigCli(dummy, set_cmd); ASSERT_TRUE(s.ok()); - s = config.HandleConfigCli(result, get_cmd); + s = config.ProcessConfigCli(result, get_cmd); ASSERT_TRUE(s.ok()); ASSERT_TRUE(result == engine_gpu_search_threshold); @@ -414,35 +417,35 @@ TEST_F(ConfigTest, SERVER_CONFIG_CLI_TEST) { std::string resource_enable_gpu = "true"; get_cmd = gen_get_command(ms::CONFIG_GPU_RESOURCE, ms::CONFIG_GPU_RESOURCE_ENABLE); set_cmd = gen_set_command(ms::CONFIG_GPU_RESOURCE, ms::CONFIG_GPU_RESOURCE_ENABLE, resource_enable_gpu); - s = config.HandleConfigCli(dummy, set_cmd); + s = config.ProcessConfigCli(dummy, set_cmd); ASSERT_TRUE(s.ok()); - s = config.HandleConfigCli(result, get_cmd); + s = config.ProcessConfigCli(result, get_cmd); ASSERT_TRUE(s.ok()); ASSERT_TRUE(result == resource_enable_gpu); std::string gpu_cache_capacity = "1"; get_cmd = gen_get_command(ms::CONFIG_GPU_RESOURCE, ms::CONFIG_GPU_RESOURCE_CACHE_CAPACITY); set_cmd = gen_set_command(ms::CONFIG_GPU_RESOURCE, ms::CONFIG_GPU_RESOURCE_CACHE_CAPACITY, gpu_cache_capacity); - s = config.HandleConfigCli(dummy, set_cmd); + s = config.ProcessConfigCli(dummy, set_cmd); ASSERT_TRUE(s.ok()); - s = config.HandleConfigCli(result, get_cmd); + s = config.ProcessConfigCli(result, get_cmd); ASSERT_TRUE(s.ok()); ASSERT_TRUE(result == gpu_cache_capacity); std::string gpu_cache_threshold = "0.2"; get_cmd = gen_get_command(ms::CONFIG_GPU_RESOURCE, ms::CONFIG_GPU_RESOURCE_CACHE_THRESHOLD); set_cmd = gen_set_command(ms::CONFIG_GPU_RESOURCE, ms::CONFIG_GPU_RESOURCE_CACHE_THRESHOLD, gpu_cache_threshold); - s = config.HandleConfigCli(dummy, set_cmd); + s = config.ProcessConfigCli(dummy, set_cmd); ASSERT_TRUE(s.ok()); - s = config.HandleConfigCli(result, get_cmd); + s = config.ProcessConfigCli(result, get_cmd); ASSERT_TRUE(result == gpu_cache_threshold); std::string search_resources = "gpu0"; get_cmd = gen_get_command(ms::CONFIG_GPU_RESOURCE, ms::CONFIG_GPU_RESOURCE_SEARCH_RESOURCES); set_cmd = gen_set_command(ms::CONFIG_GPU_RESOURCE, ms::CONFIG_GPU_RESOURCE_SEARCH_RESOURCES, search_resources); - s = config.HandleConfigCli(dummy, set_cmd); + s = config.ProcessConfigCli(dummy, set_cmd); ASSERT_TRUE(s.ok()); - s = config.HandleConfigCli(result, get_cmd); + s = config.ProcessConfigCli(result, get_cmd); ASSERT_TRUE(s.ok()); ASSERT_TRUE(result == search_resources); @@ -450,9 +453,9 @@ TEST_F(ConfigTest, SERVER_CONFIG_CLI_TEST) { get_cmd = gen_get_command(ms::CONFIG_GPU_RESOURCE, ms::CONFIG_GPU_RESOURCE_BUILD_INDEX_RESOURCES); set_cmd = gen_set_command(ms::CONFIG_GPU_RESOURCE, ms::CONFIG_GPU_RESOURCE_BUILD_INDEX_RESOURCES, build_index_resources); - s = config.HandleConfigCli(dummy, set_cmd); + s = config.ProcessConfigCli(dummy, set_cmd); ASSERT_TRUE(s.ok()); - s = config.HandleConfigCli(result, get_cmd); + s = config.ProcessConfigCli(result, get_cmd); ASSERT_TRUE(s.ok()); ASSERT_TRUE(result == build_index_resources); #endif @@ -591,7 +594,9 @@ TEST_F(ConfigTest, SERVER_CONFIG_TEST) { s = config.ValidateConfig(); ASSERT_TRUE(s.ok()); - config.PrintAll(); + std::string config_json_str; + config.GetConfigJsonStr(config_json_str); + std::cout << config_json_str << std::endl; s = config.ResetDefaultConfig(); ASSERT_TRUE(s.ok());