support config cli test 2 (#764)

* #665 support get/set config via CLI

* #665 support get/set config via CLI

* #665 add unittest for config CLI

* #665 remove config_node_map_

* #665 remove config_node_map_

* #665 fix clang-format

* #665 fix clang-format

* #665 update changelog

* #665 code clean

* #665 update API interface

* #665 update API interface

* #665 handle server status

* #665 handle unknown command

* #665 update debug log

* #665 rollback CmdRequest.cpp

CHANGELOG.md

@@ -9,6 +9,7 @@ Please mark all change in change log and use the issue from GitHub
 
 ## Feature
 - \#343 - Add Opentracing
+- \#665 - Support get/set config via CLI
 
 ## Improvement
 

core/src/sdk/grpc/ClientProxy.cpp

@@ -334,7 +334,7 @@ ClientProxy::ServerStatus() const {
 
     try {
         std::string dummy;
-        Status status = client_ptr_->Cmd(dummy, "");
+        Status status = client_ptr_->Cmd(dummy, "status");
         return "server alive";
     } catch (std::exception& ex) {
         return "connection lost";
@@ -456,4 +456,23 @@ ClientProxy::DropPartition(const PartitionParam& partition_param) {
     }
 }
 
+Status
+ClientProxy::GetConfig(const std::string& node_name, std::string& value) const {
+    try {
+        return client_ptr_->Cmd(value, "get " + node_name);
+    } catch (std::exception& ex) {
+        return Status(StatusCode::UnknownError, "Fail to get config: " + node_name);
+    }
+}
+
+Status
+ClientProxy::SetConfig(const std::string& node_name, const std::string& value) const {
+    try {
+        std::string dummy;
+        return client_ptr_->Cmd(dummy, "set " + node_name + " " + value);
+    } catch (std::exception& ex) {
+        return Status(StatusCode::UnknownError, "Fail to set config: " + node_name);
+    }
+}
+
 }  // namespace milvus

core/src/sdk/grpc/ClientProxy.h

@@ -104,6 +104,12 @@ class ClientProxy : public Connection {
     Status
     DropPartition(const PartitionParam& partition_param) override;
 
+    Status
+    GetConfig(const std::string& node_name, std::string& value) const override;
+
+    Status
+    SetConfig(const std::string& node_name, const std::string& value) const override;
+
  private:
     std::shared_ptr<::grpc::Channel> channel_;
 

core/src/sdk/include/MilvusApi.h

@@ -444,6 +444,32 @@ class Connection {
      */
     virtual Status
     DropPartition(const PartitionParam& param) = 0;
+
+    /**
+     * @brief Get config method
+     *
+     * This method is used to set config.
+     *
+     * @param node_name, config node name.
+     * @param value, config value.
+     *
+     * @return Indicate if this operation is successful.
+     */
+    virtual Status
+    GetConfig(const std::string& node_name, std::string& value) const = 0;
+
+    /**
+     * @brief Set config method
+     *
+     * This method is used to set config.
+     *
+     * @param node_name, config node name.
+     * @param value, config value.
+     *
+     * @return Indicate if this operation is successful.
+     */
+    virtual Status
+    SetConfig(const std::string& node_name, const std::string& value) const = 0;
 };
 
 }  // namespace milvus

core/src/sdk/interface/ConnectionImpl.cpp

@@ -161,4 +161,13 @@ ConnectionImpl::DropPartition(const PartitionParam& param) {
     return client_proxy_->DropPartition(param);
 }
 
+Status
+ConnectionImpl::GetConfig(const std::string& node_name, std::string& value) const {
+    return client_proxy_->GetConfig(node_name, value);
+}
+
+Status
+ConnectionImpl::SetConfig(const std::string& node_name, const std::string& value) const {
+    return client_proxy_->SetConfig(node_name, value);
+}
 }  // namespace milvus

core/src/sdk/interface/ConnectionImpl.h

@@ -106,6 +106,12 @@ class ConnectionImpl : public Connection {
     Status
     DropPartition(const PartitionParam& param) override;
 
+    Status
+    GetConfig(const std::string& node_name, std::string& value) const override;
+
+    Status
+    SetConfig(const std::string& node_name, const std::string& value) const override;
+
  private:
     std::shared_ptr<ClientProxy> client_proxy_;
 };

core/src/server/Config.cpp

@@ -33,7 +33,7 @@
 namespace milvus {
 namespace server {
 
-constexpr uint64_t GB = 1UL << 30;
+constexpr int64_t GB = 1UL << 30;
 
 static const std::unordered_map<std::string, std::string> milvus_config_version_map({{"0.6.0", "0.1"}});
 
@@ -400,6 +400,92 @@ Config::PrintAll() {
     PrintConfigSection(CONFIG_GPU_RESOURCE);
 }
 
+Status
+Config::GetConfigCli(const std::string& parent_key, const std::string& child_key, std::string& value) {
+    if (!ConfigNodeValid(parent_key, child_key)) {
+        std::string str = "Config node invalid: " + parent_key + CONFIG_NODE_DELIMITER + child_key;
+        return Status(SERVER_UNEXPECTED_ERROR, str);
+    }
+    return GetConfigValueInMem(parent_key, child_key, value);
+}
+
+Status
+Config::SetConfigCli(const std::string& parent_key, const std::string& child_key, const std::string& value) {
+    if (!ConfigNodeValid(parent_key, child_key)) {
+        std::string str = "Config node invalid: " + parent_key + CONFIG_NODE_DELIMITER + child_key;
+        return Status(SERVER_UNEXPECTED_ERROR, str);
+    }
+    if (parent_key == CONFIG_SERVER) {
+        return Status(SERVER_UNSUPPORTED_ERROR, "Not support set server_config");
+    } else if (parent_key == CONFIG_DB) {
+        return Status(SERVER_UNSUPPORTED_ERROR, "Not support set db_config");
+    } else if (parent_key == CONFIG_METRIC) {
+        return Status(SERVER_UNSUPPORTED_ERROR, "Not support set metric_config");
+    } else if (parent_key == CONFIG_CACHE) {
+        if (child_key == CONFIG_CACHE_CPU_CACHE_CAPACITY) {
+            return SetCacheConfigCpuCacheCapacity(value);
+        } else if (child_key == CONFIG_CACHE_CPU_CACHE_THRESHOLD) {
+            return SetCacheConfigCpuCacheThreshold(value);
+        } else if (child_key == CONFIG_CACHE_CACHE_INSERT_DATA) {
+            return SetCacheConfigCacheInsertData(value);
+        }
+    } else if (parent_key == CONFIG_ENGINE) {
+        if (child_key == CONFIG_ENGINE_USE_BLAS_THRESHOLD) {
+            return SetEngineConfigUseBlasThreshold(value);
+        } else if (child_key == CONFIG_ENGINE_OMP_THREAD_NUM) {
+            return SetEngineConfigOmpThreadNum(value);
+#ifdef MILVUS_GPU_VERSION
+        } else if (child_key == CONFIG_ENGINE_GPU_SEARCH_THRESHOLD) {
+            return SetEngineConfigGpuSearchThreshold(value);
+#endif
+        }
+#ifdef MILVUS_GPU_VERSION
+    } else if (parent_key == CONFIG_GPU_RESOURCE) {
+        if (child_key == CONFIG_GPU_RESOURCE_ENABLE) {
+            return SetGpuResourceConfigEnable(value);
+        } else if (child_key == CONFIG_GPU_RESOURCE_CACHE_CAPACITY) {
+            return SetGpuResourceConfigCacheCapacity(value);
+        } else if (child_key == CONFIG_GPU_RESOURCE_CACHE_THRESHOLD) {
+            return SetGpuResourceConfigCacheThreshold(value);
+        } else if (child_key == CONFIG_GPU_RESOURCE_SEARCH_RESOURCES) {
+            return SetGpuResourceConfigSearchResources(value);
+        } else if (child_key == CONFIG_GPU_RESOURCE_BUILD_INDEX_RESOURCES) {
+            return SetGpuResourceConfigBuildIndexResources(value);
+        }
+#endif
+    } else if (parent_key == CONFIG_TRACING) {
+        return Status(SERVER_UNSUPPORTED_ERROR, "Not support set tracing_config");
+    }
+}
+
+Status
+Config::HandleConfigCli(std::string& result, const std::string& cmd) {
+    std::vector<std::string> tokens;
+    std::vector<std::string> nodes;
+    server::StringHelpFunctions::SplitStringByDelimeter(cmd, " ", tokens);
+    if (tokens[0] == "get") {
+        if (tokens.size() != 2) {
+            return Status(SERVER_UNEXPECTED_ERROR, "Invalid command: " + cmd);
+        }
+        server::StringHelpFunctions::SplitStringByDelimeter(tokens[1], CONFIG_NODE_DELIMITER, nodes);
+        if (nodes.size() != 2) {
+            return Status(SERVER_UNEXPECTED_ERROR, "Invalid command: " + cmd);
+        }
+        return GetConfigCli(nodes[0], nodes[1], result);
+    } else if (tokens[0] == "set") {
+        if (tokens.size() != 3) {
+            return Status(SERVER_UNEXPECTED_ERROR, "Invalid command: " + cmd);
+        }
+        server::StringHelpFunctions::SplitStringByDelimeter(tokens[1], CONFIG_NODE_DELIMITER, nodes);
+        if (nodes.size() != 2) {
+            return Status(SERVER_UNEXPECTED_ERROR, "Invalid command: " + cmd);
+        }
+        return SetConfigCli(nodes[0], nodes[1], tokens[2]);
+    } else {
+        return Status(SERVER_UNEXPECTED_ERROR, "Invalid command: " + cmd);
+    }
+}
+
 ////////////////////////////////////////////////////////////////////////////////
 Status
 Config::CheckConfigVersion(const std::string& value) {
@@ -804,6 +890,17 @@ Config::GetConfigNode(const std::string& name) {
     return GetConfigRoot().GetChild(name);
 }
 
+bool
+Config::ConfigNodeValid(const std::string& parent_key, const std::string& child_key) {
+    if (config_map_.find(parent_key) == config_map_.end()) {
+        return false;
+    }
+    if (config_map_[parent_key].count(child_key) == 0) {
+        return false;
+    }
+    return true;
+}
+
 Status
 Config::GetConfigValueInMem(const std::string& parent_key, const std::string& child_key, std::string& value) {
     std::lock_guard<std::mutex> lock(mutex_);
@@ -815,10 +912,11 @@ Config::GetConfigValueInMem(const std::string& parent_key, const std::string& ch
     return Status(SERVER_UNEXPECTED_ERROR, "key not exist");
 }
 
-void
+Status
 Config::SetConfigValueInMem(const std::string& parent_key, const std::string& child_key, const std::string& value) {
     std::lock_guard<std::mutex> lock(mutex_);
     config_map_[parent_key][child_key] = value;
+    return Status::OK();
 }
 
 ////////////////////////////////////////////////////////////////////////////////

core/src/server/Config.h

@@ -28,6 +28,7 @@
 namespace milvus {
 namespace server {
 
+static const char* CONFIG_NODE_DELIMITER = ".";
 static const char* CONFIG_VERSION = "version";
 
 /* server config */
@@ -56,6 +57,7 @@ static const char* CONFIG_DB_ARCHIVE_DAYS_THRESHOLD_DEFAULT = "0";
 static const char* CONFIG_DB_INSERT_BUFFER_SIZE = "insert_buffer_size";
 static const char* CONFIG_DB_INSERT_BUFFER_SIZE_DEFAULT = "4";
 static const char* CONFIG_DB_PRELOAD_TABLE = "preload_table";
+static const char* CONFIG_DB_PRELOAD_TABLE_DEFAULT = "";
 
 /* cache config */
 static const char* CONFIG_CACHE = "cache_config";
@@ -120,18 +122,26 @@ class Config {
     ResetDefaultConfig();
     void
     PrintAll();
+    Status
+    HandleConfigCli(std::string& result, const std::string& cmd);
 
  private:
     ConfigNode&
     GetConfigRoot();
     ConfigNode&
     GetConfigNode(const std::string& name);
+    bool
+    ConfigNodeValid(const std::string& parent_key, const std::string& child_key);
     Status
     GetConfigValueInMem(const std::string& parent_key, const std::string& child_key, std::string& value);
-    void
+    Status
     SetConfigValueInMem(const std::string& parent_key, const std::string& child_key, const std::string& value);
     void
     PrintConfigSection(const std::string& config_node_name);
+    Status
+    GetConfigCli(const std::string& parent_key, const std::string& child_key, std::string& value);
+    Status
+    SetConfigCli(const std::string& parent_key, const std::string& child_key, const std::string& value);
 
     ///////////////////////////////////////////////////////////////////////////
     Status

core/unittest/server/test_config.cpp

@@ -35,6 +35,8 @@ static constexpr uint64_t GB = MB * 1024;
 
 }  // namespace
 
+namespace ms = milvus::server;
+
 TEST_F(ConfigTest, CONFIG_TEST) {
     milvus::server::ConfigMgr* config_mgr = milvus::server::YamlConfigMgr::GetInstance();
 
@@ -307,6 +309,154 @@ TEST_F(ConfigTest, SERVER_CONFIG_VALID_TEST) {
 #endif
 }
 
+std::string gen_get_command(const std::string& parent_node, const std::string& child_node) {
+    std::string cmd = "get " + parent_node + ms::CONFIG_NODE_DELIMITER + child_node;
+    return cmd;
+}
+
+std::string gen_set_command(const std::string& parent_node, const std::string& child_node, const std::string& value) {
+    std::string cmd = "set " + parent_node + ms::CONFIG_NODE_DELIMITER + child_node + " " + value;
+    return cmd;
+}
+
+TEST_F(ConfigTest, SERVER_CONFIG_CLI_TEST) {
+    std::string config_path(CONFIG_PATH);
+    milvus::server::Config& config = milvus::server::Config::GetInstance();
+    milvus::Status s;
+
+    std::string get_cmd, set_cmd;
+    std::string result, dummy;
+
+    /* server config */
+    std::string server_addr = "192.168.1.155";
+    get_cmd = gen_get_command(ms::CONFIG_SERVER, ms::CONFIG_SERVER_ADDRESS);
+    set_cmd = gen_set_command(ms::CONFIG_SERVER, ms::CONFIG_SERVER_ADDRESS, server_addr);
+    s = config.HandleConfigCli(dummy, set_cmd);
+    ASSERT_FALSE(s.ok());
+    s = config.HandleConfigCli(result, get_cmd);
+    ASSERT_TRUE(s.ok());
+
+    /* db config */
+    std::string db_primary_path = "/home/zilliz";
+    get_cmd = gen_get_command(ms::CONFIG_DB, ms::CONFIG_DB_PRIMARY_PATH);
+    set_cmd = gen_set_command(ms::CONFIG_DB, ms::CONFIG_DB_PRIMARY_PATH, db_primary_path);
+    s = config.HandleConfigCli(dummy, set_cmd);
+    ASSERT_FALSE(s.ok());
+    s = config.HandleConfigCli(result, get_cmd);
+    ASSERT_TRUE(s.ok());
+
+    /* metric config */
+    std::string metric_enable_monitor = "false";
+    get_cmd = gen_get_command(ms::CONFIG_METRIC, ms::CONFIG_METRIC_ENABLE_MONITOR);
+    set_cmd = gen_set_command(ms::CONFIG_METRIC, ms::CONFIG_METRIC_ENABLE_MONITOR, metric_enable_monitor);
+    s = config.HandleConfigCli(dummy, set_cmd);
+    ASSERT_FALSE(s.ok());
+    s = config.HandleConfigCli(result, get_cmd);
+    ASSERT_TRUE(s.ok());
+
+    /* cache config */
+    std::string cache_cpu_cache_capacity = "5";
+    get_cmd = gen_get_command(ms::CONFIG_CACHE, ms::CONFIG_CACHE_CPU_CACHE_CAPACITY);
+    set_cmd = gen_set_command(ms::CONFIG_CACHE, ms::CONFIG_CACHE_CPU_CACHE_CAPACITY, cache_cpu_cache_capacity);
+    s = config.HandleConfigCli(dummy, set_cmd);
+    ASSERT_TRUE(s.ok());
+    s = config.HandleConfigCli(result, get_cmd);
+    ASSERT_TRUE(s.ok());
+    ASSERT_TRUE(result == cache_cpu_cache_capacity);
+
+    std::string cache_cpu_cache_threshold = "0.1";
+    get_cmd = gen_get_command(ms::CONFIG_CACHE, ms::CONFIG_CACHE_CPU_CACHE_THRESHOLD);
+    set_cmd = gen_set_command(ms::CONFIG_CACHE, ms::CONFIG_CACHE_CPU_CACHE_THRESHOLD, cache_cpu_cache_threshold);
+    s = config.HandleConfigCli(dummy, set_cmd);
+    ASSERT_TRUE(s.ok());
+    s = config.HandleConfigCli(result, get_cmd);
+    ASSERT_TRUE(result == cache_cpu_cache_threshold);
+
+    std::string cache_insert_data = "true";
+    get_cmd = gen_get_command(ms::CONFIG_CACHE, ms::CONFIG_CACHE_CACHE_INSERT_DATA);
+    set_cmd = gen_set_command(ms::CONFIG_CACHE, ms::CONFIG_CACHE_CACHE_INSERT_DATA, cache_insert_data);
+    s = config.HandleConfigCli(dummy, set_cmd);
+    ASSERT_TRUE(s.ok());
+    s = config.HandleConfigCli(result, get_cmd);
+    ASSERT_TRUE(result == cache_insert_data);
+
+    /* engine config */
+    std::string engine_use_blas_threshold = "50";
+    get_cmd = gen_get_command(ms::CONFIG_ENGINE, ms::CONFIG_ENGINE_USE_BLAS_THRESHOLD);
+    set_cmd = gen_set_command(ms::CONFIG_ENGINE, ms::CONFIG_ENGINE_USE_BLAS_THRESHOLD, engine_use_blas_threshold);
+    s = config.HandleConfigCli(dummy, set_cmd);
+    ASSERT_TRUE(s.ok());
+    s = config.HandleConfigCli(result, get_cmd);
+    ASSERT_TRUE(s.ok());
+    ASSERT_TRUE(result == engine_use_blas_threshold);
+
+    std::string engine_omp_thread_num = "8";
+    get_cmd = gen_get_command(ms::CONFIG_ENGINE, ms::CONFIG_ENGINE_OMP_THREAD_NUM);
+    set_cmd = gen_set_command(ms::CONFIG_ENGINE, ms::CONFIG_ENGINE_OMP_THREAD_NUM, engine_omp_thread_num);
+    s = config.HandleConfigCli(dummy, set_cmd);
+    ASSERT_TRUE(s.ok());
+    s = config.HandleConfigCli(result, get_cmd);
+    ASSERT_TRUE(s.ok());
+    ASSERT_TRUE(result == engine_omp_thread_num);
+
+#ifdef MILVUS_GPU_VERSION
+    std::string engine_gpu_search_threshold = "800";
+    get_cmd = gen_get_command(ms::CONFIG_ENGINE, ms::CONFIG_ENGINE_GPU_SEARCH_THRESHOLD);
+    set_cmd = gen_set_command(ms::CONFIG_ENGINE, ms::CONFIG_ENGINE_GPU_SEARCH_THRESHOLD, engine_gpu_search_threshold);
+    s = config.HandleConfigCli(dummy, set_cmd);
+    ASSERT_TRUE(s.ok());
+    s = config.HandleConfigCli(result, get_cmd);
+    ASSERT_TRUE(s.ok());
+    ASSERT_TRUE(result == engine_gpu_search_threshold);
+
+    /* gpu resource config */
+    std::string resource_enable_gpu = "true";
+    get_cmd = gen_get_command(ms::CONFIG_GPU_RESOURCE, ms::CONFIG_GPU_RESOURCE_ENABLE);
+    set_cmd = gen_set_command(ms::CONFIG_GPU_RESOURCE, ms::CONFIG_GPU_RESOURCE_ENABLE, resource_enable_gpu);
+    s = config.HandleConfigCli(dummy, set_cmd);
+    ASSERT_TRUE(s.ok());
+    s = config.HandleConfigCli(result, get_cmd);
+    ASSERT_TRUE(s.ok());
+    ASSERT_TRUE(result == resource_enable_gpu);
+
+    std::string gpu_cache_capacity = "1";
+    get_cmd = gen_get_command(ms::CONFIG_GPU_RESOURCE, ms::CONFIG_GPU_RESOURCE_CACHE_CAPACITY);
+    set_cmd = gen_set_command(ms::CONFIG_GPU_RESOURCE, ms::CONFIG_GPU_RESOURCE_CACHE_CAPACITY, gpu_cache_capacity);
+    s = config.HandleConfigCli(dummy, set_cmd);
+    ASSERT_TRUE(s.ok());
+    s = config.HandleConfigCli(result, get_cmd);
+    ASSERT_TRUE(s.ok());
+    ASSERT_TRUE(result == gpu_cache_capacity);
+
+    std::string gpu_cache_threshold = "0.2";
+    get_cmd = gen_get_command(ms::CONFIG_GPU_RESOURCE, ms::CONFIG_GPU_RESOURCE_CACHE_THRESHOLD);
+    set_cmd = gen_set_command(ms::CONFIG_GPU_RESOURCE, ms::CONFIG_GPU_RESOURCE_CACHE_THRESHOLD, gpu_cache_threshold);
+    s = config.HandleConfigCli(dummy, set_cmd);
+    ASSERT_TRUE(s.ok());
+    s = config.HandleConfigCli(result, get_cmd);
+    ASSERT_TRUE(result == gpu_cache_threshold);
+
+    std::string search_resources = "gpu0";
+    get_cmd = gen_get_command(ms::CONFIG_GPU_RESOURCE, ms::CONFIG_GPU_RESOURCE_SEARCH_RESOURCES);
+    set_cmd = gen_set_command(ms::CONFIG_GPU_RESOURCE, ms::CONFIG_GPU_RESOURCE_SEARCH_RESOURCES, search_resources);
+    s = config.HandleConfigCli(dummy, set_cmd);
+    ASSERT_TRUE(s.ok());
+    s = config.HandleConfigCli(result, get_cmd);
+    ASSERT_TRUE(s.ok());
+    ASSERT_TRUE(result == search_resources);
+
+    std::string build_index_resources = "gpu0";
+    get_cmd = gen_get_command(ms::CONFIG_GPU_RESOURCE, ms::CONFIG_GPU_RESOURCE_BUILD_INDEX_RESOURCES);
+    set_cmd =
+        gen_set_command(ms::CONFIG_GPU_RESOURCE, ms::CONFIG_GPU_RESOURCE_BUILD_INDEX_RESOURCES, build_index_resources);
+    s = config.HandleConfigCli(dummy, set_cmd);
+    ASSERT_TRUE(s.ok());
+    s = config.HandleConfigCli(result, get_cmd);
+    ASSERT_TRUE(s.ok());
+    ASSERT_TRUE(result == build_index_resources);
+#endif
+}
+
 TEST_F(ConfigTest, SERVER_CONFIG_INVALID_TEST) {
     std::string config_path(CONFIG_PATH);
     milvus::server::Config& config = milvus::server::Config::GetInstance();