Rewrite config module (#2962)

* rewrite config module
Signed-off-by: Nwxyu <xy.wang@zilliz.com>

* clang-format
Signed-off-by: Nwxyu <xy.wang@zilliz.com>

* remove empty test
Signed-off-by: Nwxyu <xy.wang@zilliz.com>

* fix include
Signed-off-by: Nwxyu <xy.wang@zilliz.com>

* fix compile
Signed-off-by: Nwxyu <xy.wang@zilliz.com>

* fix clang-format
Signed-off-by: Nwxyu <xy.wang@zilliz.com>

core/CMakeLists.txt

@@ -52,6 +52,7 @@ import_mysql_inc()
 include(ExternalProject)
 include(DefineOptions)
 include(ThirdPartyPackages)
+include(cmake/test.cmake)
 
 using_ccache_if_defined(MILVUS_USE_CCACHE)
 

core/build.sh

 #!/bin/bash
-#
+
+# Compile jobs variable; Usage: $ jobs=12 ./build.sh ...
+if [[ ! ${jobs+1} ]]; then
+    jobs=$(nproc)
+fi
+
 BUILD_OUTPUT_DIR="cmake_build"
 BUILD_TYPE="Debug"
 BUILD_UNITTEST="OFF"
@@ -163,5 +168,5 @@ if [[ ${RUN_CPPLINT} == "ON" ]]; then
 else
 
   # compile and build
-    make -j 8 install || exit 1
+  make -j ${jobs} install || exit 1
 fi

core/cmake/ThirdPartyPackages.cmake

@@ -499,6 +499,8 @@ macro(build_prometheus)
             ${EP_LOG_OPTIONS}
             CMAKE_ARGS
             ${PROMETHEUS_CMAKE_ARGS}
+            UPDATE_COMMAND
+            ""
             BUILD_COMMAND
             ${MAKE}
             ${MAKE_BUILD_ARGS}

core/cmake/test.cmake

+#-------------------------------------------------------------------------------
+# Copyright (C) 2019-2020 Zilliz. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+# or implied. See the License for the specific language governing permissions and limitations under the License.
+#-------------------------------------------------------------------------------
+
+set(gtest_libraries
+    gtest
+    gmock
+    gtest_main
+    gmock_main
+    )
+
+macro(ADD_TEST)
+    cmake_parse_arguments(UT "" "TARGET" "SOURCES;LIBS" ${ARGN})
+    #    message("UT_TARGET = ${UT_TARGET}")
+    #    message("UT_SOURCES = ${UT_SOURCES}")
+    #    message("UT_LIBS = ${UT_LIBS}")
+    #    message("ARGN = ${ARGN}")
+
+    add_executable(${UT_TARGET} ${UT_SOURCES})
+    target_link_libraries(${UT_TARGET} ${UT_LIBS})
+endmacro()

core/src/CMakeLists.txt

@@ -18,6 +18,8 @@ include_directories(${MILVUS_THIRDPARTY_SRC})
 include_directories(${MILVUS_ENGINE_SRC}/grpc/gen-status)
 include_directories(${MILVUS_ENGINE_SRC}/grpc/gen-milvus)
 
+add_subdirectory(config)
+
 set(FOUND_OPENBLAS "unknown")
 add_subdirectory(index)
 if (FAISS_WITH_MKL)
@@ -30,8 +32,6 @@ foreach (dir ${INDEX_INCLUDE_DIRS})
 endforeach ()
 
 aux_source_directory(${MILVUS_ENGINE_SRC}/cache cache_files)
-aux_source_directory(${MILVUS_ENGINE_SRC}/config config_files)
-aux_source_directory(${MILVUS_ENGINE_SRC}/config/handler config_handler_files)
 aux_source_directory(${MILVUS_ENGINE_SRC}/metrics metrics_files)
 aux_source_directory(${MILVUS_ENGINE_SRC}/metrics/prometheus metrics_prometheus_files)
 aux_source_directory(${MILVUS_ENGINE_SRC}/db db_main_files)
@@ -309,14 +309,13 @@ target_link_libraries(tracing ${tracing_lib})
 
 set(server_libs
         milvus_engine
+        config
         metrics
         tracing
         oatpp
         )
 
 add_executable(milvus_server
-        ${config_files}
-        ${config_handler_files}
         ${metrics_files}
         ${query_files}
         ${search_files}

core/src/cache/CpuCacheMgr.cpp

@@ -15,34 +15,21 @@
 
 #include <fiu-local.h>
 
-#include "config/Config.h"
+#include "config/ServerConfig.h"
 #include "utils/Log.h"
 
 namespace milvus {
 namespace cache {
 
-namespace {
-// constexpr int64_t unit = 1024 * 1024 * 1024;
-constexpr int64_t unit = 1;
-}  // namespace
-
 CpuCacheMgr::CpuCacheMgr() {
-    // All config values have been checked in Config::ValidateConfig()
-    server::Config& config = server::Config::GetInstance();
-
-    int64_t cpu_cache_cap;
-    config.GetCacheConfigCpuCacheCapacity(cpu_cache_cap);
-    int64_t cap = cpu_cache_cap * unit;
-    LOG_SERVER_DEBUG_ << "cpu cache.size: " << cap;
-    LOG_SERVER_INFO_ << "cpu cache.size: " << cap;
-    cache_ = std::make_shared<Cache<DataObjPtr>>(cap, 1UL << 32, "[CACHE CPU]");
+    cache_ = std::make_shared<Cache<DataObjPtr>>(config.cache.cache_size(), 1UL << 32, "[CACHE CPU]");
 
-    float cpu_cache_threshold;
-    config.GetCacheConfigCpuCacheThreshold(cpu_cache_threshold);
-    cache_->set_freemem_percent(cpu_cache_threshold);
+    cache_->set_freemem_percent(config.cache.cpu_cache_threshold());
+    ConfigMgr::GetInstance().Attach("cache.cache_size", this);
+}
 
-    SetIdentity("CpuCacheMgr");
-    AddCpuCacheCapacityListener();
+CpuCacheMgr::~CpuCacheMgr() {
+    ConfigMgr::GetInstance().Detach("cache.cache_size", this);
 }
 
 CpuCacheMgr*
@@ -58,8 +45,8 @@ CpuCacheMgr::GetIndex(const std::string& key) {
 }
 
 void
-CpuCacheMgr::OnCpuCacheCapacityChanged(int64_t value) {
-    SetCapacity(value * unit);
+CpuCacheMgr::ConfigUpdate(const std::string& name) {
+    SetCapacity(config.cache.cache_size());
 }
 
 }  // namespace cache

core/src/cache/CpuCacheMgr.h

@@ -16,15 +16,17 @@
 
 #include "cache/CacheMgr.h"
 #include "cache/DataObj.h"
-#include "config/handler/CacheConfigHandler.h"
+#include "config/ConfigMgr.h"
 
 namespace milvus {
 namespace cache {
 
-class CpuCacheMgr : public CacheMgr<DataObjPtr>, public server::CacheConfigHandler {
+class CpuCacheMgr : public CacheMgr<DataObjPtr>, public ConfigObserver {
  private:
     CpuCacheMgr();
 
+    ~CpuCacheMgr();
+
  public:
     // TODO(myh): use smart pointer instead
     static CpuCacheMgr*
@@ -33,9 +35,9 @@ class CpuCacheMgr : public CacheMgr<DataObjPtr>, public server::CacheConfigHandl
     DataObjPtr
     GetIndex(const std::string& key);
 
- protected:
+ public:
     void
-    OnCpuCacheCapacityChanged(int64_t value) override;
+    ConfigUpdate(const std::string& name) override;
 };
 
 }  // namespace cache

core/src/cache/GpuCacheMgr.cpp

@@ -10,7 +10,7 @@
 // or implied. See the License for the specific language governing permissions and limitations under the License.
 
 #include "cache/GpuCacheMgr.h"
-#include "config/Config.h"
+#include "config/ServerConfig.h"
 #include "utils/Log.h"
 
 #include <fiu-local.h>
@@ -29,27 +29,15 @@ constexpr int64_t G_BYTE = 1024 * 1024 * 1024;
 }
 
 GpuCacheMgr::GpuCacheMgr(int64_t gpu_id) : gpu_id_(gpu_id) {
-    // All config values have been checked in Config::ValidateConfig()
-    server::Config& config = server::Config::GetInstance();
-
-    int64_t gpu_cache_cap;
-    config.GetGpuResourceConfigCacheCapacity(gpu_cache_cap);
-    int64_t cap = gpu_cache_cap * G_BYTE;
     std::string header = "[CACHE GPU" + std::to_string(gpu_id) + "]";
-    cache_ = std::make_shared<Cache<DataObjPtr>>(cap, 1UL << 32, header);
-
-    float gpu_mem_threshold;
-    config.GetGpuResourceConfigCacheThreshold(gpu_mem_threshold);
-    cache_->set_freemem_percent(gpu_mem_threshold);
+    cache_ = std::make_shared<Cache<DataObjPtr>>(config.gpu.cache_size(), 1UL << 32, header);
 
-    SetIdentity("GpuCacheMgr");
-    AddGpuEnableListener();
-    AddGpuCacheCapacityListener();
+    cache_->set_freemem_percent(config.gpu.cache_threshold());
+    ConfigMgr::GetInstance().Attach("gpu.cache_threshold", this);
 }
 
 GpuCacheMgr::~GpuCacheMgr() {
-    server::Config& config = server::Config::GetInstance();
-    config.CancelCallBack(server::CONFIG_GPU_RESOURCE, server::CONFIG_GPU_RESOURCE_ENABLE, identity_);
+    ConfigMgr::GetInstance().Detach("gpu.cache_threshold", this);
 }
 
 DataObjPtr
@@ -82,10 +70,8 @@ GpuCacheMgr::GetInstance(int64_t gpu_id) {
 }
 
 void
-GpuCacheMgr::OnGpuCacheCapacityChanged(int64_t capacity) {
-    for (auto& iter : instance_) {
-        iter.second->SetCapacity(capacity * G_BYTE);
-    }
+GpuCacheMgr::ConfigUpdate(const std::string& name) {
+    for (auto& it : instance_) it.second->SetCapacity(config.gpu.cache_size());
 }
 
 #endif

core/src/cache/GpuCacheMgr.h

@@ -17,7 +17,7 @@
 
 #include "cache/CacheMgr.h"
 #include "cache/DataObj.h"
-#include "config/handler/GpuResourceConfigHandler.h"
+#include "config/ConfigMgr.h"
 
 namespace milvus {
 namespace cache {
@@ -27,7 +27,7 @@ class GpuCacheMgr;
 using GpuCacheMgrPtr = std::shared_ptr<GpuCacheMgr>;
 using MutexPtr = std::shared_ptr<std::mutex>;
 
-class GpuCacheMgr : public CacheMgr<DataObjPtr>, public server::GpuResourceConfigHandler {
+class GpuCacheMgr : public CacheMgr<DataObjPtr>, public ConfigObserver {
  public:
     explicit GpuCacheMgr(int64_t gpu_id);
 
@@ -45,16 +45,15 @@ class GpuCacheMgr : public CacheMgr<DataObjPtr>, public server::GpuResourceConfi
     static GpuCacheMgrPtr
     GetInstance(int64_t gpu_id);
 
- protected:
+ public:
     void
-    OnGpuCacheCapacityChanged(int64_t capacity) override;
+    ConfigUpdate(const std::string& name) override;
 
  private:
     bool gpu_enable_ = true;
     int64_t gpu_id_;
     static std::mutex global_mutex_;
     static std::unordered_map<int64_t, GpuCacheMgrPtr> instance_;
-    std::string identity_;
 };
 #endif
 

core/src/config/CMakeLists.txt

+#-------------------------------------------------------------------------------
+# Copyright (C) 2019-2020 Zilliz. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+# or implied. See the License for the specific language governing permissions and limitations under the License.
+#-------------------------------------------------------------------------------
+
+# library
+set(CONFIG_SRCS
+    ConfigMgr.h
+    ConfigMgr.cpp
+    ConfigType.h
+    ConfigType.cpp
+    ServerConfig.h
+    ServerConfig.cpp
+    )
+
+add_library(config ${CONFIG_SRCS})
+add_dependencies(config yaml-cpp)
+
+# unitests
+if (BUILD_UNIT_TEST STREQUAL "ON")
+
+    # config_test
+    add_test(
+        TARGET ConfigTypeTest
+        SOURCES
+            ConfigTypeTest1.cpp
+            ConfigTypeTest2.cpp
+        LIBS 
+            config
+            ${gtest_libraries}
+            yaml-cpp
+        )
+
+endif()

core/src/config/Config.h

-// Copyright (C) 2019-2020 Zilliz. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software distributed under the License
-// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
-// or implied. See the License for the specific language governing permissions and limitations under the License.
-
-#pragma once
-
-#include <functional>
-#include <mutex>
-#include <string>
-#include <unordered_map>
-#include <vector>
-
-#include "config/ConfigNode.h"
-#include "utils/Status.h"
-
-namespace milvus {
-namespace server {
-
-using ConfigCallBackF = std::function<Status(const std::string&)>;
-
-extern const char* CONFIG_NODE_DELIMITER;
-extern const char* CONFIG_VERSION;
-
-/* cluster config */
-extern const char* CONFIG_CLUSTER;
-extern const char* CONFIG_CLUSTER_ENABLE;
-extern const char* CONFIG_CLUSTER_ENABLE_DEFAULT;
-extern const char* CONFIG_CLUSTER_ROLE;
-extern const char* CONFIG_CLUSTER_ROLE_DEFAULT;
-
-/* general config */
-extern const char* CONFIG_GENERAL;
-extern const char* CONFIG_GENERAL_TIMEZONE;
-extern const char* CONFIG_GENERAL_TIMEZONE_DEFAULT;
-extern const char* CONFIG_GENERAL_METAURI;
-extern const char* CONFIG_GENERAL_METAURI_DEFAULT;
-
-/* network config */
-extern const char* CONFIG_NETWORK;
-extern const char* CONFIG_NETWORK_BIND_ADDRESS;
-extern const char* CONFIG_NETWORK_BIND_ADDRESS_DEFAULT;
-extern const char* CONFIG_NETWORK_BIND_PORT;
-extern const char* CONFIG_NETWORK_BIND_PORT_DEFAULT;
-extern const char* CONFIG_NETWORK_HTTP_ENABLE;
-extern const char* CONFIG_NETWORK_HTTP_ENABLE_DEFAULT;
-extern const char* CONFIG_NETWORK_HTTP_PORT;
-extern const char* CONFIG_NETWORK_HTTP_PORT_DEFAULT;
-
-/* db config */
-extern const char* CONFIG_DB;
-extern const char* CONFIG_DB_ARCHIVE_DISK_THRESHOLD;
-extern const char* CONFIG_DB_ARCHIVE_DISK_THRESHOLD_DEFAULT;
-extern const char* CONFIG_DB_ARCHIVE_DAYS_THRESHOLD;
-extern const char* CONFIG_DB_ARCHIVE_DAYS_THRESHOLD_DEFAULT;
-
-/* storage config */
-extern const char* CONFIG_STORAGE;
-extern const char* CONFIG_STORAGE_PATH;
-extern const char* CONFIG_STORAGE_PATH_DEFAULT;
-extern const char* CONFIG_STORAGE_AUTO_FLUSH_INTERVAL;
-extern const char* CONFIG_STORAGE_AUTO_FLUSH_INTERVAL_DEFAULT;
-extern const char* CONFIG_STORAGE_FILE_CLEANUP_TIMEOUT;
-extern const int64_t CONFIG_STORAGE_FILE_CLEANUP_TIMEOUT_MIN;
-extern const int64_t CONFIG_STORAGE_FILE_CLEANUP_TIMEOUT_MAX;
-
-/* cache config */
-extern const char* CONFIG_CACHE;
-extern const char* CONFIG_CACHE_CPU_CACHE_CAPACITY;
-extern const char* CONFIG_CACHE_CPU_CACHE_CAPACITY_DEFAULT;
-extern const char* CONFIG_CACHE_CPU_CACHE_THRESHOLD;
-extern const char* CONFIG_CACHE_CPU_CACHE_THRESHOLD_DEFAULT;
-extern const char* CONFIG_CACHE_INSERT_BUFFER_SIZE;
-extern const char* CONFIG_CACHE_INSERT_BUFFER_SIZE_DEFAULT;
-extern const char* CONFIG_CACHE_CACHE_INSERT_DATA;
-extern const char* CONFIG_CACHE_CACHE_INSERT_DATA_DEFAULT;
-extern const char* CONFIG_CACHE_PRELOAD_COLLECTION;
-extern const char* CONFIG_CACHE_PRELOAD_COLLECTION_DEFAULT;
-
-/* metric config */
-extern const char* CONFIG_METRIC;
-extern const char* CONFIG_METRIC_ENABLE_MONITOR;
-extern const char* CONFIG_METRIC_ENABLE_MONITOR_DEFAULT;
-extern const char* CONFIG_METRIC_ADDRESS;
-extern const char* CONFIG_METRIC_ADDRESS_DEFAULT;
-extern const char* CONFIG_METRIC_PORT;
-extern const char* CONFIG_METRIC_PORT_DEFAULT;
-
-/* engine config */
-extern const char* CONFIG_ENGINE;
-extern const char* CONFIG_ENGINE_USE_BLAS_THRESHOLD;
-extern const char* CONFIG_ENGINE_USE_BLAS_THRESHOLD_DEFAULT;
-extern const char* CONFIG_ENGINE_OMP_THREAD_NUM;
-extern const char* CONFIG_ENGINE_OMP_THREAD_NUM_DEFAULT;
-extern const char* CONFIG_ENGINE_SIMD_TYPE;
-extern const char* CONFIG_ENGINE_SIMD_TYPE_DEFAULT;
-extern const char* CONFIG_ENGINE_SEARCH_COMBINE_MAX_NQ;
-extern const char* CONFIG_ENGINE_SEARCH_COMBINE_MAX_NQ_DEFAULT;
-
-/* gpu resource config */
-extern const char* CONFIG_GPU_RESOURCE;
-extern const char* CONFIG_GPU_RESOURCE_ENABLE;
-extern const char* CONFIG_GPU_RESOURCE_ENABLE_DEFAULT;
-extern const char* CONFIG_GPU_RESOURCE_CACHE_CAPACITY;
-extern const char* CONFIG_GPU_RESOURCE_CACHE_CAPACITY_DEFAULT;
-extern const char* CONFIG_GPU_RESOURCE_CACHE_THRESHOLD;
-extern const char* CONFIG_GPU_RESOURCE_CACHE_THRESHOLD_DEFAULT;
-extern const char* CONFIG_GPU_RESOURCE_GPU_SEARCH_THRESHOLD;
-extern const char* CONFIG_GPU_RESOURCE_GPU_SEARCH_THRESHOLD_DEFAULT;
-extern const char* CONFIG_GPU_RESOURCE_DELIMITER;
-extern const char* CONFIG_GPU_RESOURCE_SEARCH_RESOURCES;
-extern const char* CONFIG_GPU_RESOURCE_SEARCH_RESOURCES_DEFAULT;
-extern const char* CONFIG_GPU_RESOURCE_BUILD_INDEX_RESOURCES;
-extern const char* CONFIG_GPU_RESOURCE_BUILD_INDEX_RESOURCES_DEFAULT;
-
-/* tracing config */
-extern const char* CONFIG_TRACING;
-extern const char* CONFIG_TRACING_JSON_CONFIG_PATH;
-
-/* wal config */
-extern const char* CONFIG_WAL;
-extern const char* CONFIG_WAL_ENABLE;
-extern const char* CONFIG_WAL_ENABLE_DEFAULT;
-extern const char* CONFIG_WAL_RECOVERY_ERROR_IGNORE;
-extern const char* CONFIG_WAL_RECOVERY_ERROR_IGNORE_DEFAULT;
-extern const char* CONFIG_WAL_BUFFER_SIZE;
-extern const char* CONFIG_WAL_BUFFER_SIZE_DEFAULT;
-extern const int64_t CONFIG_WAL_BUFFER_SIZE_MIN;
-extern const int64_t CONFIG_WAL_BUFFER_SIZE_MAX;
-extern const char* CONFIG_WAL_WAL_PATH;
-extern const char* CONFIG_WAL_WAL_PATH_DEFAULT;
-
-/* logs config */
-extern const char* CONFIG_LOGS;
-extern const char* CONFIG_LOGS_LEVEL;
-extern const char* CONFIG_LOGS_LEVEL_DEFAULT;
-extern const char* CONFIG_LOGS_TRACE_ENABLE;
-extern const char* CONFIG_LOGS_TRACE_ENABLE_DEFAULT;
-extern const char* CONFIG_LOGS_PATH;
-extern const char* CONFIG_LOGS_MAX_LOG_FILE_SIZE;
-extern const char* CONFIG_LOGS_MAX_LOG_FILE_SIZE_DEFAULT;
-extern const int64_t CONFIG_LOGS_MAX_LOG_FILE_SIZE_MIN;
-extern const int64_t CONFIG_LOGS_MAX_LOG_FILE_SIZE_MAX;
-extern const char* CONFIG_LOGS_LOG_ROTATE_NUM;
-extern const char* CONFIG_LOGS_LOG_ROTATE_NUM_DEFAULT;
-extern const int64_t CONFIG_LOGS_LOG_ROTATE_NUM_MIN;
-extern const int64_t CONFIG_LOGS_LOG_ROTATE_NUM_MAX;
-
-class Config {
- private:
-    Config();
-
- public:
-    static Config&
-    GetInstance();
-    Status
-    LoadConfigFile(const std::string& filename);
-    Status
-    ValidateConfig();
-    Status
-    ResetDefaultConfig();
-    void
-    GetConfigJsonStr(std::string& result, int64_t indent = -1);
-    Status
-    ProcessConfigCli(std::string& result, const std::string& cmd);
-
-    Status
-    GenUniqueIdentityID(const std::string& identity, std::string& uid);
-
-    Status
-    RegisterCallBack(const std::string& node, const std::string& sub_node, const std::string& key,
-                     ConfigCallBackF& callback);
-
-    Status
-    CancelCallBack(const std::string& node, const std::string& sub_node, const std::string& key);
-
- private:
-    ConfigNode&
-    GetConfigRoot();
-    ConfigNode&
-    GetConfigNode(const std::string& name);
-    bool
-    ConfigNodeValid(const std::string& parent_key, const std::string& child_key);
-    Status
-    GetConfigValueInMem(const std::string& parent_key, const std::string& child_key, std::string& value);
-    Status
-    SetConfigValueInMem(const std::string& parent_key, const std::string& child_key, const std::string& value);
-    Status
-    GetConfigCli(std::string& value, const std::string& parent_key, const std::string& child_key);
-    Status
-    SetConfigCli(const std::string& parent_key, const std::string& child_key, const std::string& value);
-
-    Status
-    UpdateFileConfigFromMem(const std::string& parent_key, const std::string& child_key);
-
-    ///////////////////////////////////////////////////////////////////////////
-    Status
-    CheckConfigVersion(const std::string& value);
-
-    /* cluster config */
-    Status
-    CheckClusterConfigEnable(const std::string& value);
-    Status
-    CheckClusterConfigRole(const std::string& value);
-
-    /* general config */
-    Status
-    CheckGeneralConfigTimezone(const std::string& value);
-    Status
-    CheckGeneralConfigMetaURI(const std::string& value);
-
-    /* network config */
-    Status
-    CheckNetworkConfigBindAddress(const std::string& value);
-    Status
-    CheckNetworkConfigBindPort(const std::string& value);
-    Status
-    CheckNetworkConfigHTTPEnable(const std::string& value);
-    Status
-    CheckNetworkConfigHTTPPort(const std::string& value);
-
-    /* db config */
-    Status
-    CheckDBConfigArchiveDiskThreshold(const std::string& value);
-    Status
-    CheckDBConfigArchiveDaysThreshold(const std::string& value);
-
-    /* storage config */
-    Status
-    CheckStorageConfigPath(const std::string& value);
-    Status
-    CheckStorageConfigAutoFlushInterval(const std::string& value);
-    Status
-    CheckStorageConfigFileCleanupTimeout(const std::string& value);
-
-    /* metric config */
-    Status
-    CheckMetricConfigEnableMonitor(const std::string& value);
-    Status
-    CheckMetricConfigAddress(const std::string& value);
-    Status
-    CheckMetricConfigPort(const std::string& value);
-
-    /* cache config */
-    Status
-    CheckCacheConfigCpuCacheCapacity(const std::string& value);
-    Status
-    CheckCacheConfigCpuCacheThreshold(const std::string& value);
-    Status
-    CheckCacheConfigInsertBufferSize(const std::string& value);
-    Status
-    CheckCacheConfigCacheInsertData(const std::string& value);
-    Status
-    CheckCacheConfigPreloadCollection(const std::string& value);
-
-    /* engine config */
-    Status
-    CheckEngineConfigUseBlasThreshold(const std::string& value);
-    Status
-    CheckEngineConfigOmpThreadNum(const std::string& value);
-    Status
-    CheckEngineConfigSimdType(const std::string& value);
-    Status
-    CheckEngineSearchCombineMaxNq(const std::string& value);
-
-    /* gpu resource config */
-#ifdef MILVUS_GPU_VERSION
-    Status
-    CheckGpuResourceConfigEnable(const std::string& value);
-    Status
-    CheckGpuResourceConfigCacheCapacity(const std::string& value);
-    Status
-    CheckGpuResourceConfigCacheThreshold(const std::string& value);
-    Status
-    CheckGpuResourceConfigGpuSearchThreshold(const std::string& value);
-    Status
-    CheckGpuResourceConfigSearchResources(const std::vector<std::string>& value);
-    Status
-    CheckGpuResourceConfigBuildIndexResources(const std::vector<std::string>& value);
-#endif
-
-    /* tracing config */
-    Status
-    CheckTracingConfigJsonConfigPath(const std::string& value);
-
-    /* wal config */
-    Status
-    CheckWalConfigEnable(const std::string& value);
-    Status
-    CheckWalConfigRecoveryErrorIgnore(const std::string& value);
-    Status
-    CheckWalConfigBufferSize(const std::string& value);
-    Status
-    CheckWalConfigWalPath(const std::string& value);
-
-    /* logs config */
-    Status
-    CheckLogsLevel(const std::string& level);
-    Status
-    CheckLogsTraceEnable(const std::string& value);
-    Status
-    CheckLogsPath(const std::string& value);
-    Status
-    CheckLogsMaxLogFileSize(const std::string& value);
-    Status
-    CheckLogsLogRotateNum(const std::string& value);
-
-    std::string
-    GetConfigStr(const std::string& parent_key, const std::string& child_key, const std::string& default_value = "");
-    std::string
-    GetConfigSequenceStr(const std::string& parent_key, const std::string& child_key, const std::string& delim = ",",
-                         const std::string& default_value = "");
-    Status
-    GetConfigVersion(std::string& value);
-
-    Status
-    ExecCallBacks(const std::string& node, const std::string& sub_node, const std::string& value);
-
- public:
-    /* cluster config */
-    Status
-    GetClusterConfigEnable(bool& value);
-    Status
-    GetClusterConfigRole(std::string& value);
-
-    /* general config */
-    Status
-    GetGeneralConfigTimezone(std::string& value);
-    Status
-    GetGeneralConfigMetaURI(std::string& value);
-
-    /* network config */
-    Status
-    GetNetworkConfigBindAddress(std::string& value);
-    Status
-    GetNetworkConfigBindPort(std::string& value);
-    Status
-    GetNetworkConfigHTTPEnable(bool& value);
-    Status
-    GetNetworkConfigHTTPPort(std::string& value);
-
-    /* db config */
-    Status
-    GetDBConfigArchiveDiskThreshold(int64_t& value);
-    Status
-    GetDBConfigArchiveDaysThreshold(int64_t& value);
-
-    /* storage config */
-    Status
-    GetStorageConfigPath(std::string& value);
-    Status
-    GetStorageConfigAutoFlushInterval(int64_t& value);
-    Status
-    GetStorageConfigFileCleanupTimeup(int64_t& value);
-
-    /* metric config */
-    Status
-    GetMetricConfigEnableMonitor(bool& value);
-    Status
-    GetMetricConfigAddress(std::string& value);
-    Status
-    GetMetricConfigPort(std::string& value);
-
-    /* cache config */
-    Status
-    GetCacheConfigCpuCacheCapacity(int64_t& value);
-    Status
-    GetCacheConfigCpuCacheThreshold(float& value);
-    Status
-    GetCacheConfigInsertBufferSize(int64_t& value);
-    Status
-    GetCacheConfigCacheInsertData(bool& value);
-    Status
-    GetCacheConfigPreloadCollection(std::string& value);
-
-    /* engine config */
-    Status
-    GetEngineConfigUseBlasThreshold(int64_t& value);
-    Status
-    GetEngineConfigOmpThreadNum(int64_t& value);
-    Status
-    GetEngineConfigSimdType(std::string& value);
-    Status
-    GetEngineSearchCombineMaxNq(int64_t& value);
-
-    /* gpu resource config */
-#ifdef MILVUS_GPU_VERSION
-    Status
-    GetGpuResourceConfigEnable(bool& value);
-    Status
-    GetGpuResourceConfigCacheCapacity(int64_t& value);
-    Status
-    GetGpuResourceConfigCacheThreshold(float& value);
-    Status
-    GetGpuResourceConfigGpuSearchThreshold(int64_t& value);
-    Status
-    GetGpuResourceConfigSearchResources(std::vector<int64_t>& value);
-    Status
-    GetGpuResourceConfigBuildIndexResources(std::vector<int64_t>& value);
-#endif
-
-    /* tracing config */
-    Status
-    GetTracingConfigJsonConfigPath(std::string& value);
-
-    /* wal config */
-    Status
-    GetWalConfigEnable(bool& value);
-    Status
-    GetWalConfigRecoveryErrorIgnore(bool& value);
-    Status
-    GetWalConfigBufferSize(int64_t& value);
-    Status
-    GetWalConfigWalPath(std::string& value);
-
-    /* logs config */
-    Status
-    GetLogsLevel(std::string& value);
-    Status
-    GetLogsTraceEnable(bool& value);
-    Status
-    GetLogsPath(std::string& value);
-    Status
-    GetLogsMaxLogFileSize(int64_t& value);
-    Status
-    GetLogsLogRotateNum(int64_t& value);
-
-    Status
-    GetServerRestartRequired(bool& required);
-
- public:
-    /* cluster config */
-    Status
-    SetClusterConfigEnable(const std::string& value);
-    Status
-    SetClusterConfigRole(const std::string& value);
-
-    /* general config */
-    Status
-    SetGeneralConfigTimezone(const std::string& value);
-    Status
-    SetGeneralConfigMetaURI(const std::string& value);
-
-    /* network config */
-    Status
-    SetNetworkConfigBindAddress(const std::string& value);
-    Status
-    SetNetworkConfigBindPort(const std::string& value);
-    Status
-    SetNetworkConfigHTTPEnable(const std::string& value);
-    Status
-    SetNetworkConfigHTTPPort(const std::string& value);
-
-    /* db config */
-    Status
-    SetDBConfigArchiveDiskThreshold(const std::string& value);
-    Status
-    SetDBConfigArchiveDaysThreshold(const std::string& value);
-
-    /* storage config */
-    Status
-    SetStorageConfigPath(const std::string& value);
-    Status
-    SetStorageConfigAutoFlushInterval(const std::string& value);
-    Status
-    SetStorageConfigFileCleanupTimeout(const std::string& value);
-
-    /* metric config */
-    Status
-    SetMetricConfigEnableMonitor(const std::string& value);
-    Status
-    SetMetricConfigAddress(const std::string& value);
-    Status
-    SetMetricConfigPort(const std::string& value);
-
-    /* cache config */
-    Status
-    SetCacheConfigCpuCacheCapacity(const std::string& value);
-    Status
-    SetCacheConfigCpuCacheThreshold(const std::string& value);
-    Status
-    SetCacheConfigInsertBufferSize(const std::string& value);
-    Status
-    SetCacheConfigCacheInsertData(const std::string& value);
-    Status
-    SetCacheConfigPreloadCollection(const std::string& value);
-
-    /* engine config */
-    Status
-    SetEngineConfigUseBlasThreshold(const std::string& value);
-    Status
-    SetEngineConfigOmpThreadNum(const std::string& value);
-    Status
-    SetEngineConfigSimdType(const std::string& value);
-    Status
-    SetEngineSearchCombineMaxNq(const std::string& value);
-
-    /* gpu resource config */
-#ifdef MILVUS_GPU_VERSION
-    Status
-    SetGpuResourceConfigEnable(const std::string& value);
-    Status
-    SetGpuResourceConfigCacheCapacity(const std::string& value);
-    Status
-    SetGpuResourceConfigCacheThreshold(const std::string& value);
-    Status
-    SetGpuResourceConfigGpuSearchThreshold(const std::string& value);
-    Status
-    SetGpuResourceConfigSearchResources(const std::string& value);
-    Status
-    SetGpuResourceConfigBuildIndexResources(const std::string& value);
-#endif
-
-    /* tracing config */
-    Status
-    SetTracingConfigJsonConfigPath(const std::string& value);
-
-    /* wal config */
-    Status
-    SetWalConfigEnable(const std::string& value);
-    Status
-    SetWalConfigRecoveryErrorIgnore(const std::string& value);
-    Status
-    SetWalConfigBufferSize(const std::string& value);
-    Status
-    SetWalConfigWalPath(const std::string& value);
-
-    /* logs config */
-    Status
-    SetLogsLevel(const std::string& value);
-    Status
-    SetLogsTraceEnable(const std::string& value);
-    Status
-    SetLogsPath(const std::string& value);
-    Status
-    SetLogsMaxLogFileSize(const std::string& value);
-    Status
-    SetLogsLogRotateNum(const std::string& value);
-
- private:
-    bool restart_required_ = false;
-    std::string config_file_;
-    std::unordered_map<std::string, std::unordered_map<std::string, std::string>> config_map_;
-    std::unordered_map<std::string, std::unordered_map<std::string, ConfigCallBackF>> config_callback_;
-    std::mutex callback_mutex_;
-    std::mutex mutex_;
-};
-
-}  // namespace server
-}  // namespace milvus

core/src/config/ConfigMgr.cpp

+// Copyright (C) 2019-2020 Zilliz. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software distributed under the License
+// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+// or implied. See the License for the specific language governing permissions and limitations under the License.
+
+#include <yaml-cpp/yaml.h>
+#include <cstring>
+#include <limits>
+#include <unordered_map>
+
+#include "config/ConfigMgr.h"
+#include "config/ServerConfig.h"
+
+namespace {
+const int64_t MB = (1024ll * 1024);
+const int64_t GB = (1024ll * 1024 * 1024);
+
+void
+Flatten(const YAML::Node& node, std::unordered_map<std::string, std::string>& target, const std::string& prefix) {
+    for (auto& it : node) {
+        auto key = prefix.empty() ? it.first.as<std::string>() : prefix + "." + it.first.as<std::string>();
+        switch (it.second.Type()) {
+            case YAML::NodeType::Null: {
+                target[key] = "";
+                break;
+            }
+            case YAML::NodeType::Scalar: {
+                target[key] = it.second.as<std::string>();
+                break;
+            }
+            case YAML::NodeType::Sequence: {
+                std::string value;
+                for (auto& sub : it.second) value += sub.as<std::string>() + ",";
+                target[key] = value;
+                break;
+            }
+            case YAML::NodeType::Map: {
+                Flatten(it.second, target, key);
+                break;
+            }
+            case YAML::NodeType::Undefined: {
+                throw "Unexpected";
+            }
+            default:
+                break;
+        }
+    }
+}
+
+void
+ThrowIfNotSuccess(const milvus::ConfigStatus& cs) {
+    if (cs.set_return != milvus::SetReturn::SUCCESS) {
+        throw cs;
+    }
+}
+
+};  // namespace
+
+namespace milvus {
+
+ConfigMgr ConfigMgr::instance;
+
+ConfigMgr::ConfigMgr() {
+    config_list_ = {
+        /* version */
+        CreateStringConfig("version", false, &config.version.value, "unknown", nullptr, nullptr),
+
+        /* cluster */
+        CreateBoolConfig("cluster.enable", false, &config.cluster.enable.value, false, nullptr, nullptr),
+        CreateEnumConfig("cluster.role", false, &ClusterRoleMap, &config.cluster.role.value, ClusterRole::RW, nullptr,
+                         nullptr),
+
+        /* general */
+        CreateStringConfig("general.timezone", false, &config.general.timezone.value, "UTC+8", nullptr, nullptr),
+        CreateStringConfig("general.meta_uri", false, &config.general.meta_uri.value, "sqlite://:@:/", nullptr,
+                           nullptr),
+
+        /* network */
+        CreateStringConfig("network.bind.address", false, &config.network.bind.address.value, "0.0.0.0", nullptr,
+                           nullptr),
+        CreateIntegerConfig("network.bind.port", false, 0, 65535, &config.network.bind.port.value, 19530, nullptr,
+                            nullptr),
+        CreateBoolConfig("network.http.enable", false, &config.network.http.enable.value, true, nullptr, nullptr),
+        CreateIntegerConfig("network.http.port", false, 0, 65535, &config.network.http.port.value, 19121, nullptr,
+                            nullptr),
+
+        /* storage */
+        CreateStringConfig("storage.path", false, &config.storage.path.value, "/var/lib/milvus", nullptr, nullptr),
+        CreateIntegerConfig("storage.auto_flush_interval", false, 0, std::numeric_limits<int64_t>::max(),
+                            &config.storage.auto_flush_interval.value, 1, nullptr, nullptr),
+        CreateIntegerConfig("storage.file_cleanup_timeout", false, 0, 3600, &config.storage.file_cleanup_timeout.value,
+                            10, nullptr, nullptr),
+
+        /* wal */
+        CreateBoolConfig("wal.enable", false, &config.wal.enable.value, true, nullptr, nullptr),
+        CreateBoolConfig("wal.recovery_error_ignore", false, &config.wal.recovery_error_ignore.value, false, nullptr,
+                         nullptr),
+        CreateSizeConfig("wal.buffer_size", false, 64 * MB, 4096 * MB, &config.wal.buffer_size.value, 256 * MB, nullptr,
+                         nullptr),
+        CreateStringConfig("wal.path", false, &config.wal.path.value, "/var/lib/milvus/wal", nullptr, nullptr),
+
+        /* cache */
+        CreateSizeConfig("cache.cache_size", true, 0, std::numeric_limits<int64_t>::max(),
+                         &config.cache.cache_size.value, 4 * GB, nullptr, nullptr),
+        CreateFloatingConfig("cache.cpu_cache_threshold", false, 0.0, 1.0, &config.cache.cpu_cache_threshold.value, 0.7,
+                             nullptr, nullptr),
+        CreateSizeConfig("cache.insert_buffer_size", false, 0, std::numeric_limits<int64_t>::max(),
+                         &config.cache.insert_buffer_size.value, 1 * GB, nullptr, nullptr),
+        CreateBoolConfig("cache.cache_insert_data", false, &config.cache.cache_insert_data.value, false, nullptr,
+                         nullptr),
+        CreateStringConfig("cache.preload_collection", false, &config.cache.preload_collection.value, "", nullptr,
+                           nullptr),
+
+        /* gpu */
+        CreateBoolConfig("gpu.enable", false, &config.gpu.enable.value, false, nullptr, nullptr),
+        CreateSizeConfig("gpu.cache_size", true, 0, std::numeric_limits<int64_t>::max(), &config.gpu.cache_size.value,
+                         1 * GB, nullptr, nullptr),
+        CreateFloatingConfig("gpu.cache_threshold", false, 0.0, 1.0, &config.gpu.cache_threshold.value, 0.7, nullptr,
+                             nullptr),
+        CreateIntegerConfig("gpu.gpu_search_threshold", true, 0, std::numeric_limits<int64_t>::max(),
+                            &config.gpu.gpu_search_threshold.value, 1000, nullptr, nullptr),
+        CreateStringConfig("gpu.search_devices", false, &config.gpu.search_devices.value, "gpu0", nullptr, nullptr),
+        CreateStringConfig("gpu.build_index_devices", false, &config.gpu.build_index_devices.value, "gpu0", nullptr,
+                           nullptr),
+
+        /* log */
+        CreateStringConfig("logs.level", false, &config.logs.level.value, "debug", nullptr, nullptr),
+        CreateBoolConfig("logs.trace.enable", false, &config.logs.trace.enable.value, true, nullptr, nullptr),
+        CreateStringConfig("logs.path", false, &config.logs.path.value, "/var/lib/milvus/logs", nullptr, nullptr),
+        CreateSizeConfig("logs.max_log_file_size", false, 512 * MB, 4096 * MB, &config.logs.max_log_file_size.value,
+                         1024 * MB, nullptr, nullptr),
+        CreateIntegerConfig("logs.log_rotate_num", false, 0, 1024, &config.logs.log_rotate_num.value, 0, nullptr,
+                            nullptr),
+
+        /* metric */
+        CreateBoolConfig("metric.enable", false, &config.metric.enable.value, false, nullptr, nullptr),
+        CreateStringConfig("metric.address", false, &config.metric.address.value, "127.0.0.1", nullptr, nullptr),
+        CreateIntegerConfig("metric.port", false, 1024, 65535, &config.metric.port.value, 9091, nullptr, nullptr),
+
+        /* tracing */
+        CreateStringConfig("tracing.json_config_path", false, &config.tracing.json_config_path.value, "", nullptr,
+                           nullptr),
+
+        /* invisible */
+        /* engine */
+        CreateIntegerConfig("engine.search_combine_nq", true, 0, std::numeric_limits<int64_t>::max(),
+                            &config.engine.search_combine_nq.value, 64, nullptr, nullptr),
+        CreateIntegerConfig("engine.use_blas_threshold", true, 0, std::numeric_limits<int64_t>::max(),
+                            &config.engine.use_blas_threshold.value, 1100, nullptr, nullptr),
+        CreateIntegerConfig("engine.omp_thread_num", true, 0, std::numeric_limits<int64_t>::max(),
+                            &config.engine.omp_thread_num.value, 0, nullptr, nullptr),
+        CreateEnumConfig("engine.simd_type", false, &SimdMap, &config.engine.simd_type.value, SimdType::AUTO, nullptr,
+                         nullptr),
+
+        /* db */
+        CreateFloatingConfig("db.archive_disk_threshold", false, 0.0, 1.0, &config.db.archive_disk_threshold.value, 0.0,
+                             nullptr, nullptr),
+        CreateIntegerConfig("db.archive_days_threshold", false, 0, std::numeric_limits<int64_t>::max(),
+                            &config.db.archive_days_threshold.value, 0, nullptr, nullptr),
+    };
+}
+
+void
+ConfigMgr::Init() {
+    std::lock_guard<std::mutex> lock(GetConfigMutex());
+    for (auto& config : config_list_) config->Init();
+}
+
+void
+ConfigMgr::Load(const std::string& path) {
+    /* load from milvus.yaml */
+    auto yaml = YAML::LoadFile(path);
+
+    /* make it flattened */
+    std::unordered_map<std::string, std::string> flattened;
+    Flatten(yaml, flattened, "");
+
+    /* update config */
+    for (auto& it : flattened) Set(it.first, it.second, false);
+}
+
+void
+ConfigMgr::Set(const std::string& name, const std::string& value, bool update) {
+    for (auto& config : config_list_) {
+        if (std::strcmp(name.c_str(), config->name_) == 0) {
+            std::lock_guard<std::mutex> lock(GetConfigMutex());
+            if (not update || config->modifiable_) {
+                ThrowIfNotSuccess(config->Set(value, update));
+                Notify(name);
+                return;
+            }
+        }
+    }
+    throw "Config " + name + " not found.";
+}
+
+std::string
+ConfigMgr::Get(const std::string& name) const {
+    for (auto& config : config_list_)
+        if (std::strcmp(name.c_str(), config->name_) == 0) {
+            std::lock_guard<std::mutex> lock(GetConfigMutex());
+            return config->Get();
+        }
+    throw "Config " + name + " not found.";
+}
+
+std::string
+ConfigMgr::Dump() const {
+    std::stringstream ss;
+    for (auto& config : config_list_) ss << config->name_ << ": " << config->Get() << std::endl;
+    return ss.str();
+}
+
+void
+ConfigMgr::Attach(const std::string& name, ConfigObserver* observer) {
+    std::lock_guard<std::mutex> lock(observer_mutex_);
+    observers_[name].push_back(observer);
+}
+
+void
+ConfigMgr::Detach(const std::string& name, ConfigObserver* observer) {
+    std::lock_guard<std::mutex> lock(observer_mutex_);
+    if (observers_.find(name) == observers_.end())
+        return;
+    auto& ob_list = observers_[name];
+    ob_list.remove(observer);
+}
+
+void
+ConfigMgr::Notify(const std::string& name) {
+    std::lock_guard<std::mutex> lock(observer_mutex_);
+    if (observers_.find(name) == observers_.end())
+        return;
+    auto& ob_list = observers_[name];
+    for (auto& ob : ob_list) {
+        ob->ConfigUpdate(name);
+    }
+}
+
+}  // namespace milvus

core/src/config/ConfigMgr.h

@@ -11,31 +11,82 @@
 
 #pragma once
 
+#include <list>
+#include <memory>
+#include <mutex>
 #include <string>
+#include <unordered_map>
+#include <vector>
 
-#include "ConfigNode.h"
-#include "utils/Status.h"
+#include "config/ServerConfig.h"
 
 namespace milvus {
-namespace server {
 
-class ConfigMgr {
+class ConfigObserver {
  public:
-    virtual Status
-    LoadConfigFile(const std::string& filename) = 0;
+    virtual ~ConfigObserver() {
+    }
 
     virtual void
-    Print() const = 0;  // will be deleted
+    ConfigUpdate(const std::string& name) = 0;
+};
+using ConfigObserverPtr = std::shared_ptr<ConfigObserver>;
+
+class ConfigMgr {
+ public:
+    static ConfigMgr&
+    GetInstance() {
+        return instance;
+    }
+
+ private:
+    static ConfigMgr instance;
+
+ public:
+    ConfigMgr();
+
+    ConfigMgr(const ConfigMgr&) = delete;
+    ConfigMgr&
+    operator=(const ConfigMgr&) = delete;
+
+    ConfigMgr(ConfigMgr&&) = delete;
+    ConfigMgr&
+    operator=(ConfigMgr&&) = delete;
+
+ public:
+    void
+    Init();
+
+    void
+    Load(const std::string& path);
+
+    void
+    Set(const std::string& name, const std::string& value, bool update = true);
+
+    std::string
+    Get(const std::string& name) const;
+
+    std::string
+    Dump() const;
+
+ public:
+    // Shared pointer should not be used here
+    void
+    Attach(const std::string& name, ConfigObserver* observer);
+
+    void
+    Detach(const std::string& name, ConfigObserver* observer);
 
-    virtual std::string
-    DumpString() const = 0;
+ private:
+    void
+    Notify(const std::string& name);
 
-    virtual const ConfigNode&
-    GetRootNode() const = 0;
+ private:
+    std::vector<BaseConfigPtr> config_list_;
+    std::mutex mutex_;
 
-    virtual ConfigNode&
-    GetRootNode() = 0;
+    std::unordered_map<std::string, std::list<ConfigObserver*>> observers_;
+    std::mutex observer_mutex_;
 };
 
-}  // namespace server
 }  // namespace milvus

core/src/config/ConfigNode.cpp

-// Copyright (C) 2019-2020 Zilliz. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software distributed under the License
-// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
-// or implied. See the License for the specific language governing permissions and limitations under the License.
-
-#include "config/ConfigNode.h"
-#include "utils/Error.h"
-#include "utils/Log.h"
-
-#include <fiu-local.h>
-#include <algorithm>
-#include <sstream>
-#include <string>
-
-namespace milvus {
-namespace server {
-
-void
-ConfigNode::Combine(const ConfigNode& target) {
-    const std::map<std::string, std::string>& kv = target.GetConfig();
-    for (auto itr = kv.begin(); itr != kv.end(); ++itr) {
-        config_[itr->first] = itr->second;
-    }
-
-    const std::map<std::string, std::vector<std::string> >& sequences = target.GetSequences();
-    for (auto itr = sequences.begin(); itr != sequences.end(); ++itr) {
-        sequences_[itr->first] = itr->second;
-    }
-
-    const std::map<std::string, ConfigNode>& children = target.GetChildren();
-    for (auto itr = children.begin(); itr != children.end(); ++itr) {
-        children_[itr->first] = itr->second;
-    }
-}
-
-// key/value pair config
-void
-ConfigNode::SetValue(const std::string& key, const std::string& value) {
-    config_[key] = value;
-}
-
-std::string
-ConfigNode::GetValue(const std::string& param_key, const std::string& default_val) const {
-    auto ref = config_.find(param_key);
-    if (ref != config_.end()) {
-        return ref->second;
-    }
-
-    // THROW_UNEXPECTED_ERROR("Can't find parameter key: " + param_key);
-    return default_val;
-}
-
-bool
-ConfigNode::GetBoolValue(const std::string& param_key, bool default_val) const {
-    std::string val = GetValue(param_key);
-    if (!val.empty()) {
-        std::transform(val.begin(), val.end(), val.begin(), ::tolower);
-        return (val == "true" || val == "on" || val == "yes" || val == "1");
-    } else {
-        return default_val;
-    }
-}
-
-int32_t
-ConfigNode::GetInt32Value(const std::string& param_key, int32_t default_val) const {
-    std::string val = GetValue(param_key);
-    if (!val.empty()) {
-        return (int32_t)std::strtol(val.c_str(), nullptr, 10);
-    } else {
-        return default_val;
-    }
-}
-
-int64_t
-ConfigNode::GetInt64Value(const std::string& param_key, int64_t default_val) const {
-    std::string val = GetValue(param_key);
-    if (!val.empty()) {
-        return std::strtol(val.c_str(), nullptr, 10);
-    } else {
-        return default_val;
-    }
-}
-
-float
-ConfigNode::GetFloatValue(const std::string& param_key, float default_val) const {
-    std::string val = GetValue(param_key);
-    if (!val.empty()) {
-        return std::strtof(val.c_str(), nullptr);
-    } else {
-        return default_val;
-    }
-}
-
-double
-ConfigNode::GetDoubleValue(const std::string& param_key, double default_val) const {
-    std::string val = GetValue(param_key);
-    if (!val.empty()) {
-        return std::strtod(val.c_str(), nullptr);
-    } else {
-        return default_val;
-    }
-}
-
-const std::map<std::string, std::string>&
-ConfigNode::GetConfig() const {
-    return config_;
-}
-
-void
-ConfigNode::ClearConfig() {
-    config_.clear();
-}
-
-// key/object config
-void
-ConfigNode::AddChild(const std::string& type_name, const ConfigNode& config) {
-    children_[type_name] = config;
-}
-
-ConfigNode
-ConfigNode::GetChild(const std::string& type_name) const {
-    auto ref = children_.find(type_name);
-    if (ref != children_.end()) {
-        return ref->second;
-    }
-
-    ConfigNode nc;
-    return nc;
-}
-
-ConfigNode&
-ConfigNode::GetChild(const std::string& type_name) {
-    return children_[type_name];
-}
-
-void
-ConfigNode::GetChildren(ConfigNodeArr& arr) const {
-    arr.clear();
-    arr.reserve(children_.size());
-    transform(children_.begin(), children_.end(), back_inserter(arr), [](auto& ref) { return ref.second; });
-}
-
-const std::map<std::string, ConfigNode>&
-ConfigNode::GetChildren() const {
-    return children_;
-}
-
-void
-ConfigNode::ClearChildren() {
-    children_.clear();
-}
-
-// key/sequence config
-void
-ConfigNode::AddSequenceItem(const std::string& key, const std::string& item) {
-    sequences_[key].push_back(item);
-}
-
-std::vector<std::string>
-ConfigNode::GetSequence(const std::string& key) const {
-    auto itr = sequences_.find(key);
-    if (itr != sequences_.end()) {
-        return itr->second;
-    } else {
-        std::vector<std::string> temp;
-        return temp;
-    }
-}
-
-const std::map<std::string, std::vector<std::string> >&
-ConfigNode::GetSequences() const {
-    return sequences_;
-}
-
-void
-ConfigNode::ClearSequences() {
-    sequences_.clear();
-}
-
-void
-ConfigNode::PrintAll(const std::string& prefix) const {
-    for (auto& elem : config_) {
-        LOG_SERVER_INFO_ << prefix << elem.first + ": " << elem.second;
-    }
-
-    for (auto& elem : sequences_) {
-        LOG_SERVER_INFO_ << prefix << elem.first << ": ";
-        for (auto& str : elem.second) {
-            LOG_SERVER_INFO_ << prefix << "    - " << str;
-        }
-    }
-
-    for (auto& elem : children_) {
-        LOG_SERVER_INFO_ << prefix << elem.first << ": ";
-        elem.second.PrintAll(prefix + "    ");
-    }
-}
-
-std::string
-ConfigNode::DumpString(const std::string& prefix) const {
-    std::stringstream str_buffer;
-    const std::string endl = "\n";
-    for (auto& elem : config_) {
-        str_buffer << prefix << elem.first << ": " << elem.second << endl;
-    }
-
-    for (auto& elem : sequences_) {
-        str_buffer << prefix << elem.first << ": " << endl;
-        for (auto& str : elem.second) {
-            str_buffer << prefix + "    - " << str << endl;
-        }
-    }
-
-    for (auto& elem : children_) {
-        str_buffer << prefix << elem.first << ": " << endl;
-        str_buffer << elem.second.DumpString(prefix + "    ") << endl;
-    }
-
-    return str_buffer.str();
-}
-
-}  // namespace server
-}  // namespace milvus

core/src/config/ConfigNode.h

-// Copyright (C) 2019-2020 Zilliz. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software distributed under the License
-// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
-// or implied. See the License for the specific language governing permissions and limitations under the License.
-
-#pragma once
-
-#include <map>
-#include <string>
-#include <vector>
-
-namespace milvus {
-namespace server {
-
-class ConfigNode;
-typedef std::vector<ConfigNode> ConfigNodeArr;
-
-class ConfigNode {
- public:
-    void
-    Combine(const ConfigNode& target);
-
-    // key/value pair config
-    void
-    SetValue(const std::string& key, const std::string& value);
-
-    std::string
-    GetValue(const std::string& param_key, const std::string& default_val = "") const;
-    bool
-    GetBoolValue(const std::string& param_key, bool default_val = false) const;
-    int32_t
-    GetInt32Value(const std::string& param_key, int32_t default_val = 0) const;
-    int64_t
-    GetInt64Value(const std::string& param_key, int64_t default_val = 0) const;
-    float
-    GetFloatValue(const std::string& param_key, float default_val = 0.0) const;
-    double
-    GetDoubleValue(const std::string& param_key, double default_val = 0.0) const;
-
-    const std::map<std::string, std::string>&
-    GetConfig() const;
-    void
-    ClearConfig();
-
-    // key/object config
-    void
-    AddChild(const std::string& type_name, const ConfigNode& config);
-    ConfigNode
-    GetChild(const std::string& type_name) const;
-    ConfigNode&
-    GetChild(const std::string& type_name);
-    void
-    GetChildren(ConfigNodeArr& arr) const;
-
-    const std::map<std::string, ConfigNode>&
-    GetChildren() const;
-    void
-    ClearChildren();
-
-    // key/sequence config
-    void
-    AddSequenceItem(const std::string& key, const std::string& item);
-    std::vector<std::string>
-    GetSequence(const std::string& key) const;
-
-    const std::map<std::string, std::vector<std::string> >&
-    GetSequences() const;
-    void
-    ClearSequences();
-
-    void
-    PrintAll(const std::string& prefix = "") const;
-    std::string
-    DumpString(const std::string& prefix = "") const;
-
- private:
-    std::map<std::string, std::string> config_;
-    std::map<std::string, ConfigNode> children_;
-    std::map<std::string, std::vector<std::string> > sequences_;
-};
-
-}  // namespace server
-}  // namespace milvus

core/src/config/ConfigType.cpp

+// Copyright (C) 2019-2020 Zilliz. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software distributed under the License
+// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+// or implied. See the License for the specific language governing permissions and limitations under the License.
+
+#include "config/ConfigType.h"
+#include "config/ServerConfig.h"
+
+#include <strings.h>
+#include <algorithm>
+#include <cassert>
+#include <functional>
+#include <sstream>
+#include <string>
+
+namespace {
+std::unordered_map<std::string, int64_t> BYTE_UNITS = {
+    {"b", 1},
+    {"k", 1024},
+    {"m", 1024 * 1024},
+    {"g", 1024 * 1024 * 1024},
+};
+
+bool
+is_integer(const std::string& s) {
+    if (not s.empty() && (std::isdigit(s[0]) || s[0] == '-')) {
+        auto ss = s.substr(1);
+        return std::find_if(ss.begin(), ss.end(), [](unsigned char c) { return !std::isdigit(c); }) == ss.end();
+    }
+    return false;
+}
+
+bool
+is_number(const std::string& s) {
+    return !s.empty() && std::find_if(s.begin(), s.end(), [](unsigned char c) { return !std::isdigit(c); }) == s.end();
+}
+
+bool
+is_alpha(const std::string& s) {
+    return !s.empty() && std::find_if(s.begin(), s.end(), [](unsigned char c) { return !std::isalpha(c); }) == s.end();
+}
+
+template <typename T>
+bool
+boundary_check(T val, T lower_bound, T upper_bound) {
+    return lower_bound <= val && val <= upper_bound;
+}
+
+bool
+parse_bool(const std::string& str, std::string& err) {
+    if (!strcasecmp(str.c_str(), "true"))
+        return true;
+    else if (!strcasecmp(str.c_str(), "false"))
+        return false;
+    else
+        err = "The specified value must be true or false";
+    return false;
+}
+
+std::string
+str_tolower(std::string s) {
+    std::transform(s.begin(), s.end(), s.begin(), [](unsigned char c) { return std::tolower(c); });
+    return s;
+}
+
+int64_t
+parse_bytes(const std::string& str, std::string& err) {
+    try {
+        if (str.find_first_of('-') != std::string::npos) {
+            std::stringstream ss;
+            ss << "The specified value for memory (" << str << ") should be a positive integer.";
+            err = ss.str();
+            return 0;
+        }
+
+        std::string s = str;
+        if (is_number(s))
+            return std::stoll(s);
+        if (s.length() == 0)
+            return 0;
+
+        auto last_two = s.substr(s.length() - 2, 2);
+        auto last_one = s.substr(s.length() - 1);
+        if (is_alpha(last_two) && is_alpha(last_one))
+            if (last_one == "b" or last_one == "B")
+                s = s.substr(0, s.length() - 1);
+        auto& units = BYTE_UNITS;
+        auto suffix = str_tolower(s.substr(s.length() - 1));
+
+        std::string digits_part;
+        if (is_number(suffix)) {
+            digits_part = s;
+            suffix = 'b';
+        } else {
+            digits_part = s.substr(0, s.length() - 1);
+        }
+
+        if (is_number(digits_part) && (units.find(suffix) != units.end() || is_number(suffix))) {
+            auto digits = std::stoll(digits_part);
+            return digits * units[suffix];
+        } else {
+            std::stringstream ss;
+            ss << "The specified value for memory (" << str << ") should specify the units."
+               << "The postfix should be one of the `b` `k` `m` `g` characters";
+            err = ss.str();
+        }
+    } catch (...) {
+        err = "Unknown error happened on parse bytes.";
+    }
+    return 0;
+}
+
+}  // namespace
+
+// Use (void) to silent unused warnings.
+#define assertm(exp, msg) assert(((void)msg, exp))
+
+namespace milvus {
+
+std::vector<std::string>
+OptionValue(const configEnum& ce) {
+    std::vector<std::string> ret;
+    for (auto& e : ce) {
+        ret.emplace_back(e.first);
+    }
+    return ret;
+}
+
+BaseConfig::BaseConfig(const char* name, const char* alias, bool modifiable)
+    : name_(name), alias_(alias), modifiable_(modifiable) {
+}
+
+void
+BaseConfig::Init() {
+    assertm(not inited_, "already initialized");
+    inited_ = true;
+}
+
+BoolConfig::BoolConfig(const char* name, const char* alias, bool modifiable, bool* config, bool default_value,
+                       std::function<bool(bool val, std::string& err)> is_valid_fn,
+                       std::function<bool(bool val, bool prev, std::string& err)> update_fn)
+    : BaseConfig(name, alias, modifiable),
+      config_(config),
+      default_value_(default_value),
+      is_valid_fn_(std::move(is_valid_fn)),
+      update_fn_(std::move(update_fn)) {
+}
+
+void
+BoolConfig::Init() {
+    BaseConfig::Init();
+    assert(config_ != nullptr);
+    *config_ = default_value_;
+}
+
+ConfigStatus
+BoolConfig::Set(const std::string& val, bool update) {
+    assertm(inited_, "uninitialized");
+    try {
+        if (update and not modifiable_) {
+            std::stringstream ss;
+            ss << "Config " << name_ << " is immutable.";
+            return ConfigStatus(SetReturn::IMMUTABLE, ss.str());
+        }
+
+        std::string err;
+        bool value = parse_bool(val, err);
+        if (not err.empty())
+            return ConfigStatus(SetReturn::INVALID, err);
+
+        if (is_valid_fn_ && not is_valid_fn_(value, err))
+            return ConfigStatus(SetReturn::INVALID, err);
+
+        bool prev = *config_;
+        *config_ = value;
+        if (update && update_fn_ && not update_fn_(value, prev, err)) {
+            *config_ = prev;
+            return ConfigStatus(SetReturn::UPDATE_FAILURE, err);
+        }
+
+        return ConfigStatus(SetReturn::SUCCESS, "");
+    } catch (std::exception& e) {
+        return ConfigStatus(SetReturn::EXCEPTION, e.what());
+    } catch (...) {
+        return ConfigStatus(SetReturn::UNEXPECTED, "unexpected");
+    }
+}
+
+std::string
+BoolConfig::Get() {
+    assertm(inited_, "uninitialized");
+    return *config_ ? "true" : "false";
+}
+
+StringConfig::StringConfig(
+    const char* name, const char* alias, bool modifiable, std::string* config, const char* default_value,
+    std::function<bool(const std::string& val, std::string& err)> is_valid_fn,
+    std::function<bool(const std::string& val, const std::string& prev, std::string& err)> update_fn)
+    : BaseConfig(name, alias, modifiable),
+      config_(config),
+      default_value_(default_value),
+      is_valid_fn_(std::move(is_valid_fn)),
+      update_fn_(std::move(update_fn)) {
+}
+
+void
+StringConfig::Init() {
+    BaseConfig::Init();
+    assert(config_ != nullptr);
+    *config_ = default_value_;
+}
+
+ConfigStatus
+StringConfig::Set(const std::string& val, bool update) {
+    assertm(inited_, "uninitialized");
+    try {
+        if (update and not modifiable_) {
+            std::stringstream ss;
+            ss << "Config " << name_ << " is immutable.";
+            return ConfigStatus(SetReturn::IMMUTABLE, ss.str());
+        }
+
+        std::string err;
+        if (is_valid_fn_ && not is_valid_fn_(val, err))
+            return ConfigStatus(SetReturn::INVALID, err);
+
+        std::string prev = *config_;
+        *config_ = val;
+        if (update && update_fn_ && not update_fn_(val, prev, err)) {
+            *config_ = prev;
+            return ConfigStatus(SetReturn::UPDATE_FAILURE, err);
+        }
+
+        return ConfigStatus(SetReturn::SUCCESS, "");
+    } catch (std::exception& e) {
+        return ConfigStatus(SetReturn::EXCEPTION, e.what());
+    } catch (...) {
+        return ConfigStatus(SetReturn::UNEXPECTED, "unexpected");
+    }
+}
+
+std::string
+StringConfig::Get() {
+    assertm(inited_, "uninitialized");
+    return *config_;
+}
+
+EnumConfig::EnumConfig(const char* name, const char* alias, bool modifiable, configEnum* enumd, int64_t* config,
+                       int64_t default_value, std::function<bool(int64_t val, std::string& err)> is_valid_fn,
+                       std::function<bool(int64_t val, int64_t prev, std::string& err)> update_fn)
+    : BaseConfig(name, alias, modifiable),
+      config_(config),
+      enum_value_(enumd),
+      default_value_(default_value),
+      is_valid_fn_(std::move(is_valid_fn)),
+      update_fn_(std::move(update_fn)) {
+}
+
+void
+EnumConfig::Init() {
+    BaseConfig::Init();
+    assert(enum_value_ != nullptr);
+    assertm(not enum_value_->empty(), "enum value empty");
+    assert(config_ != nullptr);
+    *config_ = default_value_;
+}
+
+ConfigStatus
+EnumConfig::Set(const std::string& val, bool update) {
+    assertm(inited_, "uninitialized");
+    try {
+        if (update and not modifiable_) {
+            std::stringstream ss;
+            ss << "Config " << name_ << " is immutable.";
+            return ConfigStatus(SetReturn::IMMUTABLE, ss.str());
+        }
+
+        if (enum_value_->find(val) == enum_value_->end()) {
+            auto option_values = OptionValue(*enum_value_);
+            std::stringstream ss;
+            ss << "Config " << name_ << "(" << val << ") must be one of following: ";
+            for (size_t i = 0; i < option_values.size() - 1; ++i) {
+                ss << option_values[i] << ", ";
+            }
+            ss << option_values.back() << ".";
+            return ConfigStatus(SetReturn::ENUM_VALUE_NOTFOUND, ss.str());
+        }
+
+        int64_t value = enum_value_->at(val);
+        std::string err;
+        if (is_valid_fn_ && not is_valid_fn_(value, err)) {
+            return ConfigStatus(SetReturn::INVALID, err);
+        }
+
+        int64_t prev = *config_;
+        *config_ = value;
+        if (update && update_fn_ && not update_fn_(value, prev, err)) {
+            *config_ = prev;
+            return ConfigStatus(SetReturn::UPDATE_FAILURE, err);
+        }
+
+        return ConfigStatus(SetReturn::SUCCESS, "");
+    } catch (std::exception& e) {
+        return ConfigStatus(SetReturn::EXCEPTION, e.what());
+    } catch (...) {
+        return ConfigStatus(SetReturn::UNEXPECTED, "unexpected");
+    }
+}
+
+std::string
+EnumConfig::Get() {
+    assertm(inited_, "uninitialized");
+    for (auto& it : *enum_value_) {
+        if (*config_ == it.second) {
+            return it.first;
+        }
+    }
+    return "unknown";
+}
+
+IntegerConfig::IntegerConfig(const char* name, const char* alias, bool modifiable, int64_t lower_bound,
+                             int64_t upper_bound, int64_t* config, int64_t default_value,
+                             std::function<bool(int64_t val, std::string& err)> is_valid_fn,
+                             std::function<bool(int64_t val, int64_t prev, std::string& err)> update_fn)
+    : BaseConfig(name, alias, modifiable),
+      config_(config),
+      lower_bound_(lower_bound),
+      upper_bound_(upper_bound),
+      default_value_(default_value),
+      is_valid_fn_(std::move(is_valid_fn)),
+      update_fn_(std::move(update_fn)) {
+}
+
+void
+IntegerConfig::Init() {
+    BaseConfig::Init();
+    assert(config_ != nullptr);
+    *config_ = default_value_;
+}
+
+ConfigStatus
+IntegerConfig::Set(const std::string& val, bool update) {
+    assertm(inited_, "uninitialized");
+    try {
+        if (update and not modifiable_) {
+            std::stringstream ss;
+            ss << "Config " << name_ << " is immutable.";
+            return ConfigStatus(SetReturn::IMMUTABLE, ss.str());
+        }
+
+        if (not is_integer(val)) {
+            std::stringstream ss;
+            ss << "Config " << name_ << "(" << val << ") must be a integer.";
+            return ConfigStatus(SetReturn::INVALID, ss.str());
+        }
+
+        int64_t value = std::stoll(val);
+        if (not boundary_check<int64_t>(value, lower_bound_, upper_bound_)) {
+            std::stringstream ss;
+            ss << "Config " << name_ << "(" << val << ") must in range [" << lower_bound_ << ", " << upper_bound_
+               << "].";
+            return ConfigStatus(SetReturn::OUT_OF_RANGE, ss.str());
+        }
+
+        std::string err;
+        if (is_valid_fn_ && not is_valid_fn_(value, err))
+            return ConfigStatus(SetReturn::INVALID, err);
+
+        int64_t prev = *config_;
+        *config_ = value;
+        if (update && update_fn_ && not update_fn_(value, prev, err)) {
+            *config_ = prev;
+            return ConfigStatus(SetReturn::UPDATE_FAILURE, err);
+        }
+
+        return ConfigStatus(SetReturn::SUCCESS, "");
+    } catch (std::exception& e) {
+        return ConfigStatus(SetReturn::EXCEPTION, e.what());
+    } catch (...) {
+        return ConfigStatus(SetReturn::UNEXPECTED, "unexpected");
+    }
+}
+
+std::string
+IntegerConfig::Get() {
+    assertm(inited_, "uninitialized");
+    return std::to_string(*config_);
+}
+
+FloatingConfig::FloatingConfig(const char* name, const char* alias, bool modifiable, double lower_bound,
+                               double upper_bound, double* config, double default_value,
+                               std::function<bool(double val, std::string& err)> is_valid_fn,
+                               std::function<bool(double val, double prev, std::string& err)> update_fn)
+    : BaseConfig(name, alias, modifiable),
+      config_(config),
+      lower_bound_(lower_bound),
+      upper_bound_(upper_bound),
+      default_value_(default_value),
+      is_valid_fn_(std::move(is_valid_fn)),
+      update_fn_(std::move(update_fn)) {
+}
+
+void
+FloatingConfig::Init() {
+    BaseConfig::Init();
+    assert(config_ != nullptr);
+    *config_ = default_value_;
+}
+
+ConfigStatus
+FloatingConfig::Set(const std::string& val, bool update) {
+    assertm(inited_, "uninitialized");
+    try {
+        if (update and not modifiable_) {
+            std::stringstream ss;
+            ss << "Config " << name_ << " is immutable.";
+            return ConfigStatus(SetReturn::IMMUTABLE, ss.str());
+        }
+
+        double value = std::stod(val);
+        if (not boundary_check<double>(value, lower_bound_, upper_bound_)) {
+            std::stringstream ss;
+            ss << "Config " << name_ << "(" << val << ") must in range [" << lower_bound_ << ", " << upper_bound_
+               << "].";
+            return ConfigStatus(SetReturn::OUT_OF_RANGE, ss.str());
+        }
+
+        std::string err;
+        if (is_valid_fn_ && not is_valid_fn_(value, err))
+            return ConfigStatus(SetReturn::INVALID, err);
+
+        double prev = *config_;
+        *config_ = value;
+        if (update && update_fn_ && not update_fn_(value, prev, err)) {
+            *config_ = prev;
+
+            return ConfigStatus(SetReturn::UPDATE_FAILURE, err);
+        }
+
+        return ConfigStatus(SetReturn::SUCCESS, "");
+    } catch (std::exception& e) {
+        return ConfigStatus(SetReturn::EXCEPTION, e.what());
+    } catch (...) {
+        return ConfigStatus(SetReturn::UNEXPECTED, "unexpected");
+    }
+}
+
+std::string
+FloatingConfig::Get() {
+    assertm(inited_, "uninitialized");
+    return std::to_string(*config_);
+}
+
+SizeConfig::SizeConfig(const char* name, const char* alias, bool modifiable, int64_t lower_bound, int64_t upper_bound,
+                       int64_t* config, int64_t default_value,
+                       std::function<bool(int64_t val, std::string& err)> is_valid_fn,
+                       std::function<bool(int64_t val, int64_t prev, std::string& err)> update_fn)
+    : BaseConfig(name, alias, modifiable),
+      config_(config),
+      lower_bound_(lower_bound),
+      upper_bound_(upper_bound),
+      default_value_(default_value),
+      is_valid_fn_(std::move(is_valid_fn)),
+      update_fn_(std::move(update_fn)) {
+}
+
+void
+SizeConfig::Init() {
+    BaseConfig::Init();
+    assert(config_ != nullptr);
+    *config_ = default_value_;
+}
+
+ConfigStatus
+SizeConfig::Set(const std::string& val, bool update) {
+    assertm(inited_, "uninitialized");
+    try {
+        if (update and not modifiable_) {
+            std::stringstream ss;
+            ss << "Config " << name_ << " is immutable.";
+            return ConfigStatus(SetReturn::IMMUTABLE, ss.str());
+        }
+
+        std::string err;
+        int64_t value = parse_bytes(val, err);
+        if (not err.empty()) {
+            return ConfigStatus(SetReturn::INVALID, err);
+        }
+
+        if (not boundary_check<int64_t>(value, lower_bound_, upper_bound_)) {
+            std::stringstream ss;
+            ss << "Config " << name_ << "(" << val << ") must in range [" << lower_bound_ << " Byte, " << upper_bound_
+               << " Byte].";
+            return ConfigStatus(SetReturn::OUT_OF_RANGE, ss.str());
+        }
+
+        if (is_valid_fn_ && not is_valid_fn_(value, err)) {
+            return ConfigStatus(SetReturn::INVALID, err);
+        }
+
+        int64_t prev = *config_;
+        *config_ = value;
+        if (update && update_fn_ && not update_fn_(value, prev, err)) {
+            *config_ = prev;
+            return ConfigStatus(SetReturn::UPDATE_FAILURE, err);
+        }
+
+        return ConfigStatus(SetReturn::SUCCESS, "");
+    } catch (std::exception& e) {
+        return ConfigStatus(SetReturn::EXCEPTION, e.what());
+    } catch (...) {
+        return ConfigStatus(SetReturn::UNEXPECTED, "unexpected");
+    }
+}
+
+std::string
+SizeConfig::Get() {
+    assertm(inited_, "uninitialized");
+    return std::to_string(*config_);
+}
+
+}  // namespace milvus

core/src/config/ConfigType.h

+// Copyright (C) 2019-2020 Zilliz. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software distributed under the License
+// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+// or implied. See the License for the specific language governing permissions and limitations under the License.
+
+#pragma once
+
+#include <functional>
+#include <memory>
+#include <string>
+#include <unordered_map>
+#include <utility>
+#include <vector>
+
+namespace milvus {
+
+using configEnum = const std::unordered_map<std::string, int64_t>;
+std::vector<std::string>
+OptionValue(const configEnum& ce);
+
+enum SetReturn {
+    SUCCESS = 1,
+    IMMUTABLE,
+    ENUM_VALUE_NOTFOUND,
+    INVALID,
+    OUT_OF_RANGE,
+    UPDATE_FAILURE,
+    EXCEPTION,
+    UNEXPECTED,
+};
+
+struct ConfigStatus {
+    ConfigStatus(SetReturn sr, std::string msg) : set_return(sr), message(std::move(msg)) {
+    }
+    SetReturn set_return;
+    std::string message;
+};
+
+class BaseConfig {
+ public:
+    BaseConfig(const char* name, const char* alias, bool modifiable);
+    virtual ~BaseConfig() = default;
+
+ public:
+    bool inited_ = false;
+    const char* name_;
+    const char* alias_;
+    const bool modifiable_;
+
+ public:
+    virtual void
+    Init();
+
+    virtual ConfigStatus
+    Set(const std::string& value, bool update) = 0;
+
+    virtual std::string
+    Get() = 0;
+};
+using BaseConfigPtr = std::shared_ptr<BaseConfig>;
+
+class BoolConfig : public BaseConfig {
+ public:
+    BoolConfig(const char* name, const char* alias, bool modifiable, bool* config, bool default_value,
+               std::function<bool(bool val, std::string& err)> is_valid_fn,
+               std::function<bool(bool val, bool prev, std::string& err)> update_fn);
+
+ private:
+    bool* config_;
+    const bool default_value_;
+    std::function<bool(bool val, std::string& err)> is_valid_fn_;
+    std::function<bool(bool val, bool prev, std::string& err)> update_fn_;
+
+ public:
+    void
+    Init() override;
+
+    ConfigStatus
+    Set(const std::string& value, bool update) override;
+
+    std::string
+    Get() override;
+};
+
+class StringConfig : public BaseConfig {
+ public:
+    StringConfig(const char* name, const char* alias, bool modifiable, std::string* config, const char* default_value,
+                 std::function<bool(const std::string& val, std::string& err)> is_valid_fn,
+                 std::function<bool(const std::string& val, const std::string& prev, std::string& err)> update_fn);
+
+ private:
+    std::string* config_;
+    const char* default_value_;
+    std::function<bool(const std::string& val, std::string& err)> is_valid_fn_;
+    std::function<bool(const std::string& val, const std::string& prev, std::string& err)> update_fn_;
+
+ public:
+    void
+    Init() override;
+
+    ConfigStatus
+    Set(const std::string& value, bool update) override;
+
+    std::string
+    Get() override;
+};
+
+class EnumConfig : public BaseConfig {
+ public:
+    EnumConfig(const char* name, const char* alias, bool modifiable, configEnum* enumd, int64_t* config,
+               int64_t default_value, std::function<bool(int64_t val, std::string& err)> is_valid_fn,
+               std::function<bool(int64_t val, int64_t prev, std::string& err)> update_fn);
+
+ private:
+    int64_t* config_;
+    configEnum* enum_value_;
+    const int64_t default_value_;
+    std::function<bool(int64_t val, std::string& err)> is_valid_fn_;
+    std::function<bool(int64_t val, int64_t prev, std::string& err)> update_fn_;
+
+ public:
+    void
+    Init() override;
+
+    ConfigStatus
+    Set(const std::string& value, bool update) override;
+
+    std::string
+    Get() override;
+};
+
+class IntegerConfig : public BaseConfig {
+ public:
+    IntegerConfig(const char* name, const char* alias, bool modifiable, int64_t lower_bound, int64_t upper_bound,
+                  int64_t* config, int64_t default_value,
+                  std::function<bool(int64_t val, std::string& err)> is_valid_fn,
+                  std::function<bool(int64_t val, int64_t prev, std::string& err)> update_fn);
+
+ private:
+    int64_t* config_;
+    int64_t lower_bound_;
+    int64_t upper_bound_;
+    const int64_t default_value_;
+    std::function<bool(int64_t val, std::string& err)> is_valid_fn_;
+    std::function<bool(int64_t val, int64_t prev, std::string& err)> update_fn_;
+
+ public:
+    void
+    Init() override;
+
+    ConfigStatus
+    Set(const std::string& value, bool update) override;
+
+    std::string
+    Get() override;
+};
+
+class FloatingConfig : public BaseConfig {
+ public:
+    FloatingConfig(const char* name, const char* alias, bool modifiable, double lower_bound, double upper_bound,
+                   double* config, double default_value, std::function<bool(double val, std::string& err)> is_valid_fn,
+                   std::function<bool(double val, double prev, std::string& err)> update_fn);
+
+ private:
+    double* config_;
+    double lower_bound_;
+    double upper_bound_;
+    const double default_value_;
+    std::function<bool(double val, std::string& err)> is_valid_fn_;
+    std::function<bool(double val, double prev, std::string& err)> update_fn_;
+
+ public:
+    void
+    Init() override;
+
+    ConfigStatus
+    Set(const std::string& value, bool update) override;
+
+    std::string
+    Get() override;
+};
+
+class SizeConfig : public BaseConfig {
+ public:
+    SizeConfig(const char* name, const char* alias, bool modifiable, int64_t lower_bound, int64_t upper_bound,
+               int64_t* config, int64_t default_value, std::function<bool(int64_t val, std::string& err)> is_valid_fn,
+               std::function<bool(int64_t val, int64_t prev, std::string& err)> update_fn);
+
+ private:
+    int64_t* config_;
+    int64_t lower_bound_;
+    int64_t upper_bound_;
+    const int64_t default_value_;
+    std::function<bool(int64_t val, std::string& err)> is_valid_fn_;
+    std::function<bool(int64_t val, int64_t prev, std::string& err)> update_fn_;
+
+ public:
+    void
+    Init() override;
+
+    ConfigStatus
+    Set(const std::string& value, bool update) override;
+
+    std::string
+    Get() override;
+};
+
+#define CreateBoolConfig(name, modifiable, config_addr, default, is_valid, update) \
+    std::make_shared<BoolConfig>(name, nullptr, modifiable, config_addr, (default), is_valid, update)
+
+#define CreateStringConfig(name, modifiable, config_addr, default, is_valid, update) \
+    std::make_shared<StringConfig>(name, nullptr, modifiable, config_addr, (default), is_valid, update)
+
+#define CreateEnumConfig(name, modifiable, enumd, config_addr, default, is_valid, update) \
+    std::make_shared<EnumConfig>(name, nullptr, modifiable, enumd, config_addr, (default), is_valid, update)
+
+#define CreateIntegerConfig(name, modifiable, lower_bound, upper_bound, config_addr, default, is_valid, update)  \
+    std::make_shared<IntegerConfig>(name, nullptr, modifiable, lower_bound, upper_bound, config_addr, (default), \
+                                    is_valid, update)
+
+#define CreateFloatingConfig(name, modifiable, lower_bound, upper_bound, config_addr, default, is_valid, update)  \
+    std::make_shared<FloatingConfig>(name, nullptr, modifiable, lower_bound, upper_bound, config_addr, (default), \
+                                     is_valid, update)
+
+#define CreateSizeConfig(name, modifiable, lower_bound, upper_bound, config_addr, default, is_valid, update)  \
+    std::make_shared<SizeConfig>(name, nullptr, modifiable, lower_bound, upper_bound, config_addr, (default), \
+                                 is_valid, update)
+
+}  // namespace milvus

core/src/config/handler/ConfigHandler.h → core/src/config/ServerConfig.cpp

@@ -9,31 +9,49 @@
 // is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
 // or implied. See the License for the specific language governing permissions and limitations under the License.
 
-#pragma once
-
+#include <sstream>
 #include <string>
+#include <unordered_set>
+#include <vector>
 
-#include "config/Config.h"
-#include "utils/Log.h"
+#include "config/ServerConfig.h"
 
 namespace milvus {
-namespace server {
-
-class ConfigHandler {
- public:
-    ConfigHandler() = default;
-    virtual ~ConfigHandler() = default;
-
- protected:
-    void
-    SetIdentity(const std::string& identity) {
-        auto& config = server::Config::GetInstance();
-        config.GenUniqueIdentityID(identity, identity_);
+
+std::mutex config_mutex;
+
+std::mutex&
+GetConfigMutex() {
+    return config_mutex;
+}
+
+ServerConfig config;
+
+std::vector<std::string>
+ParsePreloadCollection(const std::string& str) {
+    std::stringstream ss(str);
+    std::vector<std::string> collections;
+    std::string collection;
+
+    while (std::getline(ss, collection, ',')) {
+        collections.push_back(collection);
+    }
+    return collections;
+}
+
+std::vector<int64_t>
+ParseGPUDevices(const std::string& str) {
+    std::stringstream ss(str);
+    std::vector<int64_t> devices;
+    std::unordered_set<int64_t> device_set;
+    std::string device;
+
+    while (std::getline(ss, device, ',')) {
+        device_set.insert(std::stoll(device.substr(3)));
     }
 
- protected:
-    std::string identity_;
-};
+    for (auto dev : device_set) devices.push_back(dev);
+    return devices;
+}
 
-}  // namespace server
 }  // namespace milvus

core/src/config/YamlConfigMgr.cpp

-// Copyright (C) 2019-2020 Zilliz. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software distributed under the License
-// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
-// or implied. See the License for the specific language governing permissions and limitations under the License.
-
-#include "config/YamlConfigMgr.h"
-#include "utils/Log.h"
-
-namespace milvus {
-namespace server {
-
-Status
-YamlConfigMgr::LoadConfigFile(const std::string& filename) {
-    try {
-        node_ = YAML::LoadFile(filename);
-        LoadConfigNode(node_, config_);
-    } catch (YAML::Exception& e) {
-        std::string str = "Exception: load config file fail: " + std::string(e.what());
-        return Status(SERVER_UNEXPECTED_ERROR, str);
-    }
-
-    return Status::OK();
-}
-
-void
-YamlConfigMgr::Print() const {
-    LOG_SERVER_INFO_ << "System config content:";
-    config_.PrintAll();
-}
-
-std::string
-YamlConfigMgr::DumpString() const {
-    return config_.DumpString("");
-}
-
-const ConfigNode&
-YamlConfigMgr::GetRootNode() const {
-    return config_;
-}
-
-ConfigNode&
-YamlConfigMgr::GetRootNode() {
-    return config_;
-}
-
-bool
-YamlConfigMgr::SetConfigValue(const YAML::Node& node, const std::string& key, ConfigNode& config) {
-    if (node[key].IsDefined()) {
-        config.SetValue(key, node[key].as<std::string>());
-        return true;
-    }
-    return false;
-}
-
-bool
-YamlConfigMgr::SetChildConfig(const YAML::Node& node, const std::string& child_name, ConfigNode& config) {
-    if (node[child_name].IsDefined()) {
-        ConfigNode sub_config;
-        LoadConfigNode(node[child_name], sub_config);
-        config.AddChild(child_name, sub_config);
-        return true;
-    }
-    return false;
-}
-
-bool
-YamlConfigMgr::SetSequence(const YAML::Node& node, const std::string& child_name, ConfigNode& config) {
-    if (node[child_name].IsDefined()) {
-        size_t cnt = node[child_name].size();
-        for (size_t i = 0; i < cnt; ++i) {
-            config.AddSequenceItem(child_name, node[child_name][i].as<std::string>());
-        }
-        return true;
-    }
-    return false;
-}
-
-void
-YamlConfigMgr::LoadConfigNode(const YAML::Node& node, ConfigNode& config) {
-    std::string key;
-    for (YAML::const_iterator it = node.begin(); it != node.end(); ++it) {
-        if (!it->first.IsNull()) {
-            key = it->first.as<std::string>();
-        }
-        if (node[key].IsScalar()) {
-            SetConfigValue(node, key, config);
-        } else if (node[key].IsMap()) {
-            SetChildConfig(node, key, config);
-        } else if (node[key].IsSequence()) {
-            SetSequence(node, key, config);
-        }
-    }
-}
-
-}  // namespace server
-}  // namespace milvus

core/src/db/DBImpl.h

@@ -22,8 +22,7 @@
 #include <unordered_map>
 #include <vector>
 
-#include "config/handler/CacheConfigHandler.h"
-#include "config/handler/EngineConfigHandler.h"
+#include "config/ConfigMgr.h"
 #include "db/DB.h"
 #include "db/IndexFailedChecker.h"
 #include "db/SimpleWaitNotify.h"
@@ -42,7 +41,7 @@ namespace meta {
 class Meta;
 }
 
-class DBImpl : public DB, public server::CacheConfigHandler, public server::EngineConfigHandler {
+class DBImpl : public DB, public ConfigObserver {
  public:
     explicit DBImpl(const DBOptions& options);
 
@@ -197,12 +196,9 @@ class DBImpl : public DB, public server::CacheConfigHandler, public server::Engi
     Status
     FlushAttrsIndex(const std::string& collection_id) override;
 
- protected:
-    void
-    OnCacheInsertDataChanged(bool value) override;
-
+ public:
     void
-    OnUseBlasThresholdChanged(int64_t threshold) override;
+    ConfigUpdate(const std::string& name) override;
 
  private:
     Status