From ee7fb3684ad70e8263b92a7070b97cc5c7dd0e91 Mon Sep 17 00:00:00 2001 From: "yudong.cai" Date: Wed, 25 Sep 2019 19:16:12 +0800 Subject: [PATCH] MS-574 add config check Former-commit-id: ac1ed68cc7fcd08cde02bccbd574ac34fdbfdb97 --- cpp/CHANGELOG.md | 3 +- cpp/src/server/Config.cpp | 971 +++++++++++-------------------- cpp/src/server/Config.h | 98 ++-- cpp/src/server/Server.cpp | 8 +- cpp/src/utils/ValidationUtil.cpp | 40 +- cpp/src/utils/ValidationUtil.h | 4 +- 6 files changed, 444 insertions(+), 680 deletions(-) diff --git a/cpp/CHANGELOG.md b/cpp/CHANGELOG.md index 667399cc..2281494f 100644 --- a/cpp/CHANGELOG.md +++ b/cpp/CHANGELOG.md @@ -18,7 +18,8 @@ Please mark all change in change log and use the ticket from JIRA. - MS-562 - Add JobMgr and TaskCreator in Scheduler - MS-566 - Refactor cmake - MS-555 - Remove old scheduler -- MS-578 - Makesure milvus5.0 don't crack 0.3.1 data +- MS-574 - Milvus configuration refactor +- MS-578 - Make sure milvus5.0 don't crack 0.3.1 data ## New Feature diff --git a/cpp/src/server/Config.cpp b/cpp/src/server/Config.cpp index 9d76fa72..b89f1e3e 100644 --- a/cpp/src/server/Config.cpp +++ b/cpp/src/server/Config.cpp @@ -33,7 +33,6 @@ namespace zilliz { namespace milvus { namespace server { -constexpr uint64_t MB = 1UL << 20; constexpr uint64_t GB = 1UL << 30; Config & @@ -71,545 +70,272 @@ Config::LoadConfigFile(const std::string &filename) { return Status::OK(); } -Status -Config::ValidateConfig() { - if (!CheckServerConfig().ok()) { - return Status(SERVER_INVALID_ARGUMENT, "Server config validation check fail"); - } - if (!CheckDBConfig().ok()) { - return Status(SERVER_INVALID_ARGUMENT, "DB config validation check fail"); - } - if (!CheckMetricConfig().ok()) { - return Status(SERVER_INVALID_ARGUMENT, "Metric config validation check fail"); - } - if (!CheckCacheConfig().ok()) { - return Status(SERVER_INVALID_ARGUMENT, "Cache config validation check fail"); - } - if (!CheckEngineConfig().ok()) { - return Status(SERVER_INVALID_ARGUMENT, "Engine config validation check fail"); - } - if (!CheckResourceConfig().ok()) { - return Status(SERVER_INVALID_ARGUMENT, "Resource config validation check fail"); +void +Config::PrintConfigSection(const std::string& config_node_name) { + std::cout << std::endl; + std::cout << config_node_name << ":" << std::endl; + if (config_map_.find(config_node_name) != config_map_.end()) { + for (auto item: config_map_[config_node_name]) { + std::cout << item.first << ": " << item.second << std::endl; + } } - return Status::OK(); } +void +Config::PrintAll() { + PrintConfigSection(CONFIG_SERVER); + PrintConfigSection(CONFIG_DB); + PrintConfigSection(CONFIG_CACHE); + PrintConfigSection(CONFIG_METRIC); + PrintConfigSection(CONFIG_ENGINE); + PrintConfigSection(CONFIG_RESOURCE); +} + +//////////////////////////////////////////////////////////////////////////////// Status -Config::CheckServerConfig() { -/* - server_config: - address: 0.0.0.0 # milvus server ip address (IPv4) - port: 19530 # the port milvus listen to, default: 19530, range: 1025 ~ 65534 - mode: single # milvus deployment type: single, cluster, read_only - time_zone: UTC+8 # Use the UTC-x or UTC+x to specify a time zone. eg. UTC+8 for China Standard Time - -*/ - bool okay = true; - ConfigNode server_config = GetConfigNode(CONFIG_SERVER); - - std::string ip_address = server_config.GetValue(CONFIG_SERVER_ADDRESS, CONFIG_SERVER_ADDRESS_DEFAULT); - if (!ValidationUtil::ValidateIpAddress(ip_address).ok()) { - std::cerr << "ERROR: invalid server IP address: " << ip_address << std::endl; - okay = false; +Config::CheckServerConfigAddress(std::string &value) { + if (!ValidationUtil::ValidateIpAddress(value).ok()) { + return Status(SERVER_INVALID_ARGUMENT, "Invalid server config address: " + value); } + return Status::OK(); +} - std::string port_str = server_config.GetValue(CONFIG_SERVER_PORT, CONFIG_SERVER_PORT_DEFAULT); - if (!ValidationUtil::ValidateStringIsNumber(port_str).ok()) { - std::cerr << "ERROR: port " << port_str << " is not a number" << std::endl; - okay = false; +Status +Config::CheckServerConfigPort(std::string &value) { + if (!ValidationUtil::ValidateStringIsNumber(value).ok()) { + return Status(SERVER_INVALID_ARGUMENT, "Invalid server config port: " + value); } else { - int32_t port = std::stol(port_str); + int32_t port = std::stoi(value); if (!(port > 1024 && port < 65535)) { - std::cerr << "ERROR: port " << port_str << " out of range (1024, 65535)" << std::endl; - okay = false; + return Status(SERVER_INVALID_ARGUMENT, "Server config port out of range (1024, 65535): " + value); } } + return Status::OK(); +} - std::string mode = server_config.GetValue(CONFIG_SERVER_MODE, CONFIG_SERVER_MODE_DEFAULT); - if (mode != "single" && mode != "cluster" && mode != "read_only") { - std::cerr << "ERROR: mode " << mode << " is not one of ['single', 'cluster', 'read_only']" << std::endl; - okay = false; +Status +Config::CheckServerConfigMode(std::string &value) { + if (value != "single" && value != "cluster" && value != "read_only") { + return Status(SERVER_INVALID_ARGUMENT, "Invalid server config mode [single, cluster, read_only]: " + value); } + return Status::OK(); +} - std::string time_zone = server_config.GetValue(CONFIG_SERVER_TIME_ZONE, CONFIG_SERVER_TIME_ZONE_DEFAULT); - int flag = 0; - if (time_zone.length() <= 3) { - flag = 1; +Status +Config::CheckServerConfigTimeZone(std::string &value) { + if (value.length() <= 3) { + return Status(SERVER_INVALID_ARGUMENT, "Invalid server config time_zone: " + value); } else { - if (time_zone.substr(0, 3) != "UTC") { - flag = 1; + if (value.substr(0, 3) != "UTC") { + return Status(SERVER_INVALID_ARGUMENT, "Invalid server config time_zone: " + value); } else { try { - stoi(time_zone.substr(3)); + stoi(value.substr(3)); } catch (...) { - flag = 1; + return Status(SERVER_INVALID_ARGUMENT, "Invalid server config time_zone: " + value); } } } - if (flag == 1) { - std::cerr << "ERROR: time_zone " << time_zone << " format wrong" << std::endl; - okay = false; - } - - return (okay ? Status::OK() : Status(SERVER_INVALID_ARGUMENT, "Illegal server config")); + return Status::OK(); } Status -Config::CheckDBConfig() { -/* - db_config: - db_path: @MILVUS_DB_PATH@ # milvus data storage path - db_slave_path: # secondry data storage path, split by semicolon - - # URI format: dialect://username:password@host:port/database - # All parts except dialect are optional, but you MUST include the delimiters - # Currently dialect supports mysql or sqlite - db_backend_url: sqlite://:@:/ - - archive_disk_threshold: 0 # triger archive action if storage size exceed this value, 0 means no limit, unit: GB - archive_days_threshold: 0 # files older than x days will be archived, 0 means no limit, unit: day - insert_buffer_size: 4 # maximum insert buffer size allowed, default: 4, unit: GB, should be at least 1 GB. - # the sum of insert_buffer_size and cpu_cache_capacity should be less than total memory, unit: GB - build_index_gpu: 0 # which gpu is used to build index, default: 0, range: 0 ~ gpu number - 1 -*/ - bool okay = true; - ConfigNode db_config = GetConfigNode(CONFIG_DB); - - std::string db_path = db_config.GetValue(CONFIG_DB_PATH); - if (db_path.empty()) { - std::cerr << "ERROR: db_path is empty" << std::endl; - okay = false; +Config::CheckDBConfigPath(const std::string &value) { + if (value.empty()) { + return Status(SERVER_INVALID_ARGUMENT, "DB config path empty"); } + return Status::OK(); +} - std::string db_backend_url = db_config.GetValue(CONFIG_DB_BACKEND_URL); - if (!ValidationUtil::ValidateDbURI(db_backend_url).ok()) { - std::cerr << "ERROR: invalid db_backend_url: " << db_backend_url << std::endl; - okay = false; +Status +Config::CheckDBConfigBackendUrl(const std::string &value) { + if (!ValidationUtil::ValidateDbURI(value).ok()) { + return Status(SERVER_INVALID_ARGUMENT, "Invalid DB config backend_url: " + value); } + return Status::OK(); +} - std::string archive_disk_threshold_str = - db_config.GetValue(CONFIG_DB_ARCHIVE_DISK_THRESHOLD, CONFIG_DB_ARCHIVE_DISK_THRESHOLD_DEFAULT); - if (!ValidationUtil::ValidateStringIsNumber(archive_disk_threshold_str).ok()) { - std::cerr << "ERROR: archive_disk_threshold " << archive_disk_threshold_str << " is not a number" << std::endl; - okay = false; +Status +Config::CheckDBConfigArchiveDiskThreshold(const std::string &value) { + if (!ValidationUtil::ValidateStringIsNumber(value).ok()) { + return Status(SERVER_INVALID_ARGUMENT, "Invalid DB config archive_disk_threshold: " + value); } + return Status::OK(); +} - std::string archive_days_threshold_str = - db_config.GetValue(CONFIG_DB_ARCHIVE_DAYS_THRESHOLD, CONFIG_DB_ARCHIVE_DAYS_THRESHOLD_DEFAULT); - if (!ValidationUtil::ValidateStringIsNumber(archive_days_threshold_str).ok()) { - std::cerr << "ERROR: archive_days_threshold " << archive_days_threshold_str << " is not a number" << std::endl; - okay = false; +Status +Config::CheckDBConfigArchiveDaysThreshold(const std::string &value) { + if (!ValidationUtil::ValidateStringIsNumber(value).ok()) { + return Status(SERVER_INVALID_ARGUMENT, "Invalid DB config archive_days_threshold: " + value); } + return Status::OK(); +} - std::string buffer_size_str = db_config.GetValue(CONFIG_DB_BUFFER_SIZE, CONFIG_DB_BUFFER_SIZE_DEFAULT); - if (!ValidationUtil::ValidateStringIsNumber(buffer_size_str).ok()) { - std::cerr << "ERROR: buffer_size " << buffer_size_str << " is not a number" << std::endl; - okay = false; +Status +Config::CheckDBConfigBufferSize(const std::string &value) { + if (!ValidationUtil::ValidateStringIsNumber(value).ok()) { + return Status(SERVER_INVALID_ARGUMENT, "Invalid DB config buffer_size: " + value); } else { - uint64_t buffer_size = (uint64_t) std::stol(buffer_size_str); - buffer_size *= GB; + int64_t buffer_size = std::stoi(value) * GB; unsigned long total_mem = 0, free_mem = 0; CommonUtil::GetSystemMemInfo(total_mem, free_mem); if (buffer_size >= total_mem) { - std::cerr << "ERROR: buffer_size exceed system memory" << std::endl; - okay = false; + return Status(SERVER_INVALID_ARGUMENT, "DB config buffer_size exceed system memory: " + value); } } + return Status::OK(); +} - std::string gpu_index_str = db_config.GetValue(CONFIG_DB_BUILD_INDEX_GPU, CONFIG_DB_BUILD_INDEX_GPU_DEFAULT); - if (!ValidationUtil::ValidateStringIsNumber(gpu_index_str).ok()) { - std::cerr << "ERROR: gpu_index " << gpu_index_str << " is not a number" << std::endl; - okay = false; +Status +Config::CheckDBConfigBuildIndexGPU(const std::string &value) { + if (!ValidationUtil::ValidateStringIsNumber(value).ok()) { + return Status(SERVER_INVALID_ARGUMENT, "Invalid DB config build_index_gpu: " + value); } else { - int32_t gpu_index = std::stol(gpu_index_str); + int32_t gpu_index = std::stoi(value); if (!ValidationUtil::ValidateGpuIndex(gpu_index).ok()) { - std::cerr << "ERROR: invalid gpu_index " << gpu_index_str << std::endl; - okay = false; + return Status(SERVER_INVALID_ARGUMENT, "Invalid DB config build_index_gpu: " + value); } } - - return (okay ? Status::OK() : Status(SERVER_INVALID_ARGUMENT, "DB config is illegal")); + return Status::OK(); } Status -Config::CheckMetricConfig() { -/* - metric_config: - is_startup: off # if monitoring start: on, off - collector: prometheus # metrics collector: prometheus - prometheus_config: # following are prometheus configure - port: 8080 # the port prometheus use to fetch metrics - (not used) push_gateway_ip_address: 127.0.0.1 # push method configure: push gateway ip address - (not used) push_gateway_port: 9091 # push method configure: push gateway port -*/ - bool okay = true; - ConfigNode metric_config = GetConfigNode(CONFIG_METRIC); - - std::string is_startup_str = - metric_config.GetValue(CONFIG_METRIC_AUTO_BOOTUP, CONFIG_METRIC_AUTO_BOOTUP_DEFAULT); - if (!ValidationUtil::ValidateStringIsBool(is_startup_str).ok()) { - std::cerr << "ERROR: invalid is_startup config: " << is_startup_str << std::endl; - okay = false; +Config::CheckMetricConfigAutoBootup(const std::string& value) { + if (!ValidationUtil::ValidateStringIsBool(value).ok()) { + return Status(SERVER_INVALID_ARGUMENT, "Invalid metric config auto_bootup: " + value); } + return Status::OK(); +} - std::string - port_str = metric_config.GetChild(CONFIG_METRIC_PROMETHEUS).GetValue(CONFIG_METRIC_PROMETHEUS_PORT, "8080"); - if (!ValidationUtil::ValidateStringIsNumber(port_str).ok()) { - std::cerr << "ERROR: port specified in prometheus_config " << port_str << " is not a number" << std::endl; - okay = false; +Status +Config::CheckMetricConfigPrometheusPort(const std::string& value) { + if (!ValidationUtil::ValidateStringIsNumber(value).ok()) { + return Status(SERVER_INVALID_ARGUMENT, "Invalid metric config prometheus_port: " + value); } - - return (okay ? Status::OK() : Status(SERVER_INVALID_ARGUMENT, "Metric config is illegal")); + return Status::OK(); } Status -Config::CheckCacheConfig() { -/* - cache_config: - cpu_cache_capacity: 16 # how many memory are used as cache, unit: GB, range: 0 ~ less than total memory - cpu_cache_free_percent: 0.85 # old data will be erased from cache when cache is full, this value specify how much memory should be kept, range: greater than zero ~ 1.0 - insert_cache_immediately: false # insert data will be load into cache immediately for hot query - gpu_cache_capacity: 5 # how many memory are used as cache in gpu, unit: GB, RANGE: 0 ~ less than total memory - gpu_cache_free_percent: 0.85 # old data will be erased from cache when cache is full, this value specify how much memory should be kept, range: greater than zero ~ 1.0 - -*/ - bool okay = true; - ConfigNode cache_config = GetConfigNode(CONFIG_CACHE); - - std::string cpu_cache_capacity_str = - cache_config.GetValue(CONFIG_CACHE_CPU_MEM_CAPACITY, CONFIG_CACHE_CPU_MEM_CAPACITY_DEFAULT); - if (!ValidationUtil::ValidateStringIsNumber(cpu_cache_capacity_str).ok()) { - std::cerr << "ERROR: cpu_cache_capacity " << cpu_cache_capacity_str << " is not a number" << std::endl; - okay = false; +Config::CheckCacheConfigCpuMemCapacity(const std::string& value) { + if (!ValidationUtil::ValidateStringIsNumber(value).ok()) { + return Status(SERVER_INVALID_ARGUMENT, "Invalid cache config cpu_mem_capacity: " + value); } else { - uint64_t cpu_cache_capacity = (uint64_t) std::stol(cpu_cache_capacity_str); - cpu_cache_capacity *= GB; + uint64_t cpu_cache_capacity = std::stoi(value) * GB; unsigned long total_mem = 0, free_mem = 0; CommonUtil::GetSystemMemInfo(total_mem, free_mem); if (cpu_cache_capacity >= total_mem) { - std::cerr << "ERROR: cpu_cache_capacity exceed system memory" << std::endl; - okay = false; + return Status(SERVER_INVALID_ARGUMENT, "Cache config cpu_mem_capacity exceed system memory: " + value); } else if (cpu_cache_capacity > (double) total_mem * 0.9) { - std::cerr << "Warning: cpu_cache_capacity value is too aggressive" << std::endl; + std::cerr << "Warning: cpu_mem_capacity value is too big" << std::endl; } - uint64_t buffer_size = - (uint64_t) GetConfigNode(CONFIG_DB).GetInt32Value(CONFIG_DB_BUFFER_SIZE, - std::stoi(CONFIG_DB_BUFFER_SIZE_DEFAULT)); - buffer_size *= GB; - if (buffer_size + cpu_cache_capacity >= total_mem) { - std::cerr << "ERROR: sum of cpu_cache_capacity and insert_buffer_size exceed system memory" << std::endl; - okay = false; + int32_t buffer_size; + Status s = GetDBConfigBufferSize(buffer_size); + if (!s.ok()) return s; + int64_t insert_buffer_size = buffer_size * GB; + if (insert_buffer_size + cpu_cache_capacity >= total_mem) { + return Status(SERVER_INVALID_ARGUMENT, "Sum of cpu_mem_capacity and buffer_size exceed system memory"); } } + return Status::OK(); +} - std::string cpu_cache_free_percent_str = - cache_config.GetValue(CONFIG_CACHE_CPU_MEM_THRESHOLD, CONFIG_CACHE_CPU_MEM_THRESHOLD_DEFAULT); - double cpu_cache_free_percent; - if (!ValidationUtil::ValidateStringIsDouble(cpu_cache_free_percent_str, cpu_cache_free_percent).ok()) { - std::cerr << "ERROR: cpu_cache_free_percent " << cpu_cache_free_percent_str << " is not a double" << std::endl; - okay = false; - } else if (cpu_cache_free_percent < std::numeric_limits::epsilon() || cpu_cache_free_percent > 1.0) { - std::cerr << "ERROR: invalid cpu_cache_free_percent " << cpu_cache_free_percent_str << std::endl; - okay = false; - } - - std::string insert_cache_immediately_str = - cache_config.GetValue(CONFIG_CACHE_CACHE_INSERT_DATA, CONFIG_CACHE_CACHE_INSERT_DATA_DEFAULT); - if (!ValidationUtil::ValidateStringIsBool(insert_cache_immediately_str).ok()) { - std::cerr << "ERROR: invalid insert_cache_immediately config: " << insert_cache_immediately_str << std::endl; - okay = false; +Status +Config::CheckCacheConfigCpuMemThreshold(const std::string& value) { + if (!ValidationUtil::ValidateStringIsFloat(value).ok()) { + return Status(SERVER_INVALID_ARGUMENT, "Invalid cache config cpu_mem_threshold: " + value); + } else { + float cpu_mem_threshold = std::stof(value); + if (cpu_mem_threshold <= 0.0 || cpu_mem_threshold >= 1.0) { + return Status(SERVER_INVALID_ARGUMENT, "Invalid cache config cpu_mem_threshold: " + value); + } } + return Status::OK(); +} - std::string gpu_cache_capacity_str = - cache_config.GetValue(CONFIG_CACHE_GPU_MEM_CAPACITY, CONFIG_CACHE_GPU_MEM_CAPACITY_DEFAULT); - if (!ValidationUtil::ValidateStringIsNumber(gpu_cache_capacity_str).ok()) { - std::cerr << "ERROR: gpu_cache_capacity " << gpu_cache_capacity_str << " is not a number" << std::endl; - okay = false; +Status +Config::CheckCacheConfigGpuMemCapacity(const std::string& value) { + if (!ValidationUtil::ValidateStringIsNumber(value).ok()) { + std::cerr << "ERROR: gpu_cache_capacity " << value << " is not a number" << std::endl; } else { - uint64_t gpu_cache_capacity = (uint64_t) std::stol(gpu_cache_capacity_str); - gpu_cache_capacity *= GB; - int gpu_index = GetConfigNode(CONFIG_DB).GetInt32Value(CONFIG_DB_BUILD_INDEX_GPU, - std::stoi(CONFIG_DB_BUILD_INDEX_GPU_DEFAULT)); + uint64_t gpu_cache_capacity = std::stoi(value) * GB; + int gpu_index; + Status s = GetDBConfigBuildIndexGPU(gpu_index); + if (!s.ok()) return s; size_t gpu_memory; if (!ValidationUtil::GetGpuMemory(gpu_index, gpu_memory).ok()) { - std::cerr << "ERROR: could not get gpu memory for device " << gpu_index << std::endl; - okay = false; + return Status(SERVER_UNEXPECTED_ERROR, + "Fail to get GPU memory for GPU device: " + std::to_string(gpu_index)); } else if (gpu_cache_capacity >= gpu_memory) { - std::cerr << "ERROR: gpu_cache_capacity " << gpu_cache_capacity - << " exceed total gpu memory " << gpu_memory << std::endl; - okay = false; + return Status(SERVER_INVALID_ARGUMENT, + "Cache config gpu_mem_capacity exceed GPU memory: " + std::to_string(gpu_memory)); } else if (gpu_cache_capacity > (double) gpu_memory * 0.9) { - std::cerr << "Warning: gpu_cache_capacity value is too aggressive" << std::endl; + std::cerr << "Warning: gpu_mem_capacity value is too big" << std::endl; } } + return Status::OK(); +} - std::string gpu_cache_free_percent_str = - cache_config.GetValue(CONFIG_CACHE_GPU_MEM_THRESHOLD, CONFIG_CACHE_GPU_MEM_THRESHOLD_DEFAULT); - double gpu_cache_free_percent; - if (!ValidationUtil::ValidateStringIsDouble(gpu_cache_free_percent_str, gpu_cache_free_percent).ok()) { - std::cerr << "ERROR: gpu_cache_free_percent " << gpu_cache_free_percent_str << " is not a double" << std::endl; - okay = false; - } else if (gpu_cache_free_percent < std::numeric_limits::epsilon() || gpu_cache_free_percent > 1.0) { - std::cerr << "ERROR: invalid gpu_cache_free_percent " << gpu_cache_free_percent << std::endl; - okay = false; +Status +Config::CheckCacheConfigGpuMemThreshold(const std::string& value) { + if (!ValidationUtil::ValidateStringIsFloat(value).ok()) { + return Status(SERVER_INVALID_ARGUMENT, "Invalid cache config gpu_mem_threshold: " + value); + } else { + float gpu_mem_threshold = std::stof(value); + if (gpu_mem_threshold <= 0.0 || gpu_mem_threshold >= 1.0) { + return Status(SERVER_INVALID_ARGUMENT, "Invalid cache config gpu_mem_threshold: " + value); + } } + return Status::OK(); +} - return (okay ? Status::OK() : Status(SERVER_INVALID_ARGUMENT, "Cache config is illegal")); +Status +Config::CheckCacheConfigCacheInsertData(const std::string& value) { + if (!ValidationUtil::ValidateStringIsBool(value).ok()) { + return Status(SERVER_INVALID_ARGUMENT, "Invalid cache config cache_insert_data: " + value); + } + return Status::OK(); } Status -Config::CheckEngineConfig() { -/* - engine_config: - use_blas_threshold: 20 - omp_thread_num: 0 # how many compute threads be used by engine, 0 means use all cpu core to compute -*/ - bool okay = true; - ConfigNode engine_config = GetConfigNode(CONFIG_ENGINE); - - std::string use_blas_threshold_str = - engine_config.GetValue(CONFIG_ENGINE_BLAS_THRESHOLD, CONFIG_ENGINE_BLAS_THRESHOLD_DEFAULT); - if (!ValidationUtil::ValidateStringIsNumber(use_blas_threshold_str).ok()) { - std::cerr << "ERROR: use_blas_threshold " << use_blas_threshold_str << " is not a number" << std::endl; - okay = false; +Config::CheckEngineConfigBlasThreshold(const std::string& value) { + if (!ValidationUtil::ValidateStringIsNumber(value).ok()) { + return Status(SERVER_INVALID_ARGUMENT, "Invalid engine config blas threshold: " + value); } + return Status::OK(); +} - std::string omp_thread_num_str = - engine_config.GetValue(CONFIG_ENGINE_OMP_THREAD_NUM, CONFIG_ENGINE_OMP_THREAD_NUM_DEFAULT); - if (!ValidationUtil::ValidateStringIsNumber(omp_thread_num_str).ok()) { - std::cerr << "ERROR: omp_thread_num " << omp_thread_num_str << " is not a number" << std::endl; - okay = false; +Status +Config::CheckEngineConfigOmpThreadNum(const std::string& value) { + if (!ValidationUtil::ValidateStringIsNumber(value).ok()) { + return Status(SERVER_INVALID_ARGUMENT, "Invalid engine config omp_thread_num: " + value); } else { - int32_t omp_thread = std::stol(omp_thread_num_str); + int32_t omp_thread = std::stoi(value); uint32_t sys_thread_cnt = 8; if (omp_thread > CommonUtil::GetSystemAvailableThreads(sys_thread_cnt)) { - std::cerr << "ERROR: omp_thread_num " << omp_thread_num_str << " > system available thread " - << sys_thread_cnt << std::endl; - okay = false; + return Status(SERVER_INVALID_ARGUMENT, "Invalid engine config omp_thread_num: " + value); } } - - return (okay ? Status::OK() : Status(SERVER_INVALID_ARGUMENT, "Engine config is illegal")); + return Status::OK(); } Status -Config::CheckResourceConfig() { - /* - resource_config: - mode: simple - pool: - - cpu - - gpu0 - - gpu100 - */ - bool okay = true; - server::ConfigNode &config = GetConfigNode(server::CONFIG_RESOURCE); - auto mode = config.GetValue(CONFIG_RESOURCE_MODE, CONFIG_RESOURCE_MODE_DEFAULT); - if (mode != "simple") { - std::cerr << "ERROR: invalid resource config: mode is " << mode << std::endl; - okay = false; +Config::CheckResourceConfigMode(const std::string& value) { + if (value != "simple") { + return Status(SERVER_INVALID_ARGUMENT, "Invalid resource config mode: " + value); } - auto pool = config.GetSequence(CONFIG_RESOURCE_POOL); - if (pool.empty()) { - std::cerr << "ERROR: invalid resource config: resources empty" << std::endl; - okay = false; - } - - return (okay ? Status::OK() : Status(SERVER_INVALID_ARGUMENT, "Resource config is illegal")); -} - -//Status -//Config::CheckResourceConfig() { -/* - resource_config: - # resource list, length: 0~N - # please set a DISK resource and a CPU resource least, or system will not return query result. - # - # example: - # resource_name: # resource name, just using in connections below - # type: DISK # resource type, optional: DISK/CPU/GPU - # device_id: 0 - # enable_executor: false # if is enable executor, optional: true, false - - resources: - ssda: - type: DISK - device_id: 0 - enable_executor: false - - cpu: - type: CPU - device_id: 0 - enable_executor: true - - gpu0: - type: GPU - device_id: 0 - enable_executor: false - gpu_resource_num: 2 - pinned_memory: 300 - temp_memory: 300 - - # connection list, length: 0~N - # example: - # connection_name: - # speed: 100 # unit: MS/s - # endpoint: ${resource_name}===${resource_name} - connections: - io: - speed: 500 - endpoint: ssda===cpu - pcie0: - speed: 11000 - endpoint: cpu===gpu0 -*/ -// bool okay = true; -// server::ConfigNode resource_config = GetConfig(CONFIG_RESOURCE); -// if (resource_config.GetChildren().empty()) { -// std::cerr << "ERROR: no context under resource" << std::endl; -// okay = false; -// } -// -// auto resources = resource_config.GetChild(CONFIG_RESOURCES).GetChildren(); -// -// if (resources.empty()) { -// std::cerr << "no resources specified" << std::endl; -// okay = false; -// } -// -// bool resource_valid_flag = false; -// bool hasDisk = false; -// bool hasCPU = false; -// bool hasExecutor = false; -// std::set resource_list; -// for (auto &resource : resources) { -// resource_list.emplace(resource.first); -// auto &resource_conf = resource.second; -// auto type = resource_conf.GetValue(CONFIG_RESOURCE_TYPE); -// -// std::string device_id_str = resource_conf.GetValue(CONFIG_RESOURCE_DEVICE_ID, "0"); -// int32_t device_id = -1; -// if (!ValidationUtil::ValidateStringIsNumber(device_id_str).ok()) { -// std::cerr << "ERROR: device_id " << device_id_str << " is not a number" << std::endl; -// okay = false; -// } else { -// device_id = std::stol(device_id_str); -// } -// -// std::string enable_executor_str = resource_conf.GetValue(CONFIG_RESOURCE_ENABLE_EXECUTOR, "off"); -// if (!ValidationUtil::ValidateStringIsBool(enable_executor_str).ok()) { -// std::cerr << "ERROR: invalid enable_executor config: " << enable_executor_str << std::endl; -// okay = false; -// } -// -// if (type == "DISK") { -// hasDisk = true; -// } else if (type == "CPU") { -// hasCPU = true; -// if (resource_conf.GetBoolValue(CONFIG_RESOURCE_ENABLE_EXECUTOR, false)) { -// hasExecutor = true; -// } -// } -// else if (type == "GPU") { -// int build_index_gpu_index = GetConfig(CONFIG_DB).GetInt32Value(CONFIG_DB_BUILD_INDEX_GPU, 0); -// if (device_id == build_index_gpu_index) { -// resource_valid_flag = true; -// } -// if (resource_conf.GetBoolValue(CONFIG_RESOURCE_ENABLE_EXECUTOR, false)) { -// hasExecutor = true; -// } -// std::string gpu_resource_num_str = resource_conf.GetValue(CONFIG_RESOURCE_NUM, "2"); -// if (!ValidationUtil::ValidateStringIsNumber(gpu_resource_num_str).ok()) { -// std::cerr << "ERROR: gpu_resource_num " << gpu_resource_num_str << " is not a number" << std::endl; -// okay = false; -// } -// bool mem_valid = true; -// std::string pinned_memory_str = resource_conf.GetValue(CONFIG_RESOURCE_PIN_MEMORY, "300"); -// if (!ValidationUtil::ValidateStringIsNumber(pinned_memory_str).ok()) { -// std::cerr << "ERROR: pinned_memory " << pinned_memory_str << " is not a number" << std::endl; -// okay = false; -// mem_valid = false; -// } -// std::string temp_memory_str = resource_conf.GetValue(CONFIG_RESOURCE_TEMP_MEMORY, "300"); -// if (!ValidationUtil::ValidateStringIsNumber(temp_memory_str).ok()) { -// std::cerr << "ERROR: temp_memory " << temp_memory_str << " is not a number" << std::endl; -// okay = false; -// mem_valid = false; -// } -// if (mem_valid) { -// size_t gpu_memory; -// if (!ValidationUtil::GetGpuMemory(device_id, gpu_memory).ok()) { -// std::cerr << "ERROR: could not get gpu memory for device " << device_id << std::endl; -// okay = false; -// } -// else { -// size_t prealoc_mem = std::stol(pinned_memory_str) + std::stol(temp_memory_str); -// if (prealoc_mem >= gpu_memory) { -// std::cerr << "ERROR: sum of pinned_memory and temp_memory " << prealoc_mem -// << " exceeds total gpu memory " << gpu_memory << " for device " << device_id << std::endl; -// okay = false; -// } -// } -// } -// } -// } -// -// if (!resource_valid_flag) { -// std::cerr << "Building index GPU can't be found in resource config." << std::endl; -// okay = false; -// } -// if (!hasDisk || !hasCPU) { -// std::cerr << "No DISK or CPU resource" << std::endl; -// okay = false; -// } -// if (!hasExecutor) { -// std::cerr << "No CPU or GPU resource has executor enabled" << std::endl; -// okay = false; -// } -// -// auto connections = resource_config.GetChild(CONFIG_RESOURCE_CONNECTIONS).GetChildren(); -// for (auto &connection : connections) { -// auto &connection_conf = connection.second; -// -// std::string speed_str = connection_conf.GetValue(CONFIG_SPEED_CONNECTIONS); -// if (ValidationUtil::ValidateStringIsNumber(speed_str) != SERVER_SUCCESS) { -// std::cerr << "ERROR: speed " << speed_str << " is not a number" << std::endl; -// okay = false; -// } -// -// std::string endpoint_str = connection_conf.GetValue(CONFIG_ENDPOINT_CONNECTIONS); -// std::string delimiter = "==="; -// auto delimiter_pos = endpoint_str.find(delimiter); -// if (delimiter_pos == std::string::npos) { -// std::cerr << "ERROR: invalid endpoint format: " << endpoint_str << std::endl; -// okay = false; -// } else { -// std::string left_resource = endpoint_str.substr(0, delimiter_pos); -// if (resource_list.find(left_resource) == resource_list.end()) { -// std::cerr << "ERROR: left resource " << left_resource << " does not exist" << std::endl; -// okay = false; -// } -// std::string right_resource = endpoint_str.substr(delimiter_pos + delimiter.length(), endpoint_str.length()); -// if (resource_list.find(right_resource) == resource_list.end()) { -// std::cerr << "ERROR: right resource " << right_resource << " does not exist" << std::endl; -// okay = false; -// } -// } -// } -// -// return (okay ? Status::OK() : Status(SERVER_INVALID_ARGUMENT, "Resource config is illegal")); -//} + return Status::OK(); +} -void -Config::PrintAll() const { - if (const ConfigMgr *mgr = ConfigMgr::GetInstance()) { - std::string str = mgr->DumpString(); -// SERVER_LOG_INFO << "\n" << str; - std::cout << "\n" << str << std::endl; +Status +Config::CheckResourceConfigPool(const std::vector& value) { + if (value.empty()) { + return Status(SERVER_INVALID_ARGUMENT, "Invalid resource config pool"); } + return Status::OK(); } +//////////////////////////////////////////////////////////////////////////////// ConfigNode & Config::GetConfigNode(const std::string &name) { ConfigMgr *mgr = ConfigMgr::GetInstance(); @@ -631,310 +357,316 @@ Config::GetConfigValueInMem(const std::string &parent_key, } } -Status +void Config::SetConfigValueInMem(const std::string &parent_key, const std::string &child_key, std::string &value) { std::lock_guard lock(mutex_); config_map_[parent_key][child_key] = value; - return Status::OK(); } //////////////////////////////////////////////////////////////////////////////// /* server config */ -Status -Config::GetServerConfigStrAddress(std::string& value) { - if (GetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_ADDRESS, value).ok()) { - return Status::OK(); - } else { +std::string +Config::GetServerConfigStrAddress() { + std::string value; + if (!GetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_ADDRESS, value).ok()) { value = GetConfigNode(CONFIG_SERVER).GetValue(CONFIG_SERVER_ADDRESS, CONFIG_SERVER_ADDRESS_DEFAULT); - return SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_ADDRESS, value); + SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_ADDRESS, value); } + return value; } -Status -Config::GetServerConfigStrPort(std::string& value) { - if (GetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_PORT, value).ok()) { - return Status::OK(); - } else { +std::string +Config::GetServerConfigStrPort() { + std::string value; + if (!GetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_PORT, value).ok()) { value = GetConfigNode(CONFIG_SERVER).GetValue(CONFIG_SERVER_PORT, CONFIG_SERVER_PORT_DEFAULT); - return SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_PORT, value); + SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_PORT, value); } + return value; } -Status -Config::GetServerConfigStrMode(std::string& value) { - if (GetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_MODE, value).ok()) { - return Status::OK(); - } else { +std::string +Config::GetServerConfigStrMode() { + std::string value; + if (!GetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_MODE, value).ok()) { value = GetConfigNode(CONFIG_SERVER).GetValue(CONFIG_SERVER_MODE, CONFIG_SERVER_MODE_DEFAULT); - return SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_MODE, value); + SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_MODE, value); } + return value; } -Status -Config::GetServerConfigStrTimeZone(std::string& value) { - if (GetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_TIME_ZONE, value).ok()) { - return Status::OK(); - } else { +std::string +Config::GetServerConfigStrTimeZone() { + std::string value; + if (!GetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_TIME_ZONE, value).ok()) { value = GetConfigNode(CONFIG_SERVER).GetValue(CONFIG_SERVER_TIME_ZONE, CONFIG_SERVER_TIME_ZONE_DEFAULT); - return SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_TIME_ZONE, value); + SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_TIME_ZONE, value); } + return value; } //////////////////////////////////////////////////////////////////////////////// /* db config */ -Status -Config::GetDBConfigStrPath(std::string& value) { - if (GetConfigValueInMem(CONFIG_DB, CONFIG_DB_PATH, value).ok()) { - return Status::OK(); - } else { +std::string +Config::GetDBConfigStrPath() { + std::string value; + if (!GetConfigValueInMem(CONFIG_DB, CONFIG_DB_PATH, value).ok()) { value = GetConfigNode(CONFIG_DB).GetValue(CONFIG_DB_PATH, CONFIG_DB_PATH_DEFAULT); - return SetConfigValueInMem(CONFIG_DB, CONFIG_DB_PATH, value); + SetConfigValueInMem(CONFIG_DB, CONFIG_DB_PATH, value); } + return value; } -Status -Config::GetDBConfigStrSlavePath(std::string& value) { - if (GetConfigValueInMem(CONFIG_DB, CONFIG_DB_SLAVE_PATH, value).ok()) { - return Status::OK(); - } else { +std::string +Config::GetDBConfigStrSlavePath() { + std::string value; + if (!GetConfigValueInMem(CONFIG_DB, CONFIG_DB_SLAVE_PATH, value).ok()) { value = GetConfigNode(CONFIG_DB).GetValue(CONFIG_DB_SLAVE_PATH, CONFIG_DB_SLAVE_PATH_DEFAULT); - return SetConfigValueInMem(CONFIG_DB, CONFIG_DB_SLAVE_PATH, value); + SetConfigValueInMem(CONFIG_DB, CONFIG_DB_SLAVE_PATH, value); } + return value; } -Status -Config::GetDBConfigStrBackendUrl(std::string& value) { - if (GetConfigValueInMem(CONFIG_DB, CONFIG_DB_BACKEND_URL, value).ok()) { - return Status::OK(); - } else { +std::string +Config::GetDBConfigStrBackendUrl() { + std::string value; + if (!GetConfigValueInMem(CONFIG_DB, CONFIG_DB_BACKEND_URL, value).ok()) { value = GetConfigNode(CONFIG_DB).GetValue(CONFIG_DB_BACKEND_URL, CONFIG_DB_BACKEND_URL_DEFAULT); - return SetConfigValueInMem(CONFIG_DB, CONFIG_DB_BACKEND_URL, value); + SetConfigValueInMem(CONFIG_DB, CONFIG_DB_BACKEND_URL, value); } + return value; } -Status -Config::GetDBConfigStrArchiveDiskThreshold(std::string& value) { - if (GetConfigValueInMem(CONFIG_DB, CONFIG_DB_ARCHIVE_DISK_THRESHOLD, value).ok()) { - return Status::OK(); - } else { +std::string +Config::GetDBConfigStrArchiveDiskThreshold() { + std::string value; + if (!GetConfigValueInMem(CONFIG_DB, CONFIG_DB_ARCHIVE_DISK_THRESHOLD, value).ok()) { value = GetConfigNode(CONFIG_DB).GetValue(CONFIG_DB_ARCHIVE_DISK_THRESHOLD, CONFIG_DB_ARCHIVE_DISK_THRESHOLD_DEFAULT); - return SetConfigValueInMem(CONFIG_DB, CONFIG_DB_ARCHIVE_DISK_THRESHOLD, value); + SetConfigValueInMem(CONFIG_DB, CONFIG_DB_ARCHIVE_DISK_THRESHOLD, value); } + return value; } -Status -Config::GetDBConfigStrArchiveDaysThreshold(std::string& value) { - if (GetConfigValueInMem(CONFIG_DB, CONFIG_DB_ARCHIVE_DAYS_THRESHOLD, value).ok()) { - return Status::OK(); - } else { +std::string +Config::GetDBConfigStrArchiveDaysThreshold() { + std::string value; + if (!GetConfigValueInMem(CONFIG_DB, CONFIG_DB_ARCHIVE_DAYS_THRESHOLD, value).ok()) { value = GetConfigNode(CONFIG_DB).GetValue(CONFIG_DB_ARCHIVE_DAYS_THRESHOLD, CONFIG_DB_ARCHIVE_DAYS_THRESHOLD_DEFAULT); - return SetConfigValueInMem(CONFIG_DB, CONFIG_DB_ARCHIVE_DAYS_THRESHOLD, value); + SetConfigValueInMem(CONFIG_DB, CONFIG_DB_ARCHIVE_DAYS_THRESHOLD, value); } + return value; } -Status -Config::GetDBConfigStrBufferSize(std::string& value) { - if (GetConfigValueInMem(CONFIG_DB, CONFIG_DB_BUFFER_SIZE, value).ok()) { - return Status::OK(); - } else { +std::string +Config::GetDBConfigStrBufferSize() { + std::string value; + if (!GetConfigValueInMem(CONFIG_DB, CONFIG_DB_BUFFER_SIZE, value).ok()) { value = GetConfigNode(CONFIG_DB).GetValue(CONFIG_DB_BUFFER_SIZE, CONFIG_DB_BUFFER_SIZE_DEFAULT); - return SetConfigValueInMem(CONFIG_DB, CONFIG_DB_BUFFER_SIZE, value); + SetConfigValueInMem(CONFIG_DB, CONFIG_DB_BUFFER_SIZE, value); } + return value; } -Status -Config::GetDBConfigStrBuildIndexGPU(std::string& value) { - if (GetConfigValueInMem(CONFIG_DB, CONFIG_DB_BUILD_INDEX_GPU, value).ok()) { - return Status::OK(); - } else { +std::string +Config::GetDBConfigStrBuildIndexGPU() { + std::string value; + if (!GetConfigValueInMem(CONFIG_DB, CONFIG_DB_BUILD_INDEX_GPU, value).ok()) { value = GetConfigNode(CONFIG_DB).GetValue(CONFIG_DB_BUILD_INDEX_GPU, CONFIG_DB_BUILD_INDEX_GPU_DEFAULT); - return SetConfigValueInMem(CONFIG_DB, CONFIG_DB_BUILD_INDEX_GPU, value); + SetConfigValueInMem(CONFIG_DB, CONFIG_DB_BUILD_INDEX_GPU, value); } + return value; } //////////////////////////////////////////////////////////////////////////////// /* metric config */ -Status -Config::GetMetricConfigStrAutoBootup(std::string& value) { - if (GetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_AUTO_BOOTUP, value).ok()) { - return Status::OK(); - } else { +std::string +Config::GetMetricConfigStrAutoBootup() { + std::string value; + if (!GetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_AUTO_BOOTUP, value).ok()) { value = GetConfigNode(CONFIG_METRIC).GetValue(CONFIG_METRIC_AUTO_BOOTUP, CONFIG_METRIC_AUTO_BOOTUP_DEFAULT); - return SetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_AUTO_BOOTUP, value); + SetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_AUTO_BOOTUP, value); } + return value; } -Status -Config::GetMetricConfigStrCollector(std::string& value) { - if (GetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_COLLECTOR, value).ok()) { - return Status::OK(); - } else { +std::string +Config::GetMetricConfigStrCollector() { + std::string value; + if (!GetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_COLLECTOR, value).ok()) { value = GetConfigNode(CONFIG_METRIC).GetValue(CONFIG_METRIC_COLLECTOR, CONFIG_METRIC_COLLECTOR_DEFAULT); - return SetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_COLLECTOR, value); + SetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_COLLECTOR, value); } + return value; } -Status -Config::GetMetricConfigStrPrometheusPort(std::string& value) { - if (GetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_PROMETHEUS_PORT, value).ok()) { - return Status::OK(); - } else { +std::string +Config::GetMetricConfigStrPrometheusPort() { + std::string value; + if (!GetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_PROMETHEUS_PORT, value).ok()) { value = GetConfigNode(CONFIG_METRIC).GetValue(CONFIG_METRIC_PROMETHEUS_PORT, CONFIG_METRIC_PROMETHEUS_PORT_DEFAULT); - return SetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_PROMETHEUS_PORT, value); + SetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_PROMETHEUS_PORT, value); } + return value; } //////////////////////////////////////////////////////////////////////////////// /* cache config */ -Status -Config::GetCacheConfigStrCpuMemCapacity(std::string& value) { - if (GetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CPU_MEM_CAPACITY, value).ok()) { - return Status::OK(); - } else { +std::string +Config::GetCacheConfigStrCpuMemCapacity() { + std::string value; + if (!GetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CPU_MEM_CAPACITY, value).ok()) { value = GetConfigNode(CONFIG_CACHE).GetValue(CONFIG_CACHE_CPU_MEM_CAPACITY, CONFIG_CACHE_CPU_MEM_CAPACITY_DEFAULT); - return SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CPU_MEM_CAPACITY, value); + SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CPU_MEM_CAPACITY, value); } + return value; } -Status -Config::GetCacheConfigStrCpuMemThreshold(std::string& value) { - if (GetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CPU_MEM_THRESHOLD, value).ok()) { - return Status::OK(); - } else { +std::string +Config::GetCacheConfigStrCpuMemThreshold() { + std::string value; + if (!GetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CPU_MEM_THRESHOLD, value).ok()) { value = GetConfigNode(CONFIG_CACHE).GetValue(CONFIG_CACHE_CPU_MEM_THRESHOLD, CONFIG_CACHE_CPU_MEM_THRESHOLD_DEFAULT); - return SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CPU_MEM_THRESHOLD, value); + SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CPU_MEM_THRESHOLD, value); } + return value; } -Status -Config::GetCacheConfigStrGpuMemCapacity(std::string& value) { - if (GetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_GPU_MEM_CAPACITY, value).ok()) { - return Status::OK(); - } else { +std::string +Config::GetCacheConfigStrGpuMemCapacity() { + std::string value; + if (!GetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_GPU_MEM_CAPACITY, value).ok()) { value = GetConfigNode(CONFIG_CACHE).GetValue(CONFIG_CACHE_GPU_MEM_CAPACITY, CONFIG_CACHE_GPU_MEM_CAPACITY_DEFAULT); - return SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_GPU_MEM_CAPACITY, value); + SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_GPU_MEM_CAPACITY, value); } + return value; } -Status -Config::GetCacheConfigStrGpuMemThreshold(std::string& value) { - if (GetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_GPU_MEM_THRESHOLD, value).ok()) { - return Status::OK(); - } else { +std::string +Config::GetCacheConfigStrGpuMemThreshold() { + std::string value; + if (!GetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_GPU_MEM_THRESHOLD, value).ok()) { value = GetConfigNode(CONFIG_CACHE).GetValue(CONFIG_CACHE_GPU_MEM_THRESHOLD, CONFIG_CACHE_GPU_MEM_THRESHOLD_DEFAULT); - return SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_GPU_MEM_THRESHOLD, value); + SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_GPU_MEM_THRESHOLD, value); } + return value; } -Status -Config::GetCacheConfigStrCacheInsertData(std::string& value) { - if (GetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CACHE_INSERT_DATA, value).ok()) { - return Status::OK(); - } else { +std::string +Config::GetCacheConfigStrCacheInsertData() { + std::string value; + if (!GetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CACHE_INSERT_DATA, value).ok()) { value = GetConfigNode(CONFIG_CACHE).GetValue(CONFIG_CACHE_CACHE_INSERT_DATA, CONFIG_CACHE_CACHE_INSERT_DATA_DEFAULT); - return SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CACHE_INSERT_DATA, value); + SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CACHE_INSERT_DATA, value); } + return value; } //////////////////////////////////////////////////////////////////////////////// /* engine config */ -Status -Config::GetEngineConfigStrBlasThreshold(std::string& value) { - if (GetConfigValueInMem(CONFIG_ENGINE, CONFIG_ENGINE_BLAS_THRESHOLD, value).ok()) { - return Status::OK(); - } else { +std::string +Config::GetEngineConfigStrBlasThreshold() { + std::string value; + if (!GetConfigValueInMem(CONFIG_ENGINE, CONFIG_ENGINE_BLAS_THRESHOLD, value).ok()) { value = GetConfigNode(CONFIG_ENGINE).GetValue(CONFIG_ENGINE_BLAS_THRESHOLD, CONFIG_ENGINE_BLAS_THRESHOLD_DEFAULT); - return SetConfigValueInMem(CONFIG_ENGINE, CONFIG_ENGINE_BLAS_THRESHOLD, value); + SetConfigValueInMem(CONFIG_ENGINE, CONFIG_ENGINE_BLAS_THRESHOLD, value); } + return value; } -Status -Config::GetEngineConfigStrOmpThreadNum(std::string& value) { - if (GetConfigValueInMem(CONFIG_ENGINE, CONFIG_ENGINE_OMP_THREAD_NUM, value).ok()) { - return Status::OK(); - } else { +std::string +Config::GetEngineConfigStrOmpThreadNum() { + std::string value; + if (!GetConfigValueInMem(CONFIG_ENGINE, CONFIG_ENGINE_OMP_THREAD_NUM, value).ok()) { value = GetConfigNode(CONFIG_ENGINE).GetValue(CONFIG_ENGINE_OMP_THREAD_NUM, CONFIG_ENGINE_OMP_THREAD_NUM_DEFAULT); - return SetConfigValueInMem(CONFIG_ENGINE, CONFIG_ENGINE_OMP_THREAD_NUM, value); + SetConfigValueInMem(CONFIG_ENGINE, CONFIG_ENGINE_OMP_THREAD_NUM, value); } + return value; } //////////////////////////////////////////////////////////////////////////////// /* resource config */ -Status -Config::GetResourceConfigStrMode(std::string& value) { - if (GetConfigValueInMem(CONFIG_RESOURCE, CONFIG_RESOURCE_MODE, value).ok()) { - return Status::OK(); - } else { +std::string +Config::GetResourceConfigStrMode() { + std::string value; + if (!GetConfigValueInMem(CONFIG_RESOURCE, CONFIG_RESOURCE_MODE, value).ok()) { value = GetConfigNode(CONFIG_RESOURCE).GetValue(CONFIG_RESOURCE_MODE, CONFIG_RESOURCE_MODE_DEFAULT); - return SetConfigValueInMem(CONFIG_RESOURCE, CONFIG_RESOURCE_MODE, value); + SetConfigValueInMem(CONFIG_RESOURCE, CONFIG_RESOURCE_MODE, value); } + return value; } //////////////////////////////////////////////////////////////////////////////// Status Config::GetServerConfigAddress(std::string& value) { - return GetServerConfigStrAddress(value); + value = GetServerConfigStrAddress(); + return CheckServerConfigAddress(value); } Status Config::GetServerConfigPort(std::string& value) { - return GetServerConfigStrPort(value); + value = GetServerConfigStrPort(); + return CheckServerConfigPort(value); } Status Config::GetServerConfigMode(std::string& value) { - return GetServerConfigStrMode(value); + value = GetServerConfigStrMode(); + return CheckServerConfigMode(value); } Status Config::GetServerConfigTimeZone(std::string& value) { - return GetServerConfigStrTimeZone(value); + value = GetServerConfigStrTimeZone(); + return CheckServerConfigTimeZone(value); } Status Config::GetDBConfigPath(std::string& value) { - return GetDBConfigStrPath(value); + value = GetDBConfigStrPath(); + return CheckDBConfigPath(value); } Status Config::GetDBConfigSlavePath(std::string& value) { - return GetDBConfigStrSlavePath(value); + value = GetDBConfigStrSlavePath(); + return Status::OK(); } Status Config::GetDBConfigBackendUrl(std::string& value) { - return GetDBConfigStrBackendUrl(value); + value = GetDBConfigStrBackendUrl(); + return CheckDBConfigBackendUrl(value); } Status Config::GetDBConfigArchiveDiskThreshold(int32_t& value) { - std::string str; - Status s = GetDBConfigStrArchiveDiskThreshold(str); + std::string str = GetDBConfigStrArchiveDiskThreshold(); + Status s = CheckDBConfigArchiveDiskThreshold(str); if (!s.ok()) return s; value = std::stoi(str); return Status::OK(); @@ -942,8 +674,8 @@ Config::GetDBConfigArchiveDiskThreshold(int32_t& value) { Status Config::GetDBConfigArchiveDaysThreshold(int32_t& value) { - std::string str; - Status s = GetDBConfigStrArchiveDaysThreshold(str); + std::string str = GetDBConfigStrArchiveDaysThreshold(); + Status s = CheckDBConfigArchiveDaysThreshold(str); if (!s.ok()) return s; value = std::stoi(str); return Status::OK(); @@ -951,8 +683,8 @@ Config::GetDBConfigArchiveDaysThreshold(int32_t& value) { Status Config::GetDBConfigBufferSize(int32_t& value) { - std::string str; - Status s = GetDBConfigStrBufferSize(str); + std::string str = GetDBConfigStrBufferSize(); + Status s = CheckDBConfigBufferSize(str); if (!s.ok()) return s; value = std::stoi(str); return Status::OK(); @@ -960,8 +692,8 @@ Config::GetDBConfigBufferSize(int32_t& value) { Status Config::GetDBConfigBuildIndexGPU(int32_t& value) { - std::string str; - Status s = GetDBConfigStrBuildIndexGPU(str); + std::string str = GetDBConfigStrBuildIndexGPU(); + Status s = CheckDBConfigBuildIndexGPU(str); if (!s.ok()) return s; value = std::stoi(str); return Status::OK(); @@ -969,8 +701,8 @@ Config::GetDBConfigBuildIndexGPU(int32_t& value) { Status Config::GetMetricConfigAutoBootup(bool& value) { - std::string str; - Status s = GetMetricConfigStrAutoBootup(str); + std::string str = GetMetricConfigStrAutoBootup(); + Status s = CheckMetricConfigPrometheusPort(str); if (!s.ok()) return s; std::transform(str.begin(), str.end(), str.begin(), ::tolower); value = (str == "true" || str == "on" || str == "yes" || str == "1"); @@ -979,18 +711,20 @@ Config::GetMetricConfigAutoBootup(bool& value) { Status Config::GetMetricConfigCollector(std::string& value) { - return GetMetricConfigStrCollector(value); + value = GetMetricConfigStrCollector(); + return Status::OK(); } Status Config::GetMetricConfigPrometheusPort(std::string& value) { - return GetMetricConfigStrPrometheusPort(value); + value = GetMetricConfigStrPrometheusPort(); + return CheckMetricConfigPrometheusPort(value); } Status Config::GetCacheConfigCpuMemCapacity(int32_t& value) { - std::string str; - Status s = GetCacheConfigStrCpuMemCapacity(str); + std::string str = GetCacheConfigStrCpuMemCapacity(); + Status s = CheckCacheConfigCpuMemCapacity(str); if (!s.ok()) return s; value = std::stoi(str); return Status::OK(); @@ -998,8 +732,8 @@ Config::GetCacheConfigCpuMemCapacity(int32_t& value) { Status Config::GetCacheConfigCpuMemThreshold(float& value) { - std::string str; - Status s = GetCacheConfigStrCpuMemThreshold(str); + std::string str = GetCacheConfigStrCpuMemThreshold(); + Status s = CheckCacheConfigCpuMemThreshold(str); if (!s.ok()) return s; value = std::stof(str); return Status::OK(); @@ -1007,8 +741,8 @@ Config::GetCacheConfigCpuMemThreshold(float& value) { Status Config::GetCacheConfigGpuMemCapacity(int32_t& value) { - std::string str; - Status s = GetCacheConfigStrGpuMemCapacity(str); + std::string str = GetCacheConfigStrGpuMemCapacity(); + Status s = CheckCacheConfigGpuMemCapacity(str); if (!s.ok()) return s; value = std::stoi(str); return Status::OK(); @@ -1016,8 +750,8 @@ Config::GetCacheConfigGpuMemCapacity(int32_t& value) { Status Config::GetCacheConfigGpuMemThreshold(float& value) { - std::string str; - Status s = GetCacheConfigStrGpuMemThreshold(str); + std::string str = GetCacheConfigStrGpuMemThreshold(); + Status s = CheckCacheConfigGpuMemThreshold(str); if (!s.ok()) return s; value = std::stof(str); return Status::OK(); @@ -1025,8 +759,8 @@ Config::GetCacheConfigGpuMemThreshold(float& value) { Status Config::GetCacheConfigCacheInsertData(bool& value) { - std::string str; - Status s = GetCacheConfigStrCacheInsertData(str); + std::string str = GetCacheConfigStrCacheInsertData(); + Status s = CheckCacheConfigCacheInsertData(str); if (!s.ok()) return s; std::transform(str.begin(), str.end(), str.begin(), ::tolower); value = (str == "true" || str == "on" || str == "yes" || str == "1"); @@ -1035,8 +769,8 @@ Config::GetCacheConfigCacheInsertData(bool& value) { Status Config::GetEngineConfigBlasThreshold(int32_t& value) { - std::string str; - Status s = GetEngineConfigStrBlasThreshold(str); + std::string str = GetEngineConfigStrBlasThreshold(); + Status s = CheckEngineConfigBlasThreshold(str); if (!s.ok()) return s; value = std::stoi(str); return Status::OK(); @@ -1044,8 +778,8 @@ Config::GetEngineConfigBlasThreshold(int32_t& value) { Status Config::GetEngineConfigOmpThreadNum(int32_t& value) { - std::string str; - Status s = GetEngineConfigStrOmpThreadNum(str); + std::string str = GetEngineConfigStrOmpThreadNum(); + Status s = CheckEngineConfigOmpThreadNum(str); if (!s.ok()) return s; value = std::stoi(str); return Status::OK(); @@ -1053,14 +787,15 @@ Config::GetEngineConfigOmpThreadNum(int32_t& value) { Status Config::GetResourceConfigMode(std::string& value) { - return GetResourceConfigStrMode(value); + value = GetResourceConfigStrMode(); + return CheckResourceConfigMode(value); } Status Config::GetResourceConfigPool(std::vector& value) { ConfigNode resource_config = GetConfigNode(CONFIG_RESOURCE); value = resource_config.GetSequence(CONFIG_RESOURCE_POOL); - return Status::OK(); + return CheckResourceConfigPool(value); } } diff --git a/cpp/src/server/Config.h b/cpp/src/server/Config.h index 3eeda325..911ec674 100644 --- a/cpp/src/server/Config.h +++ b/cpp/src/server/Config.h @@ -95,64 +95,92 @@ static const char* CONFIG_RESOURCE_POOL = "pool"; class Config { public: - static Config &GetInstance(); - + static Config& GetInstance(); Status LoadConfigFile(const std::string& filename); - Status ValidateConfig(); - void PrintAll() const; + void PrintAll(); private: ConfigNode& GetConfigNode(const std::string& name); - Status CheckServerConfig(); - Status CheckDBConfig(); - Status CheckMetricConfig(); - Status CheckCacheConfig(); - Status CheckEngineConfig(); - Status CheckResourceConfig(); - Status GetConfigValueInMem(const std::string& parent_key, const std::string& child_key, std::string& value); - Status SetConfigValueInMem(const std::string& parent_key, + void SetConfigValueInMem(const std::string& parent_key, const std::string& child_key, std::string& value); - private: + void PrintConfigSection(const std::string& config_node_name); + + /////////////////////////////////////////////////////////////////////////// + /* server config */ + Status CheckServerConfigAddress(std::string& value); + Status CheckServerConfigPort(std::string& value); + Status CheckServerConfigMode(std::string& value); + Status CheckServerConfigTimeZone(std::string& value); + + /* db config */ + Status CheckDBConfigPath(const std::string& value); + Status CheckDBConfigSlavePath(const std::string& value); + Status CheckDBConfigBackendUrl(const std::string& value); + Status CheckDBConfigArchiveDiskThreshold(const std::string& value); + Status CheckDBConfigArchiveDaysThreshold(const std::string& value); + Status CheckDBConfigBufferSize(const std::string& value); + Status CheckDBConfigBuildIndexGPU(const std::string& value); + + /* metric config */ + Status CheckMetricConfigAutoBootup(const std::string& value); + Status CheckMetricConfigPrometheusPort(const std::string& value); + + /* cache config */ + Status CheckCacheConfigCpuMemCapacity(const std::string& value); + Status CheckCacheConfigCpuMemThreshold(const std::string& value); + Status CheckCacheConfigGpuMemCapacity(const std::string& value); + Status CheckCacheConfigGpuMemThreshold(const std::string& value); + Status CheckCacheConfigCacheInsertData(const std::string& value); + + /* engine config */ + Status CheckEngineConfigBlasThreshold(const std::string& value); + Status CheckEngineConfigOmpThreadNum(const std::string& value); + + /* resource config */ + Status CheckResourceConfigMode(const std::string& value); + Status CheckResourceConfigPool(const std::vector& value); + + /////////////////////////////////////////////////////////////////////////// /* server config */ - Status GetServerConfigStrAddress(std::string& value); - Status GetServerConfigStrPort(std::string& value); - Status GetServerConfigStrMode(std::string& value); - Status GetServerConfigStrTimeZone(std::string& value); + std::string GetServerConfigStrAddress(); + std::string GetServerConfigStrPort(); + std::string GetServerConfigStrMode(); + std::string GetServerConfigStrTimeZone(); /* db config */ - Status GetDBConfigStrPath(std::string& value); - Status GetDBConfigStrSlavePath(std::string& value); - Status GetDBConfigStrBackendUrl(std::string& value); - Status GetDBConfigStrArchiveDiskThreshold(std::string& value); - Status GetDBConfigStrArchiveDaysThreshold(std::string& value); - Status GetDBConfigStrBufferSize(std::string& value); - Status GetDBConfigStrBuildIndexGPU(std::string& value); + std::string GetDBConfigStrPath(); + std::string GetDBConfigStrSlavePath(); + std::string GetDBConfigStrBackendUrl(); + std::string GetDBConfigStrArchiveDiskThreshold(); + std::string GetDBConfigStrArchiveDaysThreshold(); + std::string GetDBConfigStrBufferSize(); + std::string GetDBConfigStrBuildIndexGPU(); /* metric config */ - Status GetMetricConfigStrAutoBootup(std::string& value); - Status GetMetricConfigStrCollector(std::string& value); - Status GetMetricConfigStrPrometheusPort(std::string& value); + std::string GetMetricConfigStrAutoBootup(); + std::string GetMetricConfigStrCollector(); + std::string GetMetricConfigStrPrometheusPort(); /* cache config */ - Status GetCacheConfigStrCpuMemCapacity(std::string& value); - Status GetCacheConfigStrCpuMemThreshold(std::string& value); - Status GetCacheConfigStrGpuMemCapacity(std::string& value); - Status GetCacheConfigStrGpuMemThreshold(std::string& value); - Status GetCacheConfigStrCacheInsertData(std::string& value); + std::string GetCacheConfigStrCpuMemCapacity(); + std::string GetCacheConfigStrCpuMemThreshold(); + std::string GetCacheConfigStrGpuMemCapacity(); + std::string GetCacheConfigStrGpuMemThreshold(); + std::string GetCacheConfigStrCacheInsertData(); /* engine config */ - Status GetEngineConfigStrBlasThreshold(std::string& value); - Status GetEngineConfigStrOmpThreadNum(std::string& value); + std::string GetEngineConfigStrBlasThreshold(); + std::string GetEngineConfigStrOmpThreadNum(); /* resource config */ - Status GetResourceConfigStrMode(std::string& value); + std::string GetResourceConfigStrMode(); public: /* server config */ diff --git a/cpp/src/server/Server.cpp b/cpp/src/server/Server.cpp index d926ca5d..355f7903 100644 --- a/cpp/src/server/Server.cpp +++ b/cpp/src/server/Server.cpp @@ -235,14 +235,12 @@ Server::Stop() { ErrorCode Server::LoadConfig() { - Config& server_config = Config::GetInstance(); - server_config.LoadConfigFile(config_filename_); - auto status = server_config.ValidateConfig(); - if (!status.ok()) { + Config& config = Config::GetInstance(); + Status s = config.LoadConfigFile(config_filename_); + if (!s.ok()) { std::cerr << "Failed to load config file: " << config_filename_ << std::endl; exit(0); } - return SERVER_SUCCESS; } diff --git a/cpp/src/utils/ValidationUtil.cpp b/cpp/src/utils/ValidationUtil.cpp index c993a4ae..c9ec2d04 100644 --- a/cpp/src/utils/ValidationUtil.cpp +++ b/cpp/src/utils/ValidationUtil.cpp @@ -206,38 +206,40 @@ ValidationUtil::ValidateIpAddress(const std::string &ip_address) { } Status -ValidationUtil::ValidateStringIsNumber(const std::string &string) { - if (!string.empty() && std::all_of(string.begin(), string.end(), ::isdigit)) { - return Status::OK(); +ValidationUtil::ValidateStringIsNumber(const std::string &str) { + if (str.empty() || !std::all_of(str.begin(), str.end(), ::isdigit)) { + return Status(SERVER_INVALID_ARGUMENT, "Invalid number"); } - else { - return Status(SERVER_INVALID_ARGUMENT, "Not a number"); + try { + int32_t value = std::stoi(str); + } catch(...) { + return Status(SERVER_INVALID_ARGUMENT, "Invalid number"); } + return Status::OK(); } Status -ValidationUtil::ValidateStringIsBool(std::string &str) { - std::transform(str.begin(), str.end(), str.begin(), ::tolower); - if (str == "true" || str == "on" || str == "yes" || str == "1" || - str == "false" || str == "off" || str == "no" || str == "0" || - str.empty()) { +ValidationUtil::ValidateStringIsBool(const std::string &str) { + std::string s = str; + std::transform(s.begin(), s.end(), s.begin(), ::tolower); + if (s == "true" || s == "on" || s == "yes" || s == "1" || + s == "false" || s == "off" || s == "no" || s == "0" || + s.empty()) { return Status::OK(); } else { - return Status(SERVER_INVALID_ARGUMENT, "Not a boolean: " + str); + return Status(SERVER_INVALID_ARGUMENT, "Invalid boolean: " + str); } } Status -ValidationUtil::ValidateStringIsDouble(const std::string &str, double &val) { - char *end = nullptr; - val = std::strtod(str.c_str(), &end); - if (end != str.c_str() && *end == '\0' && val != HUGE_VAL) { - return Status::OK(); - } - else { - return Status(SERVER_INVALID_ARGUMENT, "Not a double value: " + str); +ValidationUtil::ValidateStringIsFloat(const std::string &str) { + try { + float val = std::stof(str); + } catch(...) { + return Status(SERVER_INVALID_ARGUMENT, "Invalid float: " + str); } + return Status::OK(); } Status diff --git a/cpp/src/utils/ValidationUtil.h b/cpp/src/utils/ValidationUtil.h index ad8ba5fe..789da0de 100644 --- a/cpp/src/utils/ValidationUtil.h +++ b/cpp/src/utils/ValidationUtil.h @@ -67,10 +67,10 @@ public: ValidateStringIsNumber(const std::string &str); static Status - ValidateStringIsBool(std::string &str); + ValidateStringIsBool(const std::string &str); static Status - ValidateStringIsDouble(const std::string &str, double &val); + ValidateStringIsFloat(const std::string &str); static Status ValidateDbURI(const std::string &uri); -- GitLab