diff --git a/cpp/conf/server_config.template b/cpp/conf/server_config.template index 6364698100e43de7dda50b3a5da74565e1656e02..d80dfda958ae4abb2c675015a97ffbeb113a445f 100644 --- a/cpp/conf/server_config.template +++ b/cpp/conf/server_config.template @@ -1,39 +1,37 @@ +# All the following configurations are default values. + server_config: - address: 0.0.0.0 # milvus server ip address (IPv4) - port: 19530 # port range: 1025 ~ 65534 - mode: single # deployment type: single, cluster, read_only + address: 0.0.0.0 # milvus server ip address (IPv4) + port: 19530 # port range: 1025 ~ 65534 + deploy_mode: single # deployment type: single, cluster_readonly, cluster_writable time_zone: UTC+8 db_config: - path: @MILVUS_DB_PATH@ # milvus database path - slave_path: # secondary database path, split by semicolon + primary_path: @MILVUS_DB_PATH@ # path used to store data and meta + secondary_path: # path used to store data only, split by semicolon - # URI format: dialect://username:password@host:port/database - # All parts except dialect are optional, but you MUST include the delimiters - # Currently dialect supports mysql or sqlite - backend_url: sqlite://:@:/ + backend_url: sqlite://:@:/ # URI format: dialect://username:password@host:port/database + # Keep 'dialect://:@:/', and replace other texts with real values. + # Replace 'dialect' with 'mysql' or 'sqlite' - archive_disk_threshold: 0 # GB, file will be archived when disk usage exceed, 0 for no limit - archive_days_threshold: 0 # DAYS, older files will be archived, 0 for no limit - buffer_size: 4 # GB, maximum insert buffer size allowed - build_index_gpu: 0 # gpu id used for building index + insert_buffer_size: 4 # GB, maximum insert buffer size allowed + build_index_gpu: 0 # gpu id used for building index metric_config: - auto_bootup: off # whether enable monitoring when bootup - collector: prometheus # prometheus + enable_monitor: false # enable monitoring or not + collector: prometheus # prometheus prometheus_config: - port: 8080 # port prometheus used to fetch metrics + port: 8080 # port prometheus used to fetch metrics cache_config: - cpu_mem_capacity: 16 # GB, CPU memory size used for cache - cpu_mem_threshold: 0.85 # percent of data kept when cache cleanup triggered - cache_insert_data: false # whether load data into cache when insert + cpu_mem_capacity: 16 # GB, CPU memory used for cache + cpu_mem_threshold: 0.85 # percentage of data kept when cache cleanup triggered + cache_insert_data: false # whether load inserted data into cache engine_config: blas_threshold: 20 resource_config: - mode: simple - pool: + resource_pool: - cpu - gpu0 diff --git a/cpp/src/db/DBImpl.cpp b/cpp/src/db/DBImpl.cpp index a4945ca02e9c4d97fa067347efc3e1d9d7b047b8..4536e29e9508b9e7b84bebbabcf5542c07e5239b 100644 --- a/cpp/src/db/DBImpl.cpp +++ b/cpp/src/db/DBImpl.cpp @@ -77,7 +77,7 @@ Status DBImpl::Start() { shutting_down_.store(false, std::memory_order_release); //for distribute version, some nodes are read only - if (options_.mode_ != DBOptions::MODE::READ_ONLY) { + if (options_.mode_ != DBOptions::MODE::CLUSTER_READONLY) { ENGINE_LOG_TRACE << "StartTimerTasks"; bg_timer_thread_ = std::thread(&DBImpl::BackgroundTimerTask, this); } @@ -98,7 +98,7 @@ Status DBImpl::Stop() { //wait compaction/buildindex finish bg_timer_thread_.join(); - if (options_.mode_ != DBOptions::MODE::READ_ONLY) { + if (options_.mode_ != DBOptions::MODE::CLUSTER_READONLY) { meta_ptr_->CleanUp(); } @@ -704,7 +704,7 @@ void DBImpl::BackgroundCompaction(std::set table_ids) { meta_ptr_->Archive(); int ttl = 5*meta::M_SEC;//default: file will be deleted after 5 minutes - if (options_.mode_ == DBOptions::MODE::CLUSTER) { + if (options_.mode_ == DBOptions::MODE::CLUSTER_WRITABLE) { ttl = meta::D_SEC; } meta_ptr_->CleanUpFilesWithTTL(ttl); diff --git a/cpp/src/db/Options.h b/cpp/src/db/Options.h index 2ba8c36a328bb86f63502b106ca9a5293cd6731c..dbaf34b065ab457f61c3a7ba927727c16e950786 100644 --- a/cpp/src/db/Options.h +++ b/cpp/src/db/Options.h @@ -60,9 +60,9 @@ struct DBMetaOptions { struct DBOptions { typedef enum { - SINGLE, - CLUSTER, - READ_ONLY + SINGLE = 0, + CLUSTER_READONLY, + CLUSTER_WRITABLE } MODE; uint16_t merge_trigger_number_ = 2; diff --git a/cpp/src/db/meta/MySQLMetaImpl.cpp b/cpp/src/db/meta/MySQLMetaImpl.cpp index 48302fa099a5593732b3f7bcdb3cf5b06751312c..c1c55febf0328162e8124b6749e7412e3ebc1756 100644 --- a/cpp/src/db/meta/MySQLMetaImpl.cpp +++ b/cpp/src/db/meta/MySQLMetaImpl.cpp @@ -284,7 +284,7 @@ Status MySQLMetaImpl::Initialize() { //step 5: create meta tables try { - if (mode_ != DBOptions::MODE::READ_ONLY) { + if (mode_ != DBOptions::MODE::CLUSTER_READONLY) { CleanUp(); } @@ -768,7 +768,7 @@ Status MySQLMetaImpl::DeleteTable(const std::string &table_id) { } //Scoped Connection - if (mode_ == DBOptions::MODE::CLUSTER) { + if (mode_ == DBOptions::MODE::CLUSTER_WRITABLE) { DeleteTableFiles(table_id); } diff --git a/cpp/src/metrics/PrometheusMetrics.cpp b/cpp/src/metrics/PrometheusMetrics.cpp index b607dbb020f885ddbeef77fc713d1aefb0bd15a4..6f3513072a412a79173c7da39396fee946fd4760 100644 --- a/cpp/src/metrics/PrometheusMetrics.cpp +++ b/cpp/src/metrics/PrometheusMetrics.cpp @@ -31,7 +31,7 @@ ErrorCode PrometheusMetrics::Init() { try { Config &config = Config::GetInstance(); - Status s = config.GetMetricConfigAutoBootup(startup_); + Status s = config.GetMetricConfigEnableMonitor(startup_); if (!s.ok()) return s.code(); if (!startup_) return SERVER_SUCCESS; diff --git a/cpp/src/server/Config.cpp b/cpp/src/server/Config.cpp index 0dbf4e1dd26f5e19a215449f3f0ee4b079cf2968..34d1fab7b394040e3b5ccbccaf62047e727ba996 100644 --- a/cpp/src/server/Config.cpp +++ b/cpp/src/server/Config.cpp @@ -71,6 +71,111 @@ Config::LoadConfigFile(const std::string &filename) { return Status::OK(); } +Status +Config::ValidateConfig() { + Status s; + + /* server config */ + std::string server_addr; + s = GetServerConfigAddress(server_addr); + if (!s.ok()) return s; + + std::string server_port; + s = GetServerConfigPort(server_port); + if (!s.ok()) return s; + + std::string server_mode; + s = GetServerConfigDeployMode(server_mode); + if (!s.ok()) return s; + + std::string server_time_zone; + s = GetServerConfigTimeZone(server_time_zone); + if (!s.ok()) return s; + + /* db config */ + std::string db_primary_path; + s = GetDBConfigPrimaryPath(db_primary_path); + if (!s.ok()) return s; + + std::string db_secondary_path; + s = GetDBConfigSecondaryPath(db_secondary_path); + if (!s.ok()) return s; + + std::string db_backend_url; + s = GetDBConfigBackendUrl(db_backend_url); + if (!s.ok()) return s; + + int32_t db_archive_disk_threshold; + s = GetDBConfigArchiveDiskThreshold(db_archive_disk_threshold); + if (!s.ok()) return s; + + int32_t db_archive_days_threshold; + s = GetDBConfigArchiveDaysThreshold(db_archive_days_threshold); + if (!s.ok()) return s; + + int32_t db_insert_buffer_size; + s = GetDBConfigInsertBufferSize(db_insert_buffer_size); + if (!s.ok()) return s; + + int32_t db_build_index_gpu; + s = GetDBConfigBuildIndexGPU(db_build_index_gpu); + if (!s.ok()) return s; + + /* metric config */ + bool metric_enable_monitor; + s = GetMetricConfigEnableMonitor(metric_enable_monitor); + if (!s.ok()) return s; + + std::string metric_collector; + s = GetMetricConfigCollector(metric_collector); + if (!s.ok()) return s; + + std::string metric_prometheus_port; + s = GetMetricConfigPrometheusPort(metric_prometheus_port); + if (!s.ok()) return s; + + /* cache config */ + int32_t cache_cpu_mem_capacity; + s = GetCacheConfigCpuMemCapacity(cache_cpu_mem_capacity); + if (!s.ok()) return s; + + float cache_cpu_mem_threshold; + s = GetCacheConfigCpuMemThreshold(cache_cpu_mem_threshold); + if (!s.ok()) return s; + + int32_t cache_gpu_mem_capacity; + s = GetCacheConfigGpuMemCapacity(cache_gpu_mem_capacity); + if (!s.ok()) return s; + + float cache_gpu_mem_threshold; + s = GetCacheConfigGpuMemThreshold(cache_gpu_mem_threshold); + if (!s.ok()) return s; + + bool cache_insert_data; + s = GetCacheConfigCacheInsertData(cache_insert_data); + if (!s.ok()) return s; + + /* engine config */ + int32_t engine_blas_threshold; + s = GetEngineConfigBlasThreshold(engine_blas_threshold); + if (!s.ok()) return s; + + int32_t engine_omp_thread_num; + s = GetEngineConfigOmpThreadNum(engine_omp_thread_num); + if (!s.ok()) return s; + + /* resource config */ + std::string resource_mode; + s = GetResourceConfigMode(resource_mode); + if (!s.ok()) return s; + + std::vector resource_pool; + s = GetResourceConfigPool(resource_pool); + if (!s.ok()) return s; + + return Status::OK(); +} + void Config::PrintConfigSection(const std::string &config_node_name) { std::cout << std::endl; @@ -115,9 +220,12 @@ Config::CheckServerConfigPort(const std::string &value) { } Status -Config::CheckServerConfigMode(const std::string &value) { - if (value != "single" && value != "cluster" && value != "read_only") { - return Status(SERVER_INVALID_ARGUMENT, "Invalid server config mode [single, cluster, read_only]: " + value); +Config::CheckServerConfigDeployMode(const std::string &value) { + if (value != "single" && + value != "cluster_readonly" && + value != "cluster_writable") { + return Status(SERVER_INVALID_ARGUMENT, + "Invalid server config mode [single, cluster_readonly, cluster_writable]: " + value); } return Status::OK(); } @@ -141,15 +249,15 @@ Config::CheckServerConfigTimeZone(const std::string &value) { } Status -Config::CheckDBConfigPath(const std::string &value) { +Config::CheckDBConfigPrimaryPath(const std::string &value) { if (value.empty()) { - return Status(SERVER_INVALID_ARGUMENT, "DB config path empty"); + return Status(SERVER_INVALID_ARGUMENT, "DB config primary_path empty"); } return Status::OK(); } Status -Config::CheckDBConfigSlavePath(const std::string &value) { +Config::CheckDBConfigSecondaryPath(const std::string &value) { return Status::OK(); } @@ -178,15 +286,15 @@ Config::CheckDBConfigArchiveDaysThreshold(const std::string &value) { } Status -Config::CheckDBConfigBufferSize(const std::string &value) { +Config::CheckDBConfigInsertBufferSize(const std::string &value) { if (!ValidationUtil::ValidateStringIsNumber(value).ok()) { - return Status(SERVER_INVALID_ARGUMENT, "Invalid DB config buffer_size: " + value); + return Status(SERVER_INVALID_ARGUMENT, "Invalid DB config insert_buffer_size: " + value); } else { int64_t buffer_size = std::stoi(value) * GB; uint64_t total_mem = 0, free_mem = 0; CommonUtil::GetSystemMemInfo(total_mem, free_mem); if (buffer_size >= total_mem) { - return Status(SERVER_INVALID_ARGUMENT, "DB config buffer_size exceed system memory: " + value); + return Status(SERVER_INVALID_ARGUMENT, "DB config insert_buffer_size exceed system memory: " + value); } } return Status::OK(); @@ -206,7 +314,7 @@ Config::CheckDBConfigBuildIndexGPU(const std::string &value) { } Status -Config::CheckMetricConfigAutoBootup(const std::string &value) { +Config::CheckMetricConfigEnableMonitor(const std::string &value) { if (!ValidationUtil::ValidateStringIsBool(value).ok()) { return Status(SERVER_INVALID_ARGUMENT, "Invalid metric config auto_bootup: " + value); } @@ -243,10 +351,10 @@ Config::CheckCacheConfigCpuMemCapacity(const std::string &value) { std::cerr << "Warning: cpu_mem_capacity value is too big" << std::endl; } - int32_t buffer_size; - Status s = GetDBConfigBufferSize(buffer_size); + int32_t buffer_value; + Status s = GetDBConfigInsertBufferSize(buffer_value); if (!s.ok()) return s; - int64_t insert_buffer_size = buffer_size * GB; + int64_t insert_buffer_size = buffer_value * GB; if (insert_buffer_size + cpu_cache_capacity >= total_mem) { return Status(SERVER_INVALID_ARGUMENT, "Sum of cpu_mem_capacity and buffer_size exceed system memory"); } @@ -404,12 +512,12 @@ Config::GetServerConfigStrPort() { } std::string -Config::GetServerConfigStrMode() { +Config::GetServerConfigStrDeployMode() { std::string value; - if (!GetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_MODE, value).ok()) { - value = GetConfigNode(CONFIG_SERVER).GetValue(CONFIG_SERVER_MODE, - CONFIG_SERVER_MODE_DEFAULT); - SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_MODE, value); + if (!GetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_DEPLOY_MODE, value).ok()) { + value = GetConfigNode(CONFIG_SERVER).GetValue(CONFIG_SERVER_DEPLOY_MODE, + CONFIG_SERVER_DEPLOY_MODE_DEFAULT); + SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_DEPLOY_MODE, value); } return value; } @@ -428,23 +536,23 @@ Config::GetServerConfigStrTimeZone() { //////////////////////////////////////////////////////////////////////////////// /* db config */ std::string -Config::GetDBConfigStrPath() { +Config::GetDBConfigStrPrimaryPath() { std::string value; - if (!GetConfigValueInMem(CONFIG_DB, CONFIG_DB_PATH, value).ok()) { - value = GetConfigNode(CONFIG_DB).GetValue(CONFIG_DB_PATH, - CONFIG_DB_PATH_DEFAULT); - SetConfigValueInMem(CONFIG_DB, CONFIG_DB_PATH, value); + if (!GetConfigValueInMem(CONFIG_DB, CONFIG_DB_PRIMARY_PATH, value).ok()) { + value = GetConfigNode(CONFIG_DB).GetValue(CONFIG_DB_PRIMARY_PATH, + CONFIG_DB_PRIMARY_PATH_DEFAULT); + SetConfigValueInMem(CONFIG_DB, CONFIG_DB_PRIMARY_PATH, value); } return value; } std::string -Config::GetDBConfigStrSlavePath() { +Config::GetDBConfigStrSecondaryPath() { std::string value; - if (!GetConfigValueInMem(CONFIG_DB, CONFIG_DB_SLAVE_PATH, value).ok()) { - value = GetConfigNode(CONFIG_DB).GetValue(CONFIG_DB_SLAVE_PATH, - CONFIG_DB_SLAVE_PATH_DEFAULT); - SetConfigValueInMem(CONFIG_DB, CONFIG_DB_SLAVE_PATH, value); + if (!GetConfigValueInMem(CONFIG_DB, CONFIG_DB_SECONDARY_PATH, value).ok()) { + value = GetConfigNode(CONFIG_DB).GetValue(CONFIG_DB_SECONDARY_PATH, + CONFIG_DB_SECONDARY_PATH_DEFAULT); + SetConfigValueInMem(CONFIG_DB, CONFIG_DB_SECONDARY_PATH, value); } return value; } @@ -483,12 +591,12 @@ Config::GetDBConfigStrArchiveDaysThreshold() { } std::string -Config::GetDBConfigStrBufferSize() { +Config::GetDBConfigStrInsertBufferSize() { std::string value; - if (!GetConfigValueInMem(CONFIG_DB, CONFIG_DB_BUFFER_SIZE, value).ok()) { - value = GetConfigNode(CONFIG_DB).GetValue(CONFIG_DB_BUFFER_SIZE, - CONFIG_DB_BUFFER_SIZE_DEFAULT); - SetConfigValueInMem(CONFIG_DB, CONFIG_DB_BUFFER_SIZE, value); + if (!GetConfigValueInMem(CONFIG_DB, CONFIG_DB_INSERT_BUFFER_SIZE, value).ok()) { + value = GetConfigNode(CONFIG_DB).GetValue(CONFIG_DB_INSERT_BUFFER_SIZE, + CONFIG_DB_INSERT_BUFFER_SIZE_DEFAULT); + SetConfigValueInMem(CONFIG_DB, CONFIG_DB_INSERT_BUFFER_SIZE, value); } return value; } @@ -507,12 +615,12 @@ Config::GetDBConfigStrBuildIndexGPU() { //////////////////////////////////////////////////////////////////////////////// /* metric config */ std::string -Config::GetMetricConfigStrAutoBootup() { +Config::GetMetricConfigStrEnableMonitor() { std::string value; - if (!GetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_AUTO_BOOTUP, value).ok()) { - value = GetConfigNode(CONFIG_METRIC).GetValue(CONFIG_METRIC_AUTO_BOOTUP, - CONFIG_METRIC_AUTO_BOOTUP_DEFAULT); - SetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_AUTO_BOOTUP, value); + if (!GetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_ENABLE_MONITOR, value).ok()) { + value = GetConfigNode(CONFIG_METRIC).GetValue(CONFIG_METRIC_ENABLE_MONITOR, + CONFIG_METRIC_ENABLE_MONITOR_DEFAULT); + SetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_ENABLE_MONITOR, value); } return value; } @@ -647,9 +755,9 @@ Config::GetServerConfigPort(std::string &value) { } Status -Config::GetServerConfigMode(std::string &value) { - value = GetServerConfigStrMode(); - return CheckServerConfigMode(value); +Config::GetServerConfigDeployMode(std::string &value) { + value = GetServerConfigStrDeployMode(); + return CheckServerConfigDeployMode(value); } Status @@ -659,14 +767,14 @@ Config::GetServerConfigTimeZone(std::string &value) { } Status -Config::GetDBConfigPath(std::string &value) { - value = GetDBConfigStrPath(); - return CheckDBConfigPath(value); +Config::GetDBConfigPrimaryPath(std::string &value) { + value = GetDBConfigStrPrimaryPath(); + return CheckDBConfigPrimaryPath(value); } Status -Config::GetDBConfigSlavePath(std::string &value) { - value = GetDBConfigStrSlavePath(); +Config::GetDBConfigSecondaryPath(std::string &value) { + value = GetDBConfigStrSecondaryPath(); return Status::OK(); } @@ -695,9 +803,9 @@ Config::GetDBConfigArchiveDaysThreshold(int32_t &value) { } Status -Config::GetDBConfigBufferSize(int32_t &value) { - std::string str = GetDBConfigStrBufferSize(); - Status s = CheckDBConfigBufferSize(str); +Config::GetDBConfigInsertBufferSize(int32_t &value) { + std::string str = GetDBConfigStrInsertBufferSize(); + Status s = CheckDBConfigInsertBufferSize(str); if (!s.ok()) return s; value = std::stoi(str); return Status::OK(); @@ -713,9 +821,9 @@ Config::GetDBConfigBuildIndexGPU(int32_t &value) { } Status -Config::GetMetricConfigAutoBootup(bool &value) { - std::string str = GetMetricConfigStrAutoBootup(); - Status s = CheckMetricConfigPrometheusPort(str); +Config::GetMetricConfigEnableMonitor(bool &value) { + std::string str = GetMetricConfigStrEnableMonitor(); + Status s = CheckMetricConfigEnableMonitor(str); if (!s.ok()) return s; std::transform(str.begin(), str.end(), str.begin(), ::tolower); value = (str == "true" || str == "on" || str == "yes" || str == "1"); @@ -830,10 +938,10 @@ Config::SetServerConfigPort(const std::string &value) { } Status -Config::SetServerConfigMode(const std::string &value) { - Status s = CheckServerConfigMode(value); +Config::SetServerConfigDeployMode(const std::string &value) { + Status s = CheckServerConfigDeployMode(value); if (!s.ok()) return s; - SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_MODE, value); + SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_DEPLOY_MODE, value); return Status::OK(); } @@ -847,18 +955,18 @@ Config::SetServerConfigTimeZone(const std::string &value) { /* db config */ Status -Config::SetDBConfigPath(const std::string &value) { - Status s = CheckDBConfigPath(value); +Config::SetDBConfigPrimaryPath(const std::string &value) { + Status s = CheckDBConfigPrimaryPath(value); if (!s.ok()) return s; - SetConfigValueInMem(CONFIG_DB, CONFIG_DB_PATH, value); + SetConfigValueInMem(CONFIG_DB, CONFIG_DB_PRIMARY_PATH, value); return Status::OK(); } Status -Config::SetDBConfigSlavePath(const std::string &value) { - Status s = CheckDBConfigSlavePath(value); +Config::SetDBConfigSecondaryPath(const std::string &value) { + Status s = CheckDBConfigSecondaryPath(value); if (!s.ok()) return s; - SetConfigValueInMem(CONFIG_DB, CONFIG_DB_SLAVE_PATH, value); + SetConfigValueInMem(CONFIG_DB, CONFIG_DB_SECONDARY_PATH, value); return Status::OK(); } @@ -887,10 +995,10 @@ Config::SetDBConfigArchiveDaysThreshold(const std::string &value) { } Status -Config::SetDBConfigBufferSize(const std::string &value) { - Status s = CheckDBConfigBufferSize(value); +Config::SetDBConfigInsertBufferSize(const std::string &value) { + Status s = CheckDBConfigInsertBufferSize(value); if (!s.ok()) return s; - SetConfigValueInMem(CONFIG_DB, CONFIG_DB_BUFFER_SIZE, value); + SetConfigValueInMem(CONFIG_DB, CONFIG_DB_INSERT_BUFFER_SIZE, value); return Status::OK(); } @@ -904,10 +1012,10 @@ Config::SetDBConfigBuildIndexGPU(const std::string &value) { /* metric config */ Status -Config::SetMetricConfigAutoBootup(const std::string &value) { - Status s = CheckMetricConfigAutoBootup(value); +Config::SetMetricConfigEnableMonitor(const std::string &value) { + Status s = CheckMetricConfigEnableMonitor(value); if (!s.ok()) return s; - SetConfigValueInMem(CONFIG_DB, CONFIG_METRIC_AUTO_BOOTUP, value); + SetConfigValueInMem(CONFIG_DB, CONFIG_METRIC_ENABLE_MONITOR, value); return Status::OK(); } @@ -997,3 +1105,4 @@ Config::SetResourceConfigMode(const std::string &value) { } // namespace server } // namespace milvus } // namespace zilliz + diff --git a/cpp/src/server/Config.h b/cpp/src/server/Config.h index f2d1b96999f3bea66dc7904582e0c3c34e85bb30..b1de101bd153caf3b37ed877d9a3be2d92643d75 100644 --- a/cpp/src/server/Config.h +++ b/cpp/src/server/Config.h @@ -36,25 +36,25 @@ static const char *CONFIG_SERVER_ADDRESS = "address"; static const char *CONFIG_SERVER_ADDRESS_DEFAULT = "127.0.0.1"; static const char *CONFIG_SERVER_PORT = "port"; static const char *CONFIG_SERVER_PORT_DEFAULT = "19530"; -static const char *CONFIG_SERVER_MODE = "mode"; -static const char *CONFIG_SERVER_MODE_DEFAULT = "single"; +static const char *CONFIG_SERVER_DEPLOY_MODE = "deploy_mode"; +static const char *CONFIG_SERVER_DEPLOY_MODE_DEFAULT = "single"; static const char *CONFIG_SERVER_TIME_ZONE = "time_zone"; static const char *CONFIG_SERVER_TIME_ZONE_DEFAULT = "UTC+8"; /* db config */ static const char *CONFIG_DB = "db_config"; -static const char *CONFIG_DB_PATH = "path"; -static const char *CONFIG_DB_PATH_DEFAULT = "/tmp/milvus"; -static const char *CONFIG_DB_SLAVE_PATH = "slave_path"; -static const char *CONFIG_DB_SLAVE_PATH_DEFAULT = ""; +static const char *CONFIG_DB_PRIMARY_PATH = "primary_path"; +static const char *CONFIG_DB_PRIMARY_PATH_DEFAULT = "/tmp/milvus"; +static const char *CONFIG_DB_SECONDARY_PATH = "secondary_path"; +static const char *CONFIG_DB_SECONDARY_PATH_DEFAULT = ""; static const char *CONFIG_DB_BACKEND_URL = "backend_url"; static const char *CONFIG_DB_BACKEND_URL_DEFAULT = "sqlite://:@:/"; static const char *CONFIG_DB_ARCHIVE_DISK_THRESHOLD = "archive_disk_threshold"; static const char *CONFIG_DB_ARCHIVE_DISK_THRESHOLD_DEFAULT = "0"; static const char *CONFIG_DB_ARCHIVE_DAYS_THRESHOLD = "archive_days_threshold"; static const char *CONFIG_DB_ARCHIVE_DAYS_THRESHOLD_DEFAULT = "0"; -static const char *CONFIG_DB_BUFFER_SIZE = "buffer_size"; -static const char *CONFIG_DB_BUFFER_SIZE_DEFAULT = "4"; +static const char *CONFIG_DB_INSERT_BUFFER_SIZE = "insert_buffer_size"; +static const char *CONFIG_DB_INSERT_BUFFER_SIZE_DEFAULT = "4"; static const char *CONFIG_DB_BUILD_INDEX_GPU = "build_index_gpu"; static const char *CONFIG_DB_BUILD_INDEX_GPU_DEFAULT = "0"; @@ -73,8 +73,8 @@ static const char *CONFIG_CACHE_CACHE_INSERT_DATA_DEFAULT = "false"; /* metric config */ static const char *CONFIG_METRIC = "metric_config"; -static const char *CONFIG_METRIC_AUTO_BOOTUP = "auto_bootup"; -static const char *CONFIG_METRIC_AUTO_BOOTUP_DEFAULT = "false"; +static const char *CONFIG_METRIC_ENABLE_MONITOR = "enable_monitor"; +static const char *CONFIG_METRIC_ENABLE_MONITOR_DEFAULT = "false"; static const char *CONFIG_METRIC_COLLECTOR = "collector"; static const char *CONFIG_METRIC_COLLECTOR_DEFAULT = "prometheus"; static const char *CONFIG_METRIC_PROMETHEUS = "prometheus_config"; @@ -92,12 +92,13 @@ static const char *CONFIG_ENGINE_OMP_THREAD_NUM_DEFAULT = "0"; static const char *CONFIG_RESOURCE = "resource_config"; static const char *CONFIG_RESOURCE_MODE = "mode"; static const char *CONFIG_RESOURCE_MODE_DEFAULT = "simple"; -static const char *CONFIG_RESOURCE_POOL = "pool"; +static const char *CONFIG_RESOURCE_POOL = "resource_pool"; class Config { public: static Config &GetInstance(); Status LoadConfigFile(const std::string &filename); + Status ValidateConfig(); void PrintAll(); private: @@ -117,20 +118,20 @@ class Config { /* server config */ Status CheckServerConfigAddress(const std::string &value); Status CheckServerConfigPort(const std::string &value); - Status CheckServerConfigMode(const std::string &value); + Status CheckServerConfigDeployMode(const std::string &value); Status CheckServerConfigTimeZone(const std::string &value); /* db config */ - Status CheckDBConfigPath(const std::string &value); - Status CheckDBConfigSlavePath(const std::string &value); + Status CheckDBConfigPrimaryPath(const std::string &value); + Status CheckDBConfigSecondaryPath(const std::string &value); Status CheckDBConfigBackendUrl(const std::string &value); Status CheckDBConfigArchiveDiskThreshold(const std::string &value); Status CheckDBConfigArchiveDaysThreshold(const std::string &value); - Status CheckDBConfigBufferSize(const std::string &value); + Status CheckDBConfigInsertBufferSize(const std::string &value); Status CheckDBConfigBuildIndexGPU(const std::string &value); /* metric config */ - Status CheckMetricConfigAutoBootup(const std::string &value); + Status CheckMetricConfigEnableMonitor(const std::string &value); Status CheckMetricConfigCollector(const std::string &value); Status CheckMetricConfigPrometheusPort(const std::string &value); @@ -153,20 +154,20 @@ class Config { /* server config */ std::string GetServerConfigStrAddress(); std::string GetServerConfigStrPort(); - std::string GetServerConfigStrMode(); + std::string GetServerConfigStrDeployMode(); std::string GetServerConfigStrTimeZone(); /* db config */ - std::string GetDBConfigStrPath(); - std::string GetDBConfigStrSlavePath(); + std::string GetDBConfigStrPrimaryPath(); + std::string GetDBConfigStrSecondaryPath(); std::string GetDBConfigStrBackendUrl(); std::string GetDBConfigStrArchiveDiskThreshold(); std::string GetDBConfigStrArchiveDaysThreshold(); - std::string GetDBConfigStrBufferSize(); + std::string GetDBConfigStrInsertBufferSize(); std::string GetDBConfigStrBuildIndexGPU(); /* metric config */ - std::string GetMetricConfigStrAutoBootup(); + std::string GetMetricConfigStrEnableMonitor(); std::string GetMetricConfigStrCollector(); std::string GetMetricConfigStrPrometheusPort(); @@ -188,20 +189,20 @@ class Config { /* server config */ Status GetServerConfigAddress(std::string &value); Status GetServerConfigPort(std::string &value); - Status GetServerConfigMode(std::string &value); + Status GetServerConfigDeployMode(std::string &value); Status GetServerConfigTimeZone(std::string &value); /* db config */ - Status GetDBConfigPath(std::string &value); - Status GetDBConfigSlavePath(std::string &value); + Status GetDBConfigPrimaryPath(std::string &value); + Status GetDBConfigSecondaryPath(std::string &value); Status GetDBConfigBackendUrl(std::string &value); Status GetDBConfigArchiveDiskThreshold(int32_t &value); Status GetDBConfigArchiveDaysThreshold(int32_t &value); - Status GetDBConfigBufferSize(int32_t &value); + Status GetDBConfigInsertBufferSize(int32_t &value); Status GetDBConfigBuildIndexGPU(int32_t &value); /* metric config */ - Status GetMetricConfigAutoBootup(bool &value); + Status GetMetricConfigEnableMonitor(bool &value); Status GetMetricConfigCollector(std::string &value); Status GetMetricConfigPrometheusPort(std::string &value); @@ -224,20 +225,20 @@ class Config { /* server config */ Status SetServerConfigAddress(const std::string &value); Status SetServerConfigPort(const std::string &value); - Status SetServerConfigMode(const std::string &value); + Status SetServerConfigDeployMode(const std::string &value); Status SetServerConfigTimeZone(const std::string &value); /* db config */ - Status SetDBConfigPath(const std::string &value); - Status SetDBConfigSlavePath(const std::string &value); + Status SetDBConfigPrimaryPath(const std::string &value); + Status SetDBConfigSecondaryPath(const std::string &value); Status SetDBConfigBackendUrl(const std::string &value); Status SetDBConfigArchiveDiskThreshold(const std::string &value); Status SetDBConfigArchiveDaysThreshold(const std::string &value); - Status SetDBConfigBufferSize(const std::string &value); + Status SetDBConfigInsertBufferSize(const std::string &value); Status SetDBConfigBuildIndexGPU(const std::string &value); /* metric config */ - Status SetMetricConfigAutoBootup(const std::string &value); + Status SetMetricConfigEnableMonitor(const std::string &value); Status SetMetricConfigCollector(const std::string &value); Status SetMetricConfigPrometheusPort(const std::string &value); diff --git a/cpp/src/server/DBWrapper.cpp b/cpp/src/server/DBWrapper.cpp index b142f31d1064fc38c37db2f238080bce006defc2..66104bfe5a60d58822f90062011a0585b6bd7dd7 100644 --- a/cpp/src/server/DBWrapper.cpp +++ b/cpp/src/server/DBWrapper.cpp @@ -34,9 +34,8 @@ namespace server { DBWrapper::DBWrapper() { } -Status -DBWrapper::StartService() { - Config &config = Config::GetInstance(); +Status DBWrapper::StartService() { + Config& config = Config::GetInstance(); Status s; //db config engine::DBOptions opt; @@ -45,13 +44,13 @@ DBWrapper::StartService() { if (!s.ok()) return s; std::string path; - s = config.GetDBConfigPath(path); + s = config.GetDBConfigPrimaryPath(path); if (!s.ok()) return s; opt.meta_.path_ = path + "/db"; std::string db_slave_path; - s = config.GetDBConfigSlavePath(db_slave_path); + s = config.GetDBConfigSecondaryPath(db_slave_path); if (!s.ok()) return s; StringHelpFunctions::SplitStringByDelimeter(db_slave_path, ";", opt.meta_.slave_paths_); @@ -61,18 +60,19 @@ DBWrapper::StartService() { if (!s.ok()) return s; std::string mode; - s = config.GetServerConfigMode(mode); + s = config.GetServerConfigDeployMode(mode); if (!s.ok()) return s; if (mode == "single") { opt.mode_ = engine::DBOptions::MODE::SINGLE; - } else if (mode == "cluster") { - opt.mode_ = engine::DBOptions::MODE::CLUSTER; - } else if (mode == "read_only") { - opt.mode_ = engine::DBOptions::MODE::READ_ONLY; + } else if (mode == "cluster_readonly") { + opt.mode_ = engine::DBOptions::MODE::CLUSTER_READONLY; + } else if (mode == "cluster_writable") { + opt.mode_ = engine::DBOptions::MODE::CLUSTER_WRITABLE; } else { - std::cerr << "ERROR: mode specified in server_config is not one of ['single', 'cluster', 'read_only']" - << std::endl; + std::cerr << + "ERROR: mode specified in server_config must be ['single', 'cluster_readonly', 'cluster_writable']" + << std::endl; kill(0, SIGUSR1); } @@ -86,7 +86,7 @@ DBWrapper::StartService() { } else { uint32_t sys_thread_cnt = 8; if (CommonUtil::GetSystemAvailableThreads(sys_thread_cnt)) { - omp_thread = (int32_t) ceil(sys_thread_cnt * 0.5); + omp_thread = (int32_t)ceil(sys_thread_cnt*0.5); omp_set_num_threads(omp_thread); } } @@ -120,7 +120,7 @@ DBWrapper::StartService() { kill(0, SIGUSR1); } - for (auto &path : opt.meta_.slave_paths_) { + for (auto& path : opt.meta_.slave_paths_) { status = CommonUtil::CreateDirectory(path); if (!status.ok()) { std::cerr << "ERROR! Failed to create database slave path: " << path << std::endl; @@ -131,7 +131,7 @@ DBWrapper::StartService() { //create db instance try { db_ = engine::DBFactory::Build(opt); - } catch (std::exception &ex) { + } catch(std::exception& ex) { std::cerr << "ERROR! Failed to open database: " << ex.what() << std::endl; kill(0, SIGUSR1); } @@ -141,8 +141,7 @@ DBWrapper::StartService() { return Status::OK(); } -Status -DBWrapper::StopService() { +Status DBWrapper::StopService() { if (db_) { db_->Stop(); } diff --git a/cpp/src/server/Server.cpp b/cpp/src/server/Server.cpp index 336514631cb4e01962a858d98db5c63ffb888e32..f0675fc84d2d6edb2b51e9b35b092bf421283f1a 100644 --- a/cpp/src/server/Server.cpp +++ b/cpp/src/server/Server.cpp @@ -239,6 +239,12 @@ Server::LoadConfig() { std::cerr << "Failed to load config file: " << config_filename_ << std::endl; exit(0); } + + s = config.ValidateConfig(); + if (!s.ok()) { + std::cerr << "Config check fail: " << s.message() << std::endl; + exit(0); + } return SERVER_SUCCESS; }