Skip to content

milvus
milvus

qq_35024513 / milvus
与 Fork 源项目一致

Fork自 milvus / milvus

通知 14
Star 0
Fork 0
milvus
milvus

体验新版 GitCode，发现更多精彩内容 >>

提交 5b0a13bf 编写于 作者: S starlord
浏览文件
操作

fix build error 


Former-commit-id: 3416c37d548be2e04dc2814830d935baab5104cb
上级 2b132a9c 20f7bc33
隐藏空白更改
内联 并排
Showing with 258 addition and 145 deletion
cpp/conf/server_config.template
浏览文件 @ 5b0a13bf
# All the following configurations are default values.
server_config: server_config:
address: 0.0.0.0 # milvus server ip address (IPv4) address: 0.0.0.0 # milvus server ip address (IPv4)
port: 19530 # port range: 1025 ~ 65534 port: 19530 # port range: 1025 ~ 65534
mode: single # deployment type: single, cluster, read_only deploy_mode: single # deployment type: single, cluster_readonly, cluster_writable
time_zone: UTC+8 time_zone: UTC+8
db_config: db_config:
path: @MILVUS_DB_PATH@ # milvus database path primary_path: @MILVUS_DB_PATH@ # path used to store data and meta
slave_path: # secondary database path, split by semicolon secondary_path: # path used to store data only, split by semicolon
# URI format: dialect://username:password@host:port/database backend_url: sqlite://:@:/ # URI format: dialect://username:password@host:port/database
# All parts except dialect are optional, but you MUST include the delimiters # Keep 'dialect://:@:/', and replace other texts with real values.
# Currently dialect supports mysql or sqlite # Replace 'dialect' with 'mysql' or 'sqlite'
backend_url: sqlite://:@:/
archive_disk_threshold: 0 # GB, file will be archived when disk usage exceed, 0 for no limit insert_buffer_size: 4 # GB, maximum insert buffer size allowed
archive_days_threshold: 0 # DAYS, older files will be archived, 0 for no limit build_index_gpu: 0 # gpu id used for building index
buffer_size: 4 # GB, maximum insert buffer size allowed
build_index_gpu: 0 # gpu id used for building index
metric_config: metric_config:
auto_bootup: off # whether enable monitoring when bootup enable_monitor: false # enable monitoring or not
collector: prometheus # prometheus collector: prometheus # prometheus
prometheus_config: prometheus_config:
port: 8080 # port prometheus used to fetch metrics port: 8080 # port prometheus used to fetch metrics
cache_config: cache_config:
cpu_mem_capacity: 16 # GB, CPU memory size used for cache cpu_mem_capacity: 16 # GB, CPU memory used for cache
cpu_mem_threshold: 0.85 # percent of data kept when cache cleanup triggered cpu_mem_threshold: 0.85 # percentage of data kept when cache cleanup triggered
cache_insert_data: false # whether load data into cache when insert cache_insert_data: false # whether load inserted data into cache
engine_config: engine_config:
blas_threshold: 20 blas_threshold: 20
resource_config: resource_config:
mode: simple resource_pool:
pool:
- cpu - cpu
- gpu0 - gpu0
cpp/src/db/DBImpl.cpp
浏览文件 @ 5b0a13bf
...@@ -77,7 +77,7 @@ Status DBImpl::Start() { ...@@ -77,7 +77,7 @@ Status DBImpl::Start() {
shutting_down_.store(false, std::memory_order_release); shutting_down_.store(false, std::memory_order_release);
//for distribute version, some nodes are read only //for distribute version, some nodes are read only
if (options_.mode_ != DBOptions::MODE::READ_ONLY) { if (options_.mode_ != DBOptions::MODE::CLUSTER_READONLY) {
ENGINE_LOG_TRACE << "StartTimerTasks"; ENGINE_LOG_TRACE << "StartTimerTasks";
bg_timer_thread_ = std::thread(&DBImpl::BackgroundTimerTask, this); bg_timer_thread_ = std::thread(&DBImpl::BackgroundTimerTask, this);
} }
...@@ -98,7 +98,7 @@ Status DBImpl::Stop() { ...@@ -98,7 +98,7 @@ Status DBImpl::Stop() {
//wait compaction/buildindex finish //wait compaction/buildindex finish
bg_timer_thread_.join(); bg_timer_thread_.join();
if (options_.mode_ != DBOptions::MODE::READ_ONLY) { if (options_.mode_ != DBOptions::MODE::CLUSTER_READONLY) {
meta_ptr_->CleanUp(); meta_ptr_->CleanUp();
} }
...@@ -704,7 +704,7 @@ void DBImpl::BackgroundCompaction(std::set<std::string> table_ids) { ...@@ -704,7 +704,7 @@ void DBImpl::BackgroundCompaction(std::set<std::string> table_ids) {
meta_ptr_->Archive(); meta_ptr_->Archive();
int ttl = 5*meta::M_SEC;//default: file will be deleted after 5 minutes int ttl = 5*meta::M_SEC;//default: file will be deleted after 5 minutes
if (options_.mode_ == DBOptions::MODE::CLUSTER) { if (options_.mode_ == DBOptions::MODE::CLUSTER_WRITABLE) {
ttl = meta::D_SEC; ttl = meta::D_SEC;
} }
meta_ptr_->CleanUpFilesWithTTL(ttl); meta_ptr_->CleanUpFilesWithTTL(ttl);
......
cpp/src/db/Options.h
浏览文件 @ 5b0a13bf
...@@ -60,9 +60,9 @@ struct DBMetaOptions { ...@@ -60,9 +60,9 @@ struct DBMetaOptions {
struct DBOptions { struct DBOptions {
typedef enum { typedef enum {
SINGLE, SINGLE = 0,
CLUSTER, CLUSTER_READONLY,
READ_ONLY CLUSTER_WRITABLE
} MODE; } MODE;
uint16_t merge_trigger_number_ = 2; uint16_t merge_trigger_number_ = 2;
......
cpp/src/db/meta/MySQLMetaImpl.cpp
浏览文件 @ 5b0a13bf
...@@ -284,7 +284,7 @@ Status MySQLMetaImpl::Initialize() { ...@@ -284,7 +284,7 @@ Status MySQLMetaImpl::Initialize() {
//step 5: create meta tables //step 5: create meta tables
try { try {
if (mode_ != DBOptions::MODE::READ_ONLY) { if (mode_ != DBOptions::MODE::CLUSTER_READONLY) {
CleanUp(); CleanUp();
} }
...@@ -768,7 +768,7 @@ Status MySQLMetaImpl::DeleteTable(const std::string &table_id) { ...@@ -768,7 +768,7 @@ Status MySQLMetaImpl::DeleteTable(const std::string &table_id) {
} //Scoped Connection } //Scoped Connection
if (mode_ == DBOptions::MODE::CLUSTER) { if (mode_ == DBOptions::MODE::CLUSTER_WRITABLE) {
DeleteTableFiles(table_id); DeleteTableFiles(table_id);
} }
......
cpp/src/metrics/PrometheusMetrics.cpp
浏览文件 @ 5b0a13bf
...@@ -31,7 +31,7 @@ ErrorCode ...@@ -31,7 +31,7 @@ ErrorCode
PrometheusMetrics::Init() { PrometheusMetrics::Init() {
try { try {
Config &config = Config::GetInstance(); Config &config = Config::GetInstance();
Status s = config.GetMetricConfigAutoBootup(startup_); Status s = config.GetMetricConfigEnableMonitor(startup_);
if (!s.ok()) return s.code(); if (!s.ok()) return s.code();
if (!startup_) return SERVER_SUCCESS; if (!startup_) return SERVER_SUCCESS;
......
cpp/src/server/Config.cpp
浏览文件 @ 5b0a13bf
...@@ -71,6 +71,111 @@ Config::LoadConfigFile(const std::string &filename) { ...@@ -71,6 +71,111 @@ Config::LoadConfigFile(const std::string &filename) {
return Status::OK(); return Status::OK();
} }
Status
Config::ValidateConfig() {
Status s;
/* server config */
std::string server_addr;
s = GetServerConfigAddress(server_addr);
if (!s.ok()) return s;
std::string server_port;
s = GetServerConfigPort(server_port);
if (!s.ok()) return s;
std::string server_mode;
s = GetServerConfigDeployMode(server_mode);
if (!s.ok()) return s;
std::string server_time_zone;
s = GetServerConfigTimeZone(server_time_zone);
if (!s.ok()) return s;
/* db config */
std::string db_primary_path;
s = GetDBConfigPrimaryPath(db_primary_path);
if (!s.ok()) return s;
std::string db_secondary_path;
s = GetDBConfigSecondaryPath(db_secondary_path);
if (!s.ok()) return s;
std::string db_backend_url;
s = GetDBConfigBackendUrl(db_backend_url);
if (!s.ok()) return s;
int32_t db_archive_disk_threshold;
s = GetDBConfigArchiveDiskThreshold(db_archive_disk_threshold);
if (!s.ok()) return s;
int32_t db_archive_days_threshold;
s = GetDBConfigArchiveDaysThreshold(db_archive_days_threshold);
if (!s.ok()) return s;
int32_t db_insert_buffer_size;
s = GetDBConfigInsertBufferSize(db_insert_buffer_size);
if (!s.ok()) return s;
int32_t db_build_index_gpu;
s = GetDBConfigBuildIndexGPU(db_build_index_gpu);
if (!s.ok()) return s;
/* metric config */
bool metric_enable_monitor;
s = GetMetricConfigEnableMonitor(metric_enable_monitor);
if (!s.ok()) return s;
std::string metric_collector;
s = GetMetricConfigCollector(metric_collector);
if (!s.ok()) return s;
std::string metric_prometheus_port;
s = GetMetricConfigPrometheusPort(metric_prometheus_port);
if (!s.ok()) return s;
/* cache config */
int32_t cache_cpu_mem_capacity;
s = GetCacheConfigCpuMemCapacity(cache_cpu_mem_capacity);
if (!s.ok()) return s;
float cache_cpu_mem_threshold;
s = GetCacheConfigCpuMemThreshold(cache_cpu_mem_threshold);
if (!s.ok()) return s;
int32_t cache_gpu_mem_capacity;
s = GetCacheConfigGpuMemCapacity(cache_gpu_mem_capacity);
if (!s.ok()) return s;
float cache_gpu_mem_threshold;
s = GetCacheConfigGpuMemThreshold(cache_gpu_mem_threshold);
if (!s.ok()) return s;
bool cache_insert_data;
s = GetCacheConfigCacheInsertData(cache_insert_data);
if (!s.ok()) return s;
/* engine config */
int32_t engine_blas_threshold;
s = GetEngineConfigBlasThreshold(engine_blas_threshold);
if (!s.ok()) return s;
int32_t engine_omp_thread_num;
s = GetEngineConfigOmpThreadNum(engine_omp_thread_num);
if (!s.ok()) return s;
/* resource config */
std::string resource_mode;
s = GetResourceConfigMode(resource_mode);
if (!s.ok()) return s;
std::vector<std::string> resource_pool;
s = GetResourceConfigPool(resource_pool);
if (!s.ok()) return s;
return Status::OK();
}
void void
Config::PrintConfigSection(const std::string &config_node_name) { Config::PrintConfigSection(const std::string &config_node_name) {
std::cout << std::endl; std::cout << std::endl;
...@@ -115,9 +220,12 @@ Config::CheckServerConfigPort(const std::string &value) { ...@@ -115,9 +220,12 @@ Config::CheckServerConfigPort(const std::string &value) {
} }
Status Status
Config::CheckServerConfigMode(const std::string &value) { Config::CheckServerConfigDeployMode(const std::string &value) {
if (value != "single" && value != "cluster" && value != "read_only") { if (value != "single" &&
return Status(SERVER_INVALID_ARGUMENT, "Invalid server config mode [single, cluster, read_only]: " + value); value != "cluster_readonly" &&
value != "cluster_writable") {
return Status(SERVER_INVALID_ARGUMENT,
"Invalid server config mode [single, cluster_readonly, cluster_writable]: " + value);
} }
return Status::OK(); return Status::OK();
} }
...@@ -141,15 +249,15 @@ Config::CheckServerConfigTimeZone(const std::string &value) { ...@@ -141,15 +249,15 @@ Config::CheckServerConfigTimeZone(const std::string &value) {
} }
Status Status
Config::CheckDBConfigPath(const std::string &value) { Config::CheckDBConfigPrimaryPath(const std::string &value) {
if (value.empty()) { if (value.empty()) {
return Status(SERVER_INVALID_ARGUMENT, "DB config path empty"); return Status(SERVER_INVALID_ARGUMENT, "DB config primary_path empty");
} }
return Status::OK(); return Status::OK();
} }
Status Status
Config::CheckDBConfigSlavePath(const std::string &value) { Config::CheckDBConfigSecondaryPath(const std::string &value) {
return Status::OK(); return Status::OK();
} }
...@@ -178,15 +286,15 @@ Config::CheckDBConfigArchiveDaysThreshold(const std::string &value) { ...@@ -178,15 +286,15 @@ Config::CheckDBConfigArchiveDaysThreshold(const std::string &value) {
} }
Status Status
Config::CheckDBConfigBufferSize(const std::string &value) { Config::CheckDBConfigInsertBufferSize(const std::string &value) {
if (!ValidationUtil::ValidateStringIsNumber(value).ok()) { if (!ValidationUtil::ValidateStringIsNumber(value).ok()) {
return Status(SERVER_INVALID_ARGUMENT, "Invalid DB config buffer_size: " + value); return Status(SERVER_INVALID_ARGUMENT, "Invalid DB config insert_buffer_size: " + value);
} else { } else {
int64_t buffer_size = std::stoi(value) * GB; int64_t buffer_size = std::stoi(value) * GB;
uint64_t total_mem = 0, free_mem = 0; uint64_t total_mem = 0, free_mem = 0;
CommonUtil::GetSystemMemInfo(total_mem, free_mem); CommonUtil::GetSystemMemInfo(total_mem, free_mem);
if (buffer_size >= total_mem) { if (buffer_size >= total_mem) {
return Status(SERVER_INVALID_ARGUMENT, "DB config buffer_size exceed system memory: " + value); return Status(SERVER_INVALID_ARGUMENT, "DB config insert_buffer_size exceed system memory: " + value);
} }
} }
return Status::OK(); return Status::OK();
...@@ -206,7 +314,7 @@ Config::CheckDBConfigBuildIndexGPU(const std::string &value) { ...@@ -206,7 +314,7 @@ Config::CheckDBConfigBuildIndexGPU(const std::string &value) {
} }
Status Status
Config::CheckMetricConfigAutoBootup(const std::string &value) { Config::CheckMetricConfigEnableMonitor(const std::string &value) {
if (!ValidationUtil::ValidateStringIsBool(value).ok()) { if (!ValidationUtil::ValidateStringIsBool(value).ok()) {
return Status(SERVER_INVALID_ARGUMENT, "Invalid metric config auto_bootup: " + value); return Status(SERVER_INVALID_ARGUMENT, "Invalid metric config auto_bootup: " + value);
} }
...@@ -243,10 +351,10 @@ Config::CheckCacheConfigCpuMemCapacity(const std::string &value) { ...@@ -243,10 +351,10 @@ Config::CheckCacheConfigCpuMemCapacity(const std::string &value) {
std::cerr << "Warning: cpu_mem_capacity value is too big" << std::endl; std::cerr << "Warning: cpu_mem_capacity value is too big" << std::endl;
} }
int32_t buffer_size; int32_t buffer_value;
Status s = GetDBConfigBufferSize(buffer_size); Status s = GetDBConfigInsertBufferSize(buffer_value);
if (!s.ok()) return s; if (!s.ok()) return s;
int64_t insert_buffer_size = buffer_size * GB; int64_t insert_buffer_size = buffer_value * GB;
if (insert_buffer_size + cpu_cache_capacity >= total_mem) { if (insert_buffer_size + cpu_cache_capacity >= total_mem) {
return Status(SERVER_INVALID_ARGUMENT, "Sum of cpu_mem_capacity and buffer_size exceed system memory"); return Status(SERVER_INVALID_ARGUMENT, "Sum of cpu_mem_capacity and buffer_size exceed system memory");
} }
...@@ -404,12 +512,12 @@ Config::GetServerConfigStrPort() { ...@@ -404,12 +512,12 @@ Config::GetServerConfigStrPort() {
} }
std::string std::string
Config::GetServerConfigStrMode() { Config::GetServerConfigStrDeployMode() {
std::string value; std::string value;
if (!GetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_MODE, value).ok()) { if (!GetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_DEPLOY_MODE, value).ok()) {
value = GetConfigNode(CONFIG_SERVER).GetValue(CONFIG_SERVER_MODE, value = GetConfigNode(CONFIG_SERVER).GetValue(CONFIG_SERVER_DEPLOY_MODE,
CONFIG_SERVER_MODE_DEFAULT); CONFIG_SERVER_DEPLOY_MODE_DEFAULT);
SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_MODE, value); SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_DEPLOY_MODE, value);
} }
return value; return value;
} }
...@@ -428,23 +536,23 @@ Config::GetServerConfigStrTimeZone() { ...@@ -428,23 +536,23 @@ Config::GetServerConfigStrTimeZone() {
//////////////////////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////////////////////////
/* db config */ /* db config */
std::string std::string
Config::GetDBConfigStrPath() { Config::GetDBConfigStrPrimaryPath() {
std::string value; std::string value;
if (!GetConfigValueInMem(CONFIG_DB, CONFIG_DB_PATH, value).ok()) { if (!GetConfigValueInMem(CONFIG_DB, CONFIG_DB_PRIMARY_PATH, value).ok()) {
value = GetConfigNode(CONFIG_DB).GetValue(CONFIG_DB_PATH, value = GetConfigNode(CONFIG_DB).GetValue(CONFIG_DB_PRIMARY_PATH,
CONFIG_DB_PATH_DEFAULT); CONFIG_DB_PRIMARY_PATH_DEFAULT);
SetConfigValueInMem(CONFIG_DB, CONFIG_DB_PATH, value); SetConfigValueInMem(CONFIG_DB, CONFIG_DB_PRIMARY_PATH, value);
} }
return value; return value;
} }
std::string std::string
Config::GetDBConfigStrSlavePath() { Config::GetDBConfigStrSecondaryPath() {
std::string value; std::string value;
if (!GetConfigValueInMem(CONFIG_DB, CONFIG_DB_SLAVE_PATH, value).ok()) { if (!GetConfigValueInMem(CONFIG_DB, CONFIG_DB_SECONDARY_PATH, value).ok()) {
value = GetConfigNode(CONFIG_DB).GetValue(CONFIG_DB_SLAVE_PATH, value = GetConfigNode(CONFIG_DB).GetValue(CONFIG_DB_SECONDARY_PATH,
CONFIG_DB_SLAVE_PATH_DEFAULT); CONFIG_DB_SECONDARY_PATH_DEFAULT);
SetConfigValueInMem(CONFIG_DB, CONFIG_DB_SLAVE_PATH, value); SetConfigValueInMem(CONFIG_DB, CONFIG_DB_SECONDARY_PATH, value);
} }
return value; return value;
} }
...@@ -483,12 +591,12 @@ Config::GetDBConfigStrArchiveDaysThreshold() { ...@@ -483,12 +591,12 @@ Config::GetDBConfigStrArchiveDaysThreshold() {
} }
std::string std::string
Config::GetDBConfigStrBufferSize() { Config::GetDBConfigStrInsertBufferSize() {
std::string value; std::string value;
if (!GetConfigValueInMem(CONFIG_DB, CONFIG_DB_BUFFER_SIZE, value).ok()) { if (!GetConfigValueInMem(CONFIG_DB, CONFIG_DB_INSERT_BUFFER_SIZE, value).ok()) {
value = GetConfigNode(CONFIG_DB).GetValue(CONFIG_DB_BUFFER_SIZE, value = GetConfigNode(CONFIG_DB).GetValue(CONFIG_DB_INSERT_BUFFER_SIZE,
CONFIG_DB_BUFFER_SIZE_DEFAULT); CONFIG_DB_INSERT_BUFFER_SIZE_DEFAULT);
SetConfigValueInMem(CONFIG_DB, CONFIG_DB_BUFFER_SIZE, value); SetConfigValueInMem(CONFIG_DB, CONFIG_DB_INSERT_BUFFER_SIZE, value);
} }
return value; return value;
} }
...@@ -507,12 +615,12 @@ Config::GetDBConfigStrBuildIndexGPU() { ...@@ -507,12 +615,12 @@ Config::GetDBConfigStrBuildIndexGPU() {
//////////////////////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////////////////////////
/* metric config */ /* metric config */
std::string std::string
Config::GetMetricConfigStrAutoBootup() { Config::GetMetricConfigStrEnableMonitor() {
std::string value; std::string value;
if (!GetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_AUTO_BOOTUP, value).ok()) { if (!GetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_ENABLE_MONITOR, value).ok()) {
value = GetConfigNode(CONFIG_METRIC).GetValue(CONFIG_METRIC_AUTO_BOOTUP, value = GetConfigNode(CONFIG_METRIC).GetValue(CONFIG_METRIC_ENABLE_MONITOR,
CONFIG_METRIC_AUTO_BOOTUP_DEFAULT); CONFIG_METRIC_ENABLE_MONITOR_DEFAULT);
SetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_AUTO_BOOTUP, value); SetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_ENABLE_MONITOR, value);
} }
return value; return value;
} }
...@@ -647,9 +755,9 @@ Config::GetServerConfigPort(std::string &value) { ...@@ -647,9 +755,9 @@ Config::GetServerConfigPort(std::string &value) {
} }
Status Status
Config::GetServerConfigMode(std::string &value) { Config::GetServerConfigDeployMode(std::string &value) {
value = GetServerConfigStrMode(); value = GetServerConfigStrDeployMode();
return CheckServerConfigMode(value); return CheckServerConfigDeployMode(value);
} }
Status Status
...@@ -659,14 +767,14 @@ Config::GetServerConfigTimeZone(std::string &value) { ...@@ -659,14 +767,14 @@ Config::GetServerConfigTimeZone(std::string &value) {
} }
Status Status
Config::GetDBConfigPath(std::string &value) { Config::GetDBConfigPrimaryPath(std::string &value) {
value = GetDBConfigStrPath(); value = GetDBConfigStrPrimaryPath();
return CheckDBConfigPath(value); return CheckDBConfigPrimaryPath(value);
} }
Status Status
Config::GetDBConfigSlavePath(std::string &value) { Config::GetDBConfigSecondaryPath(std::string &value) {
value = GetDBConfigStrSlavePath(); value = GetDBConfigStrSecondaryPath();
return Status::OK(); return Status::OK();
} }
...@@ -695,9 +803,9 @@ Config::GetDBConfigArchiveDaysThreshold(int32_t &value) { ...@@ -695,9 +803,9 @@ Config::GetDBConfigArchiveDaysThreshold(int32_t &value) {
} }
Status Status
Config::GetDBConfigBufferSize(int32_t &value) { Config::GetDBConfigInsertBufferSize(int32_t &value) {
std::string str = GetDBConfigStrBufferSize(); std::string str = GetDBConfigStrInsertBufferSize();
Status s = CheckDBConfigBufferSize(str); Status s = CheckDBConfigInsertBufferSize(str);
if (!s.ok()) return s; if (!s.ok()) return s;
value = std::stoi(str); value = std::stoi(str);
return Status::OK(); return Status::OK();
...@@ -713,9 +821,9 @@ Config::GetDBConfigBuildIndexGPU(int32_t &value) { ...@@ -713,9 +821,9 @@ Config::GetDBConfigBuildIndexGPU(int32_t &value) {
} }
Status Status
Config::GetMetricConfigAutoBootup(bool &value) { Config::GetMetricConfigEnableMonitor(bool &value) {
std::string str = GetMetricConfigStrAutoBootup(); std::string str = GetMetricConfigStrEnableMonitor();
Status s = CheckMetricConfigPrometheusPort(str); Status s = CheckMetricConfigEnableMonitor(str);
if (!s.ok()) return s; if (!s.ok()) return s;
std::transform(str.begin(), str.end(), str.begin(), ::tolower); std::transform(str.begin(), str.end(), str.begin(), ::tolower);
value = (str == "true" || str == "on" || str == "yes" || str == "1"); value = (str == "true" || str == "on" || str == "yes" || str == "1");
...@@ -830,10 +938,10 @@ Config::SetServerConfigPort(const std::string &value) { ...@@ -830,10 +938,10 @@ Config::SetServerConfigPort(const std::string &value) {
} }
Status Status
Config::SetServerConfigMode(const std::string &value) { Config::SetServerConfigDeployMode(const std::string &value) {
Status s = CheckServerConfigMode(value); Status s = CheckServerConfigDeployMode(value);
if (!s.ok()) return s; if (!s.ok()) return s;
SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_MODE, value); SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_DEPLOY_MODE, value);
return Status::OK(); return Status::OK();
} }
...@@ -847,18 +955,18 @@ Config::SetServerConfigTimeZone(const std::string &value) { ...@@ -847,18 +955,18 @@ Config::SetServerConfigTimeZone(const std::string &value) {
/* db config */ /* db config */
Status Status
Config::SetDBConfigPath(const std::string &value) { Config::SetDBConfigPrimaryPath(const std::string &value) {
Status s = CheckDBConfigPath(value); Status s = CheckDBConfigPrimaryPath(value);
if (!s.ok()) return s; if (!s.ok()) return s;
SetConfigValueInMem(CONFIG_DB, CONFIG_DB_PATH, value); SetConfigValueInMem(CONFIG_DB, CONFIG_DB_PRIMARY_PATH, value);
return Status::OK(); return Status::OK();
} }
Status Status
Config::SetDBConfigSlavePath(const std::string &value) { Config::SetDBConfigSecondaryPath(const std::string &value) {
Status s = CheckDBConfigSlavePath(value); Status s = CheckDBConfigSecondaryPath(value);
if (!s.ok()) return s; if (!s.ok()) return s;
SetConfigValueInMem(CONFIG_DB, CONFIG_DB_SLAVE_PATH, value); SetConfigValueInMem(CONFIG_DB, CONFIG_DB_SECONDARY_PATH, value);
return Status::OK(); return Status::OK();
} }
...@@ -887,10 +995,10 @@ Config::SetDBConfigArchiveDaysThreshold(const std::string &value) { ...@@ -887,10 +995,10 @@ Config::SetDBConfigArchiveDaysThreshold(const std::string &value) {
} }
Status Status
Config::SetDBConfigBufferSize(const std::string &value) { Config::SetDBConfigInsertBufferSize(const std::string &value) {
Status s = CheckDBConfigBufferSize(value); Status s = CheckDBConfigInsertBufferSize(value);
if (!s.ok()) return s; if (!s.ok()) return s;
SetConfigValueInMem(CONFIG_DB, CONFIG_DB_BUFFER_SIZE, value); SetConfigValueInMem(CONFIG_DB, CONFIG_DB_INSERT_BUFFER_SIZE, value);
return Status::OK(); return Status::OK();
} }
...@@ -904,10 +1012,10 @@ Config::SetDBConfigBuildIndexGPU(const std::string &value) { ...@@ -904,10 +1012,10 @@ Config::SetDBConfigBuildIndexGPU(const std::string &value) {
/* metric config */ /* metric config */
Status Status
Config::SetMetricConfigAutoBootup(const std::string &value) { Config::SetMetricConfigEnableMonitor(const std::string &value) {
Status s = CheckMetricConfigAutoBootup(value); Status s = CheckMetricConfigEnableMonitor(value);
if (!s.ok()) return s; if (!s.ok()) return s;
SetConfigValueInMem(CONFIG_DB, CONFIG_METRIC_AUTO_BOOTUP, value); SetConfigValueInMem(CONFIG_DB, CONFIG_METRIC_ENABLE_MONITOR, value);
return Status::OK(); return Status::OK();
} }
...@@ -997,3 +1105,4 @@ Config::SetResourceConfigMode(const std::string &value) { ...@@ -997,3 +1105,4 @@ Config::SetResourceConfigMode(const std::string &value) {
} // namespace server } // namespace server
} // namespace milvus } // namespace milvus
} // namespace zilliz } // namespace zilliz
cpp/src/server/Config.h
浏览文件 @ 5b0a13bf
...@@ -36,25 +36,25 @@ static const char *CONFIG_SERVER_ADDRESS = "address"; ...@@ -36,25 +36,25 @@ static const char *CONFIG_SERVER_ADDRESS = "address";
static const char *CONFIG_SERVER_ADDRESS_DEFAULT = "127.0.0.1"; static const char *CONFIG_SERVER_ADDRESS_DEFAULT = "127.0.0.1";
static const char *CONFIG_SERVER_PORT = "port"; static const char *CONFIG_SERVER_PORT = "port";
static const char *CONFIG_SERVER_PORT_DEFAULT = "19530"; static const char *CONFIG_SERVER_PORT_DEFAULT = "19530";
static const char *CONFIG_SERVER_MODE = "mode"; static const char *CONFIG_SERVER_DEPLOY_MODE = "deploy_mode";
static const char *CONFIG_SERVER_MODE_DEFAULT = "single"; static const char *CONFIG_SERVER_DEPLOY_MODE_DEFAULT = "single";
static const char *CONFIG_SERVER_TIME_ZONE = "time_zone"; static const char *CONFIG_SERVER_TIME_ZONE = "time_zone";
static const char *CONFIG_SERVER_TIME_ZONE_DEFAULT = "UTC+8"; static const char *CONFIG_SERVER_TIME_ZONE_DEFAULT = "UTC+8";
/* db config */ /* db config */
static const char *CONFIG_DB = "db_config"; static const char *CONFIG_DB = "db_config";
static const char *CONFIG_DB_PATH = "path"; static const char *CONFIG_DB_PRIMARY_PATH = "primary_path";
static const char *CONFIG_DB_PATH_DEFAULT = "/tmp/milvus"; static const char *CONFIG_DB_PRIMARY_PATH_DEFAULT = "/tmp/milvus";
static const char *CONFIG_DB_SLAVE_PATH = "slave_path"; static const char *CONFIG_DB_SECONDARY_PATH = "secondary_path";
static const char *CONFIG_DB_SLAVE_PATH_DEFAULT = ""; static const char *CONFIG_DB_SECONDARY_PATH_DEFAULT = "";
static const char *CONFIG_DB_BACKEND_URL = "backend_url"; static const char *CONFIG_DB_BACKEND_URL = "backend_url";
static const char *CONFIG_DB_BACKEND_URL_DEFAULT = "sqlite://:@:/"; static const char *CONFIG_DB_BACKEND_URL_DEFAULT = "sqlite://:@:/";
static const char *CONFIG_DB_ARCHIVE_DISK_THRESHOLD = "archive_disk_threshold"; static const char *CONFIG_DB_ARCHIVE_DISK_THRESHOLD = "archive_disk_threshold";
static const char *CONFIG_DB_ARCHIVE_DISK_THRESHOLD_DEFAULT = "0"; static const char *CONFIG_DB_ARCHIVE_DISK_THRESHOLD_DEFAULT = "0";
static const char *CONFIG_DB_ARCHIVE_DAYS_THRESHOLD = "archive_days_threshold"; static const char *CONFIG_DB_ARCHIVE_DAYS_THRESHOLD = "archive_days_threshold";
static const char *CONFIG_DB_ARCHIVE_DAYS_THRESHOLD_DEFAULT = "0"; static const char *CONFIG_DB_ARCHIVE_DAYS_THRESHOLD_DEFAULT = "0";
static const char *CONFIG_DB_BUFFER_SIZE = "buffer_size"; static const char *CONFIG_DB_INSERT_BUFFER_SIZE = "insert_buffer_size";
static const char *CONFIG_DB_BUFFER_SIZE_DEFAULT = "4"; static const char *CONFIG_DB_INSERT_BUFFER_SIZE_DEFAULT = "4";
static const char *CONFIG_DB_BUILD_INDEX_GPU = "build_index_gpu"; static const char *CONFIG_DB_BUILD_INDEX_GPU = "build_index_gpu";
static const char *CONFIG_DB_BUILD_INDEX_GPU_DEFAULT = "0"; static const char *CONFIG_DB_BUILD_INDEX_GPU_DEFAULT = "0";
...@@ -73,8 +73,8 @@ static const char *CONFIG_CACHE_CACHE_INSERT_DATA_DEFAULT = "false"; ...@@ -73,8 +73,8 @@ static const char *CONFIG_CACHE_CACHE_INSERT_DATA_DEFAULT = "false";
/* metric config */ /* metric config */
static const char *CONFIG_METRIC = "metric_config"; static const char *CONFIG_METRIC = "metric_config";
static const char *CONFIG_METRIC_AUTO_BOOTUP = "auto_bootup"; static const char *CONFIG_METRIC_ENABLE_MONITOR = "enable_monitor";
static const char *CONFIG_METRIC_AUTO_BOOTUP_DEFAULT = "false"; static const char *CONFIG_METRIC_ENABLE_MONITOR_DEFAULT = "false";
static const char *CONFIG_METRIC_COLLECTOR = "collector"; static const char *CONFIG_METRIC_COLLECTOR = "collector";
static const char *CONFIG_METRIC_COLLECTOR_DEFAULT = "prometheus"; static const char *CONFIG_METRIC_COLLECTOR_DEFAULT = "prometheus";
static const char *CONFIG_METRIC_PROMETHEUS = "prometheus_config"; static const char *CONFIG_METRIC_PROMETHEUS = "prometheus_config";
...@@ -92,12 +92,13 @@ static const char *CONFIG_ENGINE_OMP_THREAD_NUM_DEFAULT = "0"; ...@@ -92,12 +92,13 @@ static const char *CONFIG_ENGINE_OMP_THREAD_NUM_DEFAULT = "0";
static const char *CONFIG_RESOURCE = "resource_config"; static const char *CONFIG_RESOURCE = "resource_config";
static const char *CONFIG_RESOURCE_MODE = "mode"; static const char *CONFIG_RESOURCE_MODE = "mode";
static const char *CONFIG_RESOURCE_MODE_DEFAULT = "simple"; static const char *CONFIG_RESOURCE_MODE_DEFAULT = "simple";
static const char *CONFIG_RESOURCE_POOL = "pool"; static const char *CONFIG_RESOURCE_POOL = "resource_pool";
class Config { class Config {
public: public:
static Config &GetInstance(); static Config &GetInstance();
Status LoadConfigFile(const std::string &filename); Status LoadConfigFile(const std::string &filename);
Status ValidateConfig();
void PrintAll(); void PrintAll();
private: private:
...@@ -117,20 +118,20 @@ class Config { ...@@ -117,20 +118,20 @@ class Config {
/* server config */ /* server config */
Status CheckServerConfigAddress(const std::string &value); Status CheckServerConfigAddress(const std::string &value);
Status CheckServerConfigPort(const std::string &value); Status CheckServerConfigPort(const std::string &value);
Status CheckServerConfigMode(const std::string &value); Status CheckServerConfigDeployMode(const std::string &value);
Status CheckServerConfigTimeZone(const std::string &value); Status CheckServerConfigTimeZone(const std::string &value);
/* db config */ /* db config */
Status CheckDBConfigPath(const std::string &value); Status CheckDBConfigPrimaryPath(const std::string &value);
Status CheckDBConfigSlavePath(const std::string &value); Status CheckDBConfigSecondaryPath(const std::string &value);
Status CheckDBConfigBackendUrl(const std::string &value); Status CheckDBConfigBackendUrl(const std::string &value);
Status CheckDBConfigArchiveDiskThreshold(const std::string &value); Status CheckDBConfigArchiveDiskThreshold(const std::string &value);
Status CheckDBConfigArchiveDaysThreshold(const std::string &value); Status CheckDBConfigArchiveDaysThreshold(const std::string &value);
Status CheckDBConfigBufferSize(const std::string &value); Status CheckDBConfigInsertBufferSize(const std::string &value);
Status CheckDBConfigBuildIndexGPU(const std::string &value); Status CheckDBConfigBuildIndexGPU(const std::string &value);
/* metric config */ /* metric config */
Status CheckMetricConfigAutoBootup(const std::string &value); Status CheckMetricConfigEnableMonitor(const std::string &value);
Status CheckMetricConfigCollector(const std::string &value); Status CheckMetricConfigCollector(const std::string &value);
Status CheckMetricConfigPrometheusPort(const std::string &value); Status CheckMetricConfigPrometheusPort(const std::string &value);
...@@ -153,20 +154,20 @@ class Config { ...@@ -153,20 +154,20 @@ class Config {
/* server config */ /* server config */
std::string GetServerConfigStrAddress(); std::string GetServerConfigStrAddress();
std::string GetServerConfigStrPort(); std::string GetServerConfigStrPort();
std::string GetServerConfigStrMode(); std::string GetServerConfigStrDeployMode();
std::string GetServerConfigStrTimeZone(); std::string GetServerConfigStrTimeZone();
/* db config */ /* db config */
std::string GetDBConfigStrPath(); std::string GetDBConfigStrPrimaryPath();
std::string GetDBConfigStrSlavePath(); std::string GetDBConfigStrSecondaryPath();
std::string GetDBConfigStrBackendUrl(); std::string GetDBConfigStrBackendUrl();
std::string GetDBConfigStrArchiveDiskThreshold(); std::string GetDBConfigStrArchiveDiskThreshold();
std::string GetDBConfigStrArchiveDaysThreshold(); std::string GetDBConfigStrArchiveDaysThreshold();
std::string GetDBConfigStrBufferSize(); std::string GetDBConfigStrInsertBufferSize();
std::string GetDBConfigStrBuildIndexGPU(); std::string GetDBConfigStrBuildIndexGPU();
/* metric config */ /* metric config */
std::string GetMetricConfigStrAutoBootup(); std::string GetMetricConfigStrEnableMonitor();
std::string GetMetricConfigStrCollector(); std::string GetMetricConfigStrCollector();
std::string GetMetricConfigStrPrometheusPort(); std::string GetMetricConfigStrPrometheusPort();
...@@ -188,20 +189,20 @@ class Config { ...@@ -188,20 +189,20 @@ class Config {
/* server config */ /* server config */
Status GetServerConfigAddress(std::string &value); Status GetServerConfigAddress(std::string &value);
Status GetServerConfigPort(std::string &value); Status GetServerConfigPort(std::string &value);
Status GetServerConfigMode(std::string &value); Status GetServerConfigDeployMode(std::string &value);
Status GetServerConfigTimeZone(std::string &value); Status GetServerConfigTimeZone(std::string &value);
/* db config */ /* db config */
Status GetDBConfigPath(std::string &value); Status GetDBConfigPrimaryPath(std::string &value);
Status GetDBConfigSlavePath(std::string &value); Status GetDBConfigSecondaryPath(std::string &value);
Status GetDBConfigBackendUrl(std::string &value); Status GetDBConfigBackendUrl(std::string &value);
Status GetDBConfigArchiveDiskThreshold(int32_t &value); Status GetDBConfigArchiveDiskThreshold(int32_t &value);
Status GetDBConfigArchiveDaysThreshold(int32_t &value); Status GetDBConfigArchiveDaysThreshold(int32_t &value);
Status GetDBConfigBufferSize(int32_t &value); Status GetDBConfigInsertBufferSize(int32_t &value);
Status GetDBConfigBuildIndexGPU(int32_t &value); Status GetDBConfigBuildIndexGPU(int32_t &value);
/* metric config */ /* metric config */
Status GetMetricConfigAutoBootup(bool &value); Status GetMetricConfigEnableMonitor(bool &value);
Status GetMetricConfigCollector(std::string &value); Status GetMetricConfigCollector(std::string &value);
Status GetMetricConfigPrometheusPort(std::string &value); Status GetMetricConfigPrometheusPort(std::string &value);
...@@ -224,20 +225,20 @@ class Config { ...@@ -224,20 +225,20 @@ class Config {
/* server config */ /* server config */
Status SetServerConfigAddress(const std::string &value); Status SetServerConfigAddress(const std::string &value);
Status SetServerConfigPort(const std::string &value); Status SetServerConfigPort(const std::string &value);
Status SetServerConfigMode(const std::string &value); Status SetServerConfigDeployMode(const std::string &value);
Status SetServerConfigTimeZone(const std::string &value); Status SetServerConfigTimeZone(const std::string &value);
/* db config */ /* db config */
Status SetDBConfigPath(const std::string &value); Status SetDBConfigPrimaryPath(const std::string &value);
Status SetDBConfigSlavePath(const std::string &value); Status SetDBConfigSecondaryPath(const std::string &value);
Status SetDBConfigBackendUrl(const std::string &value); Status SetDBConfigBackendUrl(const std::string &value);
Status SetDBConfigArchiveDiskThreshold(const std::string &value); Status SetDBConfigArchiveDiskThreshold(const std::string &value);
Status SetDBConfigArchiveDaysThreshold(const std::string &value); Status SetDBConfigArchiveDaysThreshold(const std::string &value);
Status SetDBConfigBufferSize(const std::string &value); Status SetDBConfigInsertBufferSize(const std::string &value);
Status SetDBConfigBuildIndexGPU(const std::string &value); Status SetDBConfigBuildIndexGPU(const std::string &value);
/* metric config */ /* metric config */
Status SetMetricConfigAutoBootup(const std::string &value); Status SetMetricConfigEnableMonitor(const std::string &value);
Status SetMetricConfigCollector(const std::string &value); Status SetMetricConfigCollector(const std::string &value);
Status SetMetricConfigPrometheusPort(const std::string &value); Status SetMetricConfigPrometheusPort(const std::string &value);
......
cpp/src/server/DBWrapper.cpp
浏览文件 @ 5b0a13bf
...@@ -34,9 +34,8 @@ namespace server { ...@@ -34,9 +34,8 @@ namespace server {
DBWrapper::DBWrapper() { DBWrapper::DBWrapper() {
} }
Status Status DBWrapper::StartService() {
DBWrapper::StartService() { Config& config = Config::GetInstance();
Config &config = Config::GetInstance();
Status s; Status s;
//db config //db config
engine::DBOptions opt; engine::DBOptions opt;
...@@ -45,13 +44,13 @@ DBWrapper::StartService() { ...@@ -45,13 +44,13 @@ DBWrapper::StartService() {
if (!s.ok()) return s; if (!s.ok()) return s;
std::string path; std::string path;
s = config.GetDBConfigPath(path); s = config.GetDBConfigPrimaryPath(path);
if (!s.ok()) return s; if (!s.ok()) return s;
opt.meta_.path_ = path + "/db"; opt.meta_.path_ = path + "/db";
std::string db_slave_path; std::string db_slave_path;
s = config.GetDBConfigSlavePath(db_slave_path); s = config.GetDBConfigSecondaryPath(db_slave_path);
if (!s.ok()) return s; if (!s.ok()) return s;
StringHelpFunctions::SplitStringByDelimeter(db_slave_path, ";", opt.meta_.slave_paths_); StringHelpFunctions::SplitStringByDelimeter(db_slave_path, ";", opt.meta_.slave_paths_);
...@@ -61,18 +60,19 @@ DBWrapper::StartService() { ...@@ -61,18 +60,19 @@ DBWrapper::StartService() {
if (!s.ok()) return s; if (!s.ok()) return s;
std::string mode; std::string mode;
s = config.GetServerConfigMode(mode); s = config.GetServerConfigDeployMode(mode);
if (!s.ok()) return s; if (!s.ok()) return s;
if (mode == "single") { if (mode == "single") {
opt.mode_ = engine::DBOptions::MODE::SINGLE; opt.mode_ = engine::DBOptions::MODE::SINGLE;
} else if (mode == "cluster") { } else if (mode == "cluster_readonly") {
opt.mode_ = engine::DBOptions::MODE::CLUSTER; opt.mode_ = engine::DBOptions::MODE::CLUSTER_READONLY;
} else if (mode == "read_only") { } else if (mode == "cluster_writable") {
opt.mode_ = engine::DBOptions::MODE::READ_ONLY; opt.mode_ = engine::DBOptions::MODE::CLUSTER_WRITABLE;
} else { } else {
std::cerr << "ERROR: mode specified in server_config is not one of ['single', 'cluster', 'read_only']" std::cerr <<
<< std::endl; "ERROR: mode specified in server_config must be ['single', 'cluster_readonly', 'cluster_writable']"
<< std::endl;
kill(0, SIGUSR1); kill(0, SIGUSR1);
} }
...@@ -86,7 +86,7 @@ DBWrapper::StartService() { ...@@ -86,7 +86,7 @@ DBWrapper::StartService() {
} else { } else {
uint32_t sys_thread_cnt = 8; uint32_t sys_thread_cnt = 8;
if (CommonUtil::GetSystemAvailableThreads(sys_thread_cnt)) { if (CommonUtil::GetSystemAvailableThreads(sys_thread_cnt)) {
omp_thread = (int32_t) ceil(sys_thread_cnt * 0.5); omp_thread = (int32_t)ceil(sys_thread_cnt*0.5);
omp_set_num_threads(omp_thread); omp_set_num_threads(omp_thread);
} }
} }
...@@ -120,7 +120,7 @@ DBWrapper::StartService() { ...@@ -120,7 +120,7 @@ DBWrapper::StartService() {
kill(0, SIGUSR1); kill(0, SIGUSR1);
} }
for (auto &path : opt.meta_.slave_paths_) { for (auto& path : opt.meta_.slave_paths_) {
status = CommonUtil::CreateDirectory(path); status = CommonUtil::CreateDirectory(path);
if (!status.ok()) { if (!status.ok()) {
std::cerr << "ERROR! Failed to create database slave path: " << path << std::endl; std::cerr << "ERROR! Failed to create database slave path: " << path << std::endl;
...@@ -131,7 +131,7 @@ DBWrapper::StartService() { ...@@ -131,7 +131,7 @@ DBWrapper::StartService() {
//create db instance //create db instance
try { try {
db_ = engine::DBFactory::Build(opt); db_ = engine::DBFactory::Build(opt);
} catch (std::exception &ex) { } catch(std::exception& ex) {
std::cerr << "ERROR! Failed to open database: " << ex.what() << std::endl; std::cerr << "ERROR! Failed to open database: " << ex.what() << std::endl;
kill(0, SIGUSR1); kill(0, SIGUSR1);
} }
...@@ -141,8 +141,7 @@ DBWrapper::StartService() { ...@@ -141,8 +141,7 @@ DBWrapper::StartService() {
return Status::OK(); return Status::OK();
} }
Status Status DBWrapper::StopService() {
DBWrapper::StopService() {
if (db_) { if (db_) {
db_->Stop(); db_->Stop();
} }
......
cpp/src/server/Server.cpp
浏览文件 @ 5b0a13bf
...@@ -239,6 +239,12 @@ Server::LoadConfig() { ...@@ -239,6 +239,12 @@ Server::LoadConfig() {
std::cerr << "Failed to load config file: " << config_filename_ << std::endl; std::cerr << "Failed to load config file: " << config_filename_ << std::endl;
exit(0); exit(0);
} }
s = config.ValidateConfig();
if (!s.ok()) {
std::cerr << "Config check fail: " << s.message() << std::endl;
exit(0);
}
return SERVER_SUCCESS; return SERVER_SUCCESS;
} }
......
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或