From b26ce06463a5ada3327f182c7508bf80ae7ea654 Mon Sep 17 00:00:00 2001 From: Cai Yudong Date: Mon, 15 Jun 2020 23:23:10 +0800 Subject: [PATCH] remove secondary path (#2560) * config code clean Signed-off-by: yudong.cai * remove slave_paths_ from db options Signed-off-by: yudong.cai * fix clang-format Signed-off-by: yudong.cai * update version Signed-off-by: yudong.cai * update changelog Signed-off-by: yudong.cai * enable all db unittest Signed-off-by: yudong.cai * update some comments Signed-off-by: yudong.cai * update comments Signed-off-by: yudong.cai --- CHANGELOG.md | 1 + core/src/config/Config.cpp | 350 +----------------- core/src/config/Config.h | 14 +- core/src/db/Options.h | 1 - core/src/db/Utils.cpp | 70 +--- core/src/server/DBWrapper.cpp | 32 -- core/src/server/Server.cpp | 4 - .../request/ReLoadSegmentsRequest.cpp | 14 - core/src/server/init/StorageChecker.cpp | 12 - core/unittest/db/test_misc.cpp | 18 - 10 files changed, 26 insertions(+), 490 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 67f59df9..5a0f324a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ Please mark all change in change log and use the issue from GitHub - \#2509 Count up query statistics for debug ease ## Improvement +- \#2543 Remove secondary_path related code ## Task diff --git a/core/src/config/Config.cpp b/core/src/config/Config.cpp index 2197bbd7..87a1762d 100644 --- a/core/src/config/Config.cpp +++ b/core/src/config/Config.cpp @@ -87,11 +87,11 @@ const int64_t CONFIG_STORAGE_FILE_CLEANUP_TIMEOUT_MAX = 3600; /* cache config */ const char* CONFIG_CACHE = "cache"; const char* CONFIG_CACHE_CPU_CACHE_CAPACITY = "cache_size"; -const char* CONFIG_CACHE_CPU_CACHE_CAPACITY_DEFAULT = "4294967296"; +const char* CONFIG_CACHE_CPU_CACHE_CAPACITY_DEFAULT = "4294967296"; /* 4 GB */ const char* CONFIG_CACHE_CPU_CACHE_THRESHOLD = "cpu_cache_threshold"; const char* CONFIG_CACHE_CPU_CACHE_THRESHOLD_DEFAULT = "0.7"; const char* CONFIG_CACHE_INSERT_BUFFER_SIZE = "insert_buffer_size"; -const char* CONFIG_CACHE_INSERT_BUFFER_SIZE_DEFAULT = "1073741824"; // 1024 * 1024 * 1024 +const char* CONFIG_CACHE_INSERT_BUFFER_SIZE_DEFAULT = "1073741824"; /* 1 GB */ const char* CONFIG_CACHE_CACHE_INSERT_DATA = "cache_insert_data"; const char* CONFIG_CACHE_CACHE_INSERT_DATA_DEFAULT = "false"; const char* CONFIG_CACHE_PRELOAD_COLLECTION = "preload_collection"; @@ -124,7 +124,7 @@ const char* CONFIG_GPU_RESOURCE_ENABLE_DEFAULT = "true"; const char* CONFIG_GPU_RESOURCE_ENABLE_DEFAULT = "false"; #endif const char* CONFIG_GPU_RESOURCE_CACHE_CAPACITY = "cache_size"; -const char* CONFIG_GPU_RESOURCE_CACHE_CAPACITY_DEFAULT = "1073741824"; // 1024 * 1024 * 1024 +const char* CONFIG_GPU_RESOURCE_CACHE_CAPACITY_DEFAULT = "1073741824"; /* 1 GB */ const char* CONFIG_GPU_RESOURCE_CACHE_THRESHOLD = "cache_threshold"; const char* CONFIG_GPU_RESOURCE_CACHE_THRESHOLD_DEFAULT = "0.7"; const char* CONFIG_GPU_RESOURCE_GPU_SEARCH_THRESHOLD = "gpu_search_threshold"; @@ -146,9 +146,9 @@ const char* CONFIG_WAL_ENABLE_DEFAULT = "true"; const char* CONFIG_WAL_RECOVERY_ERROR_IGNORE = "recovery_error_ignore"; const char* CONFIG_WAL_RECOVERY_ERROR_IGNORE_DEFAULT = "true"; const char* CONFIG_WAL_BUFFER_SIZE = "buffer_size"; -const char* CONFIG_WAL_BUFFER_SIZE_DEFAULT = "268435456"; -const int64_t CONFIG_WAL_BUFFER_SIZE_MIN = 67108864; -const int64_t CONFIG_WAL_BUFFER_SIZE_MAX = 4294967296; +const char* CONFIG_WAL_BUFFER_SIZE_DEFAULT = "268435456"; /* 256 MB */ +const int64_t CONFIG_WAL_BUFFER_SIZE_MIN = 67108864; /* 64 MB */ +const int64_t CONFIG_WAL_BUFFER_SIZE_MAX = 4294967296; /* 4 GB */ const char* CONFIG_WAL_WAL_PATH = "path"; const char* CONFIG_WAL_WAL_PATH_DEFAULT = "/tmp/milvus/wal"; @@ -161,9 +161,9 @@ const char* CONFIG_LOGS_TRACE_ENABLE_DEFAULT = "true"; const char* CONFIG_LOGS_PATH = "path"; const char* CONFIG_LOGS_PATH_DEFAULT = "/tmp/milvus/logs"; const char* CONFIG_LOGS_MAX_LOG_FILE_SIZE = "max_log_file_size"; -const char* CONFIG_LOGS_MAX_LOG_FILE_SIZE_DEFAULT = "1073741824"; -const int64_t CONFIG_LOGS_MAX_LOG_FILE_SIZE_MIN = 536870912; -const int64_t CONFIG_LOGS_MAX_LOG_FILE_SIZE_MAX = 4294967296; +const char* CONFIG_LOGS_MAX_LOG_FILE_SIZE_DEFAULT = "1073741824"; /* 1 GB */ +const int64_t CONFIG_LOGS_MAX_LOG_FILE_SIZE_MIN = 536870912; /* 512 MB */ +const int64_t CONFIG_LOGS_MAX_LOG_FILE_SIZE_MAX = 4294967296; /* 4 GB */ const char* CONFIG_LOGS_LOG_ROTATE_NUM = "log_rotate_num"; const char* CONFIG_LOGS_LOG_ROTATE_NUM_DEFAULT = "0"; const int64_t CONFIG_LOGS_LOG_ROTATE_NUM_MIN = 0; @@ -276,22 +276,7 @@ Config::ValidateConfig() { std::string http_port; STATUS_CHECK(GetNetworkConfigHTTPPort(http_port)); - // std::string server_mode; - // STATUS_CHECK(GetServerConfigDeployMode(server_mode)); - - // std::string server_time_zone; - // STATUS_CHECK(GetServerConfigTimeZone(server_time_zone)); - - // bool server_web_enable; - // STATUS_CHECK(GetServerConfigWebEnable(server_web_enable)); - - // std::string server_web_port; - // STATUS_CHECK(GetServerConfigWebPort(server_web_port)); - /* db config */ - // std::string db_backend_url; - // STATUS_CHECK(GetDBConfigBackendUrl(db_backend_url)); - int64_t db_archive_disk_threshold; STATUS_CHECK(GetDBConfigArchiveDiskThreshold(db_archive_disk_threshold)); @@ -436,16 +421,7 @@ Config::ResetDefaultConfig() { STATUS_CHECK(SetNetworkConfigHTTPEnable(CONFIG_NETWORK_HTTP_ENABLE_DEFAULT)); STATUS_CHECK(SetNetworkConfigHTTPPort(CONFIG_NETWORK_HTTP_PORT_DEFAULT)); - /* server config */ - // STATUS_CHECK(SetServerConfigAddress(CONFIG_SERVER_ADDRESS_DEFAULT)); - // STATUS_CHECK(SetServerConfigPort(CONFIG_SERVER_PORT_DEFAULT)); - // STATUS_CHECK(SetServerConfigDeployMode(CONFIG_SERVER_DEPLOY_MODE_DEFAULT)); - // STATUS_CHECK(SetServerConfigTimeZone(CONFIG_SERVER_TIME_ZONE_DEFAULT)); - // STATUS_CHECK(SetServerConfigWebEnable(CONFIG_SERVER_WEB_ENABLE_DEFAULT)); - // STATUS_CHECK(SetServerConfigWebPort(CONFIG_SERVER_WEB_PORT_DEFAULT)); - /* db config */ - // STATUS_CHECK(SetDBConfigBackendUrl(CONFIG_DB_BACKEND_URL_DEFAULT)); STATUS_CHECK(SetDBConfigArchiveDiskThreshold(CONFIG_DB_ARCHIVE_DISK_THRESHOLD_DEFAULT)); STATUS_CHECK(SetDBConfigArchiveDaysThreshold(CONFIG_DB_ARCHIVE_DAYS_THRESHOLD_DEFAULT)); @@ -552,15 +528,6 @@ Config::SetConfigCli(const std::string& parent_key, const std::string& child_key } else { status = Status(SERVER_UNEXPECTED_ERROR, invalid_node_str); } - } else if (parent_key == CONFIG_DB) { - // if (child_key == CONFIG_DB_BACKEND_URL) { - // status = SetDBConfigBackendUrl(value); - // } else if (child_key == CONFIG_DB_PRELOAD_COLLECTION) { - // if (child_key == CONFIG_DB_PRELOAD_COLLECTION) { - // status = SetDBConfigPreloadCollection(value); - // } else { - // status = Status(SERVER_UNEXPECTED_ERROR, invalid_node_str); - // } } else if (parent_key == CONFIG_STORAGE) { if (child_key == CONFIG_STORAGE_PATH) { status = SetStorageConfigPath(value); @@ -702,9 +669,9 @@ Config::ProcessConfigCli(std::string& result, const std::string& cmd) { StringHelpFunctions::MergeStringWithDelimeter(nodes_s, CONFIG_NODE_DELIMITER, result); nodes[1] = result; } - // if (nodes.size() != 2) { - // return Status(SERVER_UNEXPECTED_ERROR, "Invalid command: " + cmd); - // } + // if (nodes.size() != 2) { + // return Status(SERVER_UNEXPECTED_ERROR, "Invalid command: " + cmd); + // } return GetConfigCli(result, nodes[0], nodes[1]); } } else if (tokens[0] == "set_config") { @@ -1016,116 +983,7 @@ Config::CheckNetworkConfigHTTPPort(const std::string& value) { return Status::OK(); } -/* server config */ -// Status -// Config::CheckServerConfigAddress(const std::string& value) { -// auto exist_error = !ValidationUtil::ValidateIpAddress(value).ok(); -// fiu_do_on("check_config_address_fail", exist_error = true); -// -// if (exist_error) { -// std::string msg = -// "Invalid server IP address: " + value + ". Possible reason: server_config.address is invalid."; -// return Status(SERVER_INVALID_ARGUMENT, msg); -// } -// return Status::OK(); -// } - -// Status -// Config::CheckServerConfigPort(const std::string& value) { -// auto exist_error = !ValidationUtil::ValidateStringIsNumber(value).ok(); -// fiu_do_on("check_config_port_fail", exist_error = true); -// -// if (exist_error) { -// std::string msg = "Invalid server port: " + value + ". Possible reason: server_config.port is not a number."; -// return Status(SERVER_INVALID_ARGUMENT, msg); -// } else { -// try { -// int32_t port = std::stoi(value); -// if (!(port > PORT_NUMBER_MIN && port < PORT_NUMBER_MAX)) { -// std::string msg = "Invalid server port: " + value + -// ". Possible reason: server_config.port is not in range (1024, 65535)."; -// return Status(SERVER_INVALID_ARGUMENT, msg); -// } -// } catch (...) { -// return Status(SERVER_INVALID_ARGUMENT, "Invalid server_config.port: " + value); -// } -// } -// return Status::OK(); -// } - -// Status -// Config::CheckServerConfigDeployMode(const std::string& value) { -// fiu_return_on("check_config_deploy_mode_fail", -// Status(SERVER_INVALID_ARGUMENT, -// "server_config.deploy_mode is not one of single, cluster_readonly, and cluster_writable.")); -// -// if (value != "single" && value != "cluster_readonly" && value != "cluster_writable") { -// return Status(SERVER_INVALID_ARGUMENT, -// "server_config.deploy_mode is not one of single, cluster_readonly, and cluster_writable."); -// } -// return Status::OK(); -//} - -// Status -// Config::CheckServerConfigTimeZone(const std::string& value) { -// fiu_return_on("check_config_time_zone_fail", -// Status(SERVER_INVALID_ARGUMENT, "Invalid server_config.time_zone: " + value)); -// -// if (value.length() <= 3) { -// return Status(SERVER_INVALID_ARGUMENT, "Invalid server_config.time_zone: " + value); -// } else { -// if (value.substr(0, 3) != "UTC") { -// return Status(SERVER_INVALID_ARGUMENT, "Invalid server_config.time_zone: " + value); -// } else { -// if (!ValidationUtil::IsNumber(value.substr(4))) { -// return Status(SERVER_INVALID_ARGUMENT, "Invalid server_config.time_zone: " + value); -// } -// } -// } -// return Status::OK(); -// } - -// Status -// Config::CheckServerConfigWebEnable(const std::string& value) { -// return ValidationUtil::ValidateStringIsBool(value); -// } - -// Status -// Config::CheckServerConfigWebPort(const std::string& value) { -// if (!ValidationUtil::ValidateStringIsNumber(value).ok()) { -// std::string msg = -// "Invalid web server port: " + value + ". Possible reason: server_config.web_port is not a number."; -// return Status(SERVER_INVALID_ARGUMENT, msg); -// } else { -// try { -// int32_t port = std::stoi(value); -// if (!(port > PORT_NUMBER_MIN && port < PORT_NUMBER_MAX)) { -// std::string msg = "Invalid web server port: " + value + -// ". Possible reason: server_config.web_port is not in range (1024, 65535)."; -// return Status(SERVER_INVALID_ARGUMENT, msg); -// } -// } catch (...) { -// return Status(SERVER_INVALID_ARGUMENT, "Invalid server_config.web_port: " + value); -// } -// } -// return Status::OK(); -// } - /* DB config */ -// Status -// Config::CheckDBConfigBackendUrl(const std::string& value) { -// auto exist_error = !ValidationUtil::ValidateDbURI(value).ok(); -// fiu_do_on("check_config_backend_url_fail", exist_error = true); -// -// if (exist_error) { -// std::string msg = -// "Invalid backend url: " + value + ". Possible reason: db_config.db_backend_url is invalid. " + -// "The correct format should be like sqlite://:@:/ or mysql://root:123456@127.0.0.1:3306/milvus."; -// return Status(SERVER_INVALID_ARGUMENT, msg); -// } -// return Status::OK(); -// } - Status Config::CheckDBConfigArchiveDiskThreshold(const std::string& value) { auto exist_error = !ValidationUtil::ValidateStringIsNumber(value).ok(); @@ -1308,7 +1166,6 @@ Status Config::CheckCacheConfigCpuCacheCapacity(const std::string& value) { fiu_return_on("check_config_cache_size_fail", Status(SERVER_INVALID_ARGUMENT, "")); -#if 1 std::string err; int64_t cache_size = parse_bytes(value, err); if (not err.empty()) { @@ -1341,42 +1198,6 @@ Config::CheckCacheConfigCpuCacheCapacity(const std::string& value) { return Status(SERVER_INVALID_ARGUMENT, msg); } } -#else - if (!ValidationUtil::ValidateStringIsNumber(value).ok()) { - std::string msg = "Invalid cpu cache capacity: " + value + - ". Possible reason: cache_config.cpu_cache_capacity is not a positive integer."; - return Status(SERVER_INVALID_ARGUMENT, msg); - } else { - int64_t cpu_cache_capacity = std::stoll(value) * GB; - if (cpu_cache_capacity <= 0) { - std::string msg = "Invalid cpu cache capacity: " + value + - ". Possible reason: cache_config.cpu_cache_capacity is not a positive integer."; - return Status(SERVER_INVALID_ARGUMENT, msg); - } - - int64_t total_mem = 0, free_mem = 0; - CommonUtil::GetSystemMemInfo(total_mem, free_mem); - if (cpu_cache_capacity >= total_mem) { - std::string msg = "Invalid cpu cache capacity: " + value + - ". Possible reason: cache_config.cpu_cache_capacity exceeds system memory."; - return Status(SERVER_INVALID_ARGUMENT, msg); - } else if (static_cast(cpu_cache_capacity) > static_cast(total_mem * 0.9)) { - std::cerr << "WARNING: cpu cache capacity value is too big" << std::endl; - } - - std::string str = GetConfigStr(CONFIG_CACHE, CONFIG_CACHE_INSERT_BUFFER_SIZE, "0"); - int64_t buffer_value = std::stoll(str); - - int64_t insert_buffer_size = buffer_value * GB; - fiu_do_on("Config.CheckCacheConfigCpuCacheCapacity.large_insert_buffer", insert_buffer_size = total_mem + 1); - if (insert_buffer_size + cpu_cache_capacity >= total_mem) { - std::string msg = "Invalid cpu cache capacity: " + value + - ". Possible reason: sum of cache_config.cpu_cache_capacity and " - "cache_config.insert_buffer_size exceeds system memory."; - return Status(SERVER_INVALID_ARGUMENT, msg); - } - } -#endif return Status::OK(); } @@ -1403,7 +1224,6 @@ Status Config::CheckCacheConfigInsertBufferSize(const std::string& value) { fiu_return_on("check_config_insert_buffer_size_fail", Status(SERVER_INVALID_ARGUMENT, "")); -#if 1 std::string err; int64_t buffer_size = parse_bytes(value, err); if (not err.empty()) { @@ -1428,32 +1248,6 @@ Config::CheckCacheConfigInsertBufferSize(const std::string& value) { return Status(SERVER_INVALID_ARGUMENT, msg); } } -#else - if (!ValidationUtil::ValidateStringIsNumber(value).ok()) { - std::string msg = "Invalid insert buffer size: " + value + - ". Possible reason: cache_config.insert_buffer_size is not a positive integer."; - return Status(SERVER_INVALID_ARGUMENT, msg); - } else { - int64_t buffer_size = std::stoll(value) * GB; - if (buffer_size <= 0) { - std::string msg = "Invalid insert buffer size: " + value + - ". Possible reason: cache_config.insert_buffer_size is not a positive integer."; - return Status(SERVER_INVALID_ARGUMENT, msg); - } - - std::string str = GetConfigStr(CONFIG_CACHE, CONFIG_CACHE_CPU_CACHE_CAPACITY, "0"); - std::string err; - int64_t cache_size = parse_bytes(str, err); - - int64_t total_mem = 0, free_mem = 0; - CommonUtil::GetSystemMemInfo(total_mem, free_mem); - if (buffer_size + cache_size >= total_mem) { - std::string msg = "Invalid insert buffer size: " + value + - ". Possible reason: cache_config.insert_buffer_size exceeds system memory."; - return Status(SERVER_INVALID_ARGUMENT, msg); - } - } -#endif return Status::OK(); } @@ -1549,9 +1343,8 @@ Config::CheckEngineConfigSimdType(const std::string& value) { return Status::OK(); } -#ifdef MILVUS_GPU_VERSION - /* gpu resource config */ +#ifdef MILVUS_GPU_VERSION Status Config::CheckGpuResourceConfigEnable(const std::string& value) { fiu_return_on("check_config_gpu_resource_enable_fail", Status(SERVER_INVALID_ARGUMENT, "")); @@ -1567,7 +1360,6 @@ Status Config::CheckGpuResourceConfigCacheCapacity(const std::string& value) { fiu_return_on("check_gpu_cache_size_fail", Status(SERVER_INVALID_ARGUMENT, "")); -#if 1 std::string err; int64_t gpu_cache_size = parse_bytes(value, err); if (not err.empty()) { @@ -1594,31 +1386,6 @@ Config::CheckGpuResourceConfigCacheCapacity(const std::string& value) { } } } -#else - if (!ValidationUtil::ValidateStringIsNumber(value).ok()) { - std::string msg = - "Invalid gpu cache capacity: " + value + ". Possible reason: gpu.cache_capacity is not a positive integer."; - return Status(SERVER_INVALID_ARGUMENT, msg); - } else { - int64_t gpu_cache_capacity = std::stoll(value) * GB; - std::vector gpu_ids; - STATUS_CHECK(GetGpuResourceConfigBuildIndexResources(gpu_ids)); - - for (int64_t gpu_id : gpu_ids) { - int64_t gpu_memory; - if (!ValidationUtil::GetGpuMemory(gpu_id, gpu_memory).ok()) { - std::string msg = "Fail to get GPU memory for GPU device: " + std::to_string(gpu_id); - return Status(SERVER_UNEXPECTED_ERROR, msg); - } else if (gpu_cache_capacity >= gpu_memory) { - std::string msg = "Invalid gpu cache capacity: " + value + - ". Possible reason: gpu.cache_capacity exceeds GPU memory."; - return Status(SERVER_INVALID_ARGUMENT, msg); - } else if (gpu_cache_capacity > (double)gpu_memory * 0.9) { - std::cerr << "Warning: gpu cache capacity value is too big" << std::endl; - } - } - } -#endif return Status::OK(); } @@ -1735,7 +1502,6 @@ Config::CheckGpuResourceConfigBuildIndexResources(const std::vector return Status::OK(); } - #endif /* tracing config */ @@ -2025,51 +1791,7 @@ Config::GetNetworkConfigHTTPPort(std::string& value) { return CheckNetworkConfigHTTPPort(value); } -/* server config */ -// Status -// Config::GetServerConfigAddress(std::string& value) { -// value = GetConfigStr(CONFIG_SERVER, CONFIG_SERVER_ADDRESS, CONFIG_SERVER_ADDRESS_DEFAULT); -// return CheckServerConfigAddress(value); -// } - -// Status -// Config::GetServerConfigPort(std::string& value) { -// value = GetConfigStr(CONFIG_SERVER, CONFIG_SERVER_PORT, CONFIG_SERVER_PORT_DEFAULT); -// return CheckServerConfigPort(value); -// } - -// Status -// Config::GetServerConfigDeployMode(std::string& value) { -// value = GetConfigStr(CONFIG_SERVER, CONFIG_SERVER_DEPLOY_MODE, CONFIG_SERVER_DEPLOY_MODE_DEFAULT); -// return CheckServerConfigDeployMode(value); -// } - -// Status -// Config::GetServerConfigTimeZone(std::string& value) { -// value = GetConfigStr(CONFIG_SERVER, CONFIG_SERVER_TIME_ZONE, CONFIG_SERVER_TIME_ZONE_DEFAULT); -// return CheckServerConfigTimeZone(value); -// } - -// Status -// Config::GetServerConfigWebEnable(bool& value) { -// std::string str = GetConfigStr(CONFIG_SERVER, CONFIG_SERVER_WEB_ENABLE, CONFIG_SERVER_WEB_ENABLE_DEFAULT); -// STATUS_CHECK(CheckServerConfigWebEnable(str)); -// return StringHelpFunctions::ConvertToBoolean(str, value); -// } - -// Status -// Config::GetServerConfigWebPort(std::string& value) { -// value = GetConfigStr(CONFIG_SERVER, CONFIG_SERVER_WEB_PORT, CONFIG_SERVER_WEB_PORT_DEFAULT); -// return CheckServerConfigWebPort(value); -// } - /* DB config */ -// Status -// Config::GetDBConfigBackendUrl(std::string& value) { -// value = GetConfigStr(CONFIG_DB, CONFIG_DB_BACKEND_URL, CONFIG_DB_BACKEND_URL_DEFAULT); -// return CheckDBConfigBackendUrl(value); -// } - Status Config::GetDBConfigArchiveDiskThreshold(int64_t& value) { std::string str = @@ -2246,7 +1968,6 @@ Config::GetEngineConfigSimdType(std::string& value) { /* gpu resource config */ #ifdef MILVUS_GPU_VERSION - Status Config::GetGpuResourceConfigEnable(bool& value) { std::string str = GetConfigStr(CONFIG_GPU_RESOURCE, CONFIG_GPU_RESOURCE_ENABLE, CONFIG_GPU_RESOURCE_ENABLE_DEFAULT); @@ -2340,7 +2061,6 @@ Config::GetGpuResourceConfigBuildIndexResources(std::vector& value) { } return Status::OK(); } - #endif /* tracing config */ @@ -2453,6 +2173,7 @@ Config::SetClusterConfigEnable(const std::string& value) { STATUS_CHECK(CheckClusterConfigEnable(value)); return SetConfigValueInMem(CONFIG_CLUSTER, CONFIG_CLUSTER_ENABLE, value); } + Status Config::SetClusterConfigRole(const std::string& value) { STATUS_CHECK(CheckClusterConfigRole(value)); @@ -2497,50 +2218,7 @@ Config::SetNetworkConfigHTTPPort(const std::string& value) { return SetConfigValueInMem(CONFIG_NETWORK, CONFIG_NETWORK_HTTP_PORT, value); } -/* server config */ -// Status -// Config::SetServerConfigAddress(const std::string& value) { -// STATUS_CHECK(CheckServerConfigAddress(value)); -// return SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_ADDRESS, value); -// } - -// Status -// Config::SetServerConfigPort(const std::string& value) { -// STATUS_CHECK(CheckServerConfigPort(value)); -// return SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_PORT, value); -// } - -// Status -// Config::SetServerConfigDeployMode(const std::string& value) { -// STATUS_CHECK(CheckServerConfigDeployMode(value)); -// return SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_DEPLOY_MODE, value); -// } - -// Status -// Config::SetServerConfigTimeZone(const std::string& value) { -// STATUS_CHECK(CheckServerConfigTimeZone(value)); -// return SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_TIME_ZONE, value); -// } - -// Status -// Config::SetServerConfigWebEnable(const std::string& value) { -// STATUS_CHECK(CheckServerConfigWebEnable(value)); -// return SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_WEB_ENABLE, value); -// } - -// Status -// Config::SetServerConfigWebPort(const std::string& value) { -// STATUS_CHECK(CheckServerConfigWebPort(value)); -// return SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_WEB_PORT, value); -// } - /* db config */ -// Status -// Config::SetDBConfigBackendUrl(const std::string& value) { -// STATUS_CHECK(CheckDBConfigBackendUrl(value)); -// return SetConfigValueInMem(CONFIG_DB, CONFIG_DB_BACKEND_URL, value); -// } - Status Config::SetDBConfigArchiveDiskThreshold(const std::string& value) { STATUS_CHECK(CheckDBConfigArchiveDiskThreshold(value)); diff --git a/core/src/config/Config.h b/core/src/config/Config.h index a39d0065..44aeab9e 100644 --- a/core/src/config/Config.h +++ b/core/src/config/Config.h @@ -55,8 +55,6 @@ extern const char* CONFIG_NETWORK_HTTP_PORT_DEFAULT; /* db config */ extern const char* CONFIG_DB; -// extern const char* CONFIG_DB_BACKEND_URL; -// extern const char* CONFIG_DB_BACKEND_URL_DEFAULT; extern const char* CONFIG_DB_ARCHIVE_DISK_THRESHOLD; extern const char* CONFIG_DB_ARCHIVE_DISK_THRESHOLD_DEFAULT; extern const char* CONFIG_DB_ARCHIVE_DAYS_THRESHOLD; @@ -226,8 +224,6 @@ class Config { CheckNetworkConfigHTTPPort(const std::string& value); /* db config */ - // Status - // CheckDBConfigBackendUrl(const std::string& value); Status CheckDBConfigArchiveDiskThreshold(const std::string& value); Status @@ -269,8 +265,8 @@ class Config { Status CheckEngineConfigSimdType(const std::string& value); -#ifdef MILVUS_GPU_VERSION /* gpu resource config */ +#ifdef MILVUS_GPU_VERSION Status CheckGpuResourceConfigEnable(const std::string& value); Status @@ -346,8 +342,6 @@ class Config { GetNetworkConfigHTTPPort(std::string& value); /* db config */ - // Status - // GetDBConfigBackendUrl(std::string& value); Status GetDBConfigArchiveDiskThreshold(int64_t& value); Status @@ -389,8 +383,8 @@ class Config { Status GetEngineConfigSimdType(std::string& value); -#ifdef MILVUS_GPU_VERSION /* gpu resource config */ +#ifdef MILVUS_GPU_VERSION Status GetGpuResourceConfigEnable(bool& value); Status @@ -458,8 +452,6 @@ class Config { SetNetworkConfigHTTPPort(const std::string& value); /* db config */ - // Status - // SetDBConfigBackendUrl(const std::string& value); Status SetDBConfigArchiveDiskThreshold(const std::string& value); Status @@ -500,9 +492,9 @@ class Config { SetEngineConfigOmpThreadNum(const std::string& value); Status SetEngineConfigSimdType(const std::string& value); -#ifdef MILVUS_GPU_VERSION /* gpu resource config */ +#ifdef MILVUS_GPU_VERSION Status SetGpuResourceConfigEnable(const std::string& value); Status diff --git a/core/src/db/Options.h b/core/src/db/Options.h index 91e7f91b..25bfaf76 100644 --- a/core/src/db/Options.h +++ b/core/src/db/Options.h @@ -57,7 +57,6 @@ struct ArchiveConf { struct DBMetaOptions { std::string path_; - std::vector slave_paths_; std::string backend_uri_; ArchiveConf archive_conf_ = ArchiveConf("delete"); }; // DBMetaOptions diff --git a/core/src/db/Utils.cpp b/core/src/db/Utils.cpp index d37a6f83..45021b27 100644 --- a/core/src/db/Utils.cpp +++ b/core/src/db/Utils.cpp @@ -35,9 +35,6 @@ namespace { const char* TABLES_FOLDER = "/tables/"; -uint64_t index_file_counter = 0; -std::mutex index_file_counter_mutex; - static std::string ConstructParentFolder(const std::string& db_path, const meta::SegmentSchema& table_file) { std::string table_path = db_path + TABLES_FOLDER + table_file.collection_id_; @@ -45,32 +42,6 @@ ConstructParentFolder(const std::string& db_path, const meta::SegmentSchema& tab return partition_path; } -static std::string -GetCollectionFileParentFolder(const DBMetaOptions& options, const meta::SegmentSchema& table_file) { - uint64_t path_count = options.slave_paths_.size() + 1; - std::string target_path = options.path_; - uint64_t index = 0; - - if (meta::SegmentSchema::NEW_INDEX == table_file.file_type_) { - // index file is large file and to be persisted permanently - // we need to distribute index files to each db_path averagely - // round robin according to a file counter - std::lock_guard lock(index_file_counter_mutex); - index = index_file_counter % path_count; - ++index_file_counter; - } else { - // for other type files, they could be merged or deleted - // so we round robin according to their file id - index = table_file.id_ % path_count; - } - - if (index > 0) { - target_path = options.slave_paths_[index - 1]; - } - - return ConstructParentFolder(target_path, table_file); -} - } // namespace int64_t @@ -90,34 +61,18 @@ CreateCollectionPath(const DBMetaOptions& options, const std::string& collection LOG_ENGINE_ERROR_ << status.message(); return status; } - - for (auto& path : options.slave_paths_) { - table_path = path + TABLES_FOLDER + collection_id; - status = server::CommonUtil::CreateDirectory(table_path); - fiu_do_on("CreateCollectionPath.creat_slave_path", status = Status(DB_INVALID_PATH, "")); - if (!status.ok()) { - LOG_ENGINE_ERROR_ << status.message(); - return status; - } - } - return Status::OK(); } Status DeleteCollectionPath(const DBMetaOptions& options, const std::string& collection_id, bool force) { - std::vector paths = options.slave_paths_; - paths.push_back(options.path_); - - for (auto& path : paths) { - std::string table_path = path + TABLES_FOLDER + collection_id; - if (force) { - boost::filesystem::remove_all(table_path); - LOG_ENGINE_DEBUG_ << "Remove collection folder: " << table_path; - } else if (boost::filesystem::exists(table_path) && boost::filesystem::is_empty(table_path)) { - boost::filesystem::remove_all(table_path); - LOG_ENGINE_DEBUG_ << "Remove collection folder: " << table_path; - } + std::string table_path = options.path_ + TABLES_FOLDER + collection_id; + if (force) { + boost::filesystem::remove_all(table_path); + LOG_ENGINE_DEBUG_ << "Remove collection folder: " << table_path; + } else if (boost::filesystem::exists(table_path) && boost::filesystem::is_empty(table_path)) { + boost::filesystem::remove_all(table_path); + LOG_ENGINE_DEBUG_ << "Remove collection folder: " << table_path; } // bool s3_enable = false; @@ -139,7 +94,7 @@ DeleteCollectionPath(const DBMetaOptions& options, const std::string& collection Status CreateCollectionFilePath(const DBMetaOptions& options, meta::SegmentSchema& table_file) { - std::string parent_path = GetCollectionFileParentFolder(options, table_file); + std::string parent_path = ConstructParentFolder(options.path_, table_file); auto status = server::CommonUtil::CreateDirectory(parent_path); fiu_do_on("CreateCollectionFilePath.fail_create", status = Status(DB_INVALID_PATH, "")); @@ -173,15 +128,6 @@ GetCollectionFilePath(const DBMetaOptions& options, meta::SegmentSchema& table_f return Status::OK(); } - for (auto& path : options.slave_paths_) { - parent_path = ConstructParentFolder(path, table_file); - file_path = parent_path + "/" + table_file.file_id_; - if (boost::filesystem::exists(parent_path)) { - table_file.location_ = file_path; - return Status::OK(); - } - } - std::string msg = "Collection file doesn't exist: " + file_path; if (table_file.file_size_ > 0) { // no need to pop error for empty file LOG_ENGINE_ERROR_ << msg << " in path: " << options.path_ << " for collection: " << table_file.collection_id_; diff --git a/core/src/server/DBWrapper.cpp b/core/src/server/DBWrapper.cpp index adc3acbf..0191600c 100644 --- a/core/src/server/DBWrapper.cpp +++ b/core/src/server/DBWrapper.cpp @@ -82,7 +82,6 @@ DBWrapper::StartService() { } opt.insert_buffer_size_ = insert_buffer_size; -#if 1 bool cluster_enable = false; std::string cluster_role; STATUS_CHECK(config.GetClusterConfigEnable(cluster_enable)); @@ -98,27 +97,6 @@ DBWrapper::StartService() { kill(0, SIGUSR1); } -#else - std::string mode; - s = config.GetServerConfigDeployMode(mode); - if (!s.ok()) { - std::cerr << s.ToString() << std::endl; - return s; - } - - if (mode == "single") { - opt.mode_ = engine::DBOptions::MODE::SINGLE; - } else if (mode == "cluster_readonly") { - opt.mode_ = engine::DBOptions::MODE::CLUSTER_READONLY; - } else if (mode == "cluster_writable") { - opt.mode_ = engine::DBOptions::MODE::CLUSTER_WRITABLE; - } else { - std::cerr << "Error: server_config.deploy_mode in server_config.yaml is not one of " - << "single, cluster_readonly, and cluster_writable." << std::endl; - kill(0, SIGUSR1); - } -#endif - // get wal configurations s = config.GetWalConfigEnable(opt.wal_enable_); if (!s.ok()) { @@ -214,16 +192,6 @@ DBWrapper::StartService() { kill(0, SIGUSR1); } - for (auto& path : opt.meta_.slave_paths_) { - s = CommonUtil::CreateDirectory(path); - if (!s.ok()) { - std::cerr << "Error: Failed to create database secondary path: " << path - << ". Possible reason: db_config.secondary_path is wrong in server_config.yaml or not available." - << std::endl; - kill(0, SIGUSR1); - } - } - // create db instance try { db_ = engine::DBFactory::Build(opt); diff --git a/core/src/server/Server.cpp b/core/src/server/Server.cpp index cdbdb088..1b1622c9 100644 --- a/core/src/server/Server.cpp +++ b/core/src/server/Server.cpp @@ -240,10 +240,6 @@ Server::Start() { STATUS_CHECK(config.GetClusterConfigEnable(cluster_enable)); STATUS_CHECK(config.GetClusterConfigRole(cluster_role)); - // std::string deploy_mode; - // STATUS_CHECK(config.GetServerConfigDeployMode(deploy_mode)); - - // if (deploy_mode == "single" || deploy_mode == "cluster_writable") { if ((not cluster_enable) || cluster_role == "rw") { std::string db_path; STATUS_CHECK(config.GetStorageConfigPath(db_path)); diff --git a/core/src/server/delivery/request/ReLoadSegmentsRequest.cpp b/core/src/server/delivery/request/ReLoadSegmentsRequest.cpp index f871ef5b..192e8988 100644 --- a/core/src/server/delivery/request/ReLoadSegmentsRequest.cpp +++ b/core/src/server/delivery/request/ReLoadSegmentsRequest.cpp @@ -37,7 +37,6 @@ Status ReLoadSegmentsRequest::OnExecute() { auto& config = Config::GetInstance(); -#if 1 bool cluster_enable = false; std::string cluster_role; STATUS_CHECK(config.GetClusterConfigEnable(cluster_enable)); @@ -47,19 +46,6 @@ ReLoadSegmentsRequest::OnExecute() { // TODO: No need to reload segment files return Status(SERVER_SUCCESS, ""); } -#else - std::string deploy_mode; - auto status = config.GetServerConfigDeployMode(deploy_mode); - if (!status.ok()) { - return status; - } - - fiu_do_on("ReLoadSegmentsRequest.OnExecute.readonly", deploy_mode = "cluster_readonly"); - if (deploy_mode == "single" || deploy_mode == "cluster_writable") { - // TODO: No need to reload segment files - return Status(SERVER_SUCCESS, ""); - } -#endif try { std::string hdr = "ReloadSegmentsRequest(collection=" + collection_name_ + ")"; diff --git a/core/src/server/init/StorageChecker.cpp b/core/src/server/init/StorageChecker.cpp index 431aba50..9175fa8f 100644 --- a/core/src/server/init/StorageChecker.cpp +++ b/core/src/server/init/StorageChecker.cpp @@ -44,7 +44,6 @@ StorageChecker::CheckStoragePermission() { return Status(SERVER_UNEXPECTED_ERROR, err_msg); } -#if 1 bool cluster_enable = false; std::string cluster_role; STATUS_CHECK(config.GetClusterConfigEnable(cluster_enable)); @@ -53,17 +52,6 @@ StorageChecker::CheckStoragePermission() { if (cluster_enable && cluster_role == "ro") { return Status::OK(); } -#else - std::string deploy_mode; - status = config.GetServerConfigDeployMode(deploy_mode); - if (!status.ok()) { - return status; - } - - if (deploy_mode == "cluster_readonly") { - return Status::OK(); - } -#endif /* Check db directory write permission */ std::string primary_path; diff --git a/core/unittest/db/test_misc.cpp b/core/unittest/db/test_misc.cpp index 6035bce7..ddd9ebfa 100644 --- a/core/unittest/db/test_misc.cpp +++ b/core/unittest/db/test_misc.cpp @@ -86,8 +86,6 @@ TEST(DBMiscTest, META_TEST) { TEST(DBMiscTest, UTILS_TEST) { milvus::engine::DBMetaOptions options; options.path_ = "/tmp/milvus_test/main"; - options.slave_paths_.push_back("/tmp/milvus_test/slave_1"); - options.slave_paths_.push_back("/tmp/milvus_test/slave_2"); const std::string COLLECTION_NAME = "test_tbl"; @@ -98,25 +96,9 @@ TEST(DBMiscTest, UTILS_TEST) { ASSERT_FALSE(status.ok()); fiu_disable("CommonUtil.CreateDirectory.create_parent_fail"); - FIU_ENABLE_FIU("CreateCollectionPath.creat_slave_path"); - status = milvus::engine::utils::CreateCollectionPath(options, COLLECTION_NAME); - ASSERT_FALSE(status.ok()); - fiu_disable("CreateCollectionPath.creat_slave_path"); - status = milvus::engine::utils::CreateCollectionPath(options, COLLECTION_NAME); ASSERT_TRUE(status.ok()); ASSERT_TRUE(boost::filesystem::exists(options.path_)); - for (auto& path : options.slave_paths_) { - ASSERT_TRUE(boost::filesystem::exists(path)); - } - - // options.slave_paths.push_back("/"); - // status = engine::utils::CreateCollectionPath(options, COLLECTION_NAME); - // ASSERT_FALSE(status.ok()); - // - // options.path = "/"; - // status = engine::utils::CreateCollectionPath(options, COLLECTION_NAME); - // ASSERT_FALSE(status.ok()); milvus::engine::meta::SegmentSchema file; file.id_ = 50; -- GitLab