提交 b17d2afa 编写于 作者: P peng.xu

Merge branch '0.5.0' into '0.5.0'

MS-665 rename build_index_gpu to index_build_device, and move it to resource config

See merge request megasearch/milvus!764

Former-commit-id: 887719e3c3fc943e11bde77b99b71d55b970cfa6
......@@ -26,6 +26,7 @@ Please mark all change in change log and use the ticket from JIRA.
- MS-653 - When config check fail, Milvus close without message
- MS-654 - Describe index timeout when building index
- MS-658 - Fix SQ8 Hybrid can't search
- MS-665 - IVF_SQ8H search crash when no GPU resource in search_resources
- \#20 - C++ sdk example get grpc error
- \#23 - Add unittest to improve code coverage
- \#31 - make clang-format failed after run build.sh -l
......
......@@ -16,7 +16,6 @@ db_config:
insert_buffer_size: 4 # GB, maximum insert buffer size allowed
# sum of insert_buffer_size and cpu_cache_capacity cannot exceed total memory
build_index_gpu: 0 # gpu id used for building index
preload_table: # preload data at startup, '*' means load all tables, empty value means no preload
# you can specify preload tables like this: table1,table2,table3
......@@ -39,6 +38,6 @@ engine_config:
# if nq >= use_blas_threshold, use OpenBlas, slower with stable response times
resource_config:
resource_pool:
- cpu
search_resources: # define the GPUs used for search computation, valid value: gpux
- gpu0
index_build_device: gpu0 # GPU used for building index
\ No newline at end of file
......@@ -503,7 +503,7 @@ ExecutionEngineImpl::GpuCache(uint64_t gpu_id) {
Status
ExecutionEngineImpl::Init() {
server::Config& config = server::Config::GetInstance();
Status s = config.GetDBConfigBuildIndexGPU(gpu_num_);
Status s = config.GetResourceConfigIndexBuildDevice(gpu_num_);
if (!s.ok()) {
return s;
}
......
......@@ -50,7 +50,7 @@ load_simple_config() {
std::string mode;
config.GetResourceConfigMode(mode);
std::vector<std::string> pool;
config.GetResourceConfigPool(pool);
config.GetResourceConfigSearchResources(pool);
// get resources
bool use_cpu_to_compute = false;
......
......@@ -48,7 +48,7 @@ get_gpu_pool() {
server::Config& config = server::Config::GetInstance();
std::vector<std::string> pool;
Status s = config.GetResourceConfigPool(pool);
Status s = config.GetResourceConfigSearchResources(pool);
if (!s.ok()) {
SERVER_LOG_ERROR << s.message();
}
......
......@@ -184,7 +184,7 @@ Action::SpecifiedResourceLabelTaskScheduler(ResourceMgrWPtr res_mgr, ResourcePtr
// get build index gpu resource
server::Config& config = server::Config::GetInstance();
int32_t build_index_gpu;
Status stat = config.GetDBConfigBuildIndexGPU(build_index_gpu);
Status stat = config.GetResourceConfigIndexBuildDevice(build_index_gpu);
bool find_gpu_res = false;
for (uint64_t i = 0; i < compute_resources.size(); ++i) {
......
......@@ -18,6 +18,7 @@
#include <sys/stat.h>
#include <algorithm>
#include <iostream>
#include <regex>
#include <string>
#include <vector>
......@@ -129,12 +130,6 @@ Config::ValidateConfig() {
return s;
}
int32_t db_build_index_gpu;
s = GetDBConfigBuildIndexGPU(db_build_index_gpu);
if (!s.ok()) {
return s;
}
/* metric config */
bool metric_enable_monitor;
s = GetMetricConfigEnableMonitor(metric_enable_monitor);
......@@ -205,8 +200,14 @@ Config::ValidateConfig() {
return s;
}
std::vector<std::string> resource_pool;
s = GetResourceConfigPool(resource_pool);
std::vector<std::string> search_resources;
s = GetResourceConfigSearchResources(search_resources);
if (!s.ok()) {
return s;
}
int32_t resource_index_build_device;
s = GetResourceConfigIndexBuildDevice(resource_index_build_device);
if (!s.ok()) {
return s;
}
......@@ -270,11 +271,6 @@ Config::ResetDefaultConfig() {
return s;
}
s = SetDBConfigBuildIndexGPU(CONFIG_DB_BUILD_INDEX_GPU_DEFAULT);
if (!s.ok()) {
return s;
}
/* metric config */
s = SetMetricConfigEnableMonitor(CONFIG_METRIC_ENABLE_MONITOR_DEFAULT);
if (!s.ok()) {
......@@ -334,6 +330,11 @@ Config::ResetDefaultConfig() {
return s;
}
s = SetResourceConfigIndexBuildDevice(CONFIG_RESOURCE_INDEX_BUILD_DEVICE_DEFAULT);
if (!s.ok()) {
return s;
}
return Status::OK();
}
......@@ -459,19 +460,6 @@ Config::CheckDBConfigInsertBufferSize(const std::string& value) {
return Status::OK();
}
Status
Config::CheckDBConfigBuildIndexGPU(const std::string& value) {
if (!ValidationUtil::ValidateStringIsNumber(value).ok()) {
return Status(SERVER_INVALID_ARGUMENT, "Invalid DB config build_index_gpu: " + value);
} else {
int32_t gpu_index = std::stoi(value);
if (!ValidationUtil::ValidateGpuIndex(gpu_index).ok()) {
return Status(SERVER_INVALID_ARGUMENT, "Invalid DB config build_index_gpu: " + value);
}
}
return Status::OK();
}
Status
Config::CheckMetricConfigEnableMonitor(const std::string& value) {
if (!ValidationUtil::ValidateStringIsBool(value).ok()) {
......@@ -544,7 +532,7 @@ Config::CheckCacheConfigGpuCacheCapacity(const std::string& value) {
} else {
uint64_t gpu_cache_capacity = std::stoi(value) * GB;
int gpu_index;
Status s = GetDBConfigBuildIndexGPU(gpu_index);
Status s = GetResourceConfigIndexBuildDevice(gpu_index);
if (!s.ok()) {
return s;
}
......@@ -616,9 +604,38 @@ Config::CheckResourceConfigMode(const std::string& value) {
}
Status
Config::CheckResourceConfigPool(const std::vector<std::string>& value) {
CheckGpuDevice(const std::string& value) {
const std::regex pat("gpu(\\d+)");
std::cmatch m;
if (!std::regex_match(value.c_str(), m, pat)) {
return Status(SERVER_INVALID_ARGUMENT, "Invalid gpu device: " + value);
}
int32_t gpu_index = std::stoi(value.substr(3));
if (!ValidationUtil::ValidateGpuIndex(gpu_index).ok()) {
return Status(SERVER_INVALID_ARGUMENT, "Invalid gpu device: " + value);
}
return Status::OK();
}
Status
Config::CheckResourceConfigSearchResources(const std::vector<std::string>& value) {
if (value.empty()) {
return Status(SERVER_INVALID_ARGUMENT, "Invalid resource config pool");
return Status(SERVER_INVALID_ARGUMENT, "Empty resource config search_resources");
}
for (auto& gpu_device : value) {
if (!CheckGpuDevice(gpu_device).ok()) {
return Status(SERVER_INVALID_ARGUMENT, "Invalid resource config search_resources: " + gpu_device);
}
}
return Status::OK();
}
Status
Config::CheckResourceConfigIndexBuildDevice(const std::string& value) {
if (!CheckGpuDevice(value).ok()) {
return Status(SERVER_INVALID_ARGUMENT, "Invalid resource config index_build_device: " + value);
}
return Status::OK();
}
......@@ -739,18 +756,6 @@ Config::GetDBConfigInsertBufferSize(int32_t& value) {
return Status::OK();
}
Status
Config::GetDBConfigBuildIndexGPU(int32_t& value) {
std::string str = GetConfigStr(CONFIG_DB, CONFIG_DB_BUILD_INDEX_GPU, CONFIG_DB_BUILD_INDEX_GPU_DEFAULT);
Status s = CheckDBConfigBuildIndexGPU(str);
if (!s.ok()) {
return s;
}
value = std::stoi(str);
return Status::OK();
}
Status
Config::GetDBConfigPreloadTable(std::string& value) {
value = GetConfigStr(CONFIG_DB, CONFIG_DB_PRELOAD_TABLE);
......@@ -880,10 +885,23 @@ Config::GetResourceConfigMode(std::string& value) {
}
Status
Config::GetResourceConfigPool(std::vector<std::string>& value) {
Config::GetResourceConfigSearchResources(std::vector<std::string>& value) {
ConfigNode resource_config = GetConfigNode(CONFIG_RESOURCE);
value = resource_config.GetSequence(CONFIG_RESOURCE_POOL);
return CheckResourceConfigPool(value);
value = resource_config.GetSequence(CONFIG_RESOURCE_SEARCH_RESOURCES);
return CheckResourceConfigSearchResources(value);
}
Status
Config::GetResourceConfigIndexBuildDevice(int32_t& value) {
std::string str =
GetConfigStr(CONFIG_RESOURCE, CONFIG_RESOURCE_INDEX_BUILD_DEVICE, CONFIG_RESOURCE_INDEX_BUILD_DEVICE_DEFAULT);
Status s = CheckResourceConfigIndexBuildDevice(str);
if (!s.ok()) {
return s;
}
value = std::stoi(str.substr(3));
return Status::OK();
}
///////////////////////////////////////////////////////////////////////////////
......@@ -999,17 +1017,6 @@ Config::SetDBConfigInsertBufferSize(const std::string& value) {
return Status::OK();
}
Status
Config::SetDBConfigBuildIndexGPU(const std::string& value) {
Status s = CheckDBConfigBuildIndexGPU(value);
if (!s.ok()) {
return s;
}
SetConfigValueInMem(CONFIG_DB, CONFIG_DB_BUILD_INDEX_GPU, value);
return Status::OK();
}
/* metric config */
Status
Config::SetMetricConfigEnableMonitor(const std::string& value) {
......@@ -1135,5 +1142,16 @@ Config::SetResourceConfigMode(const std::string& value) {
return Status::OK();
}
Status
Config::SetResourceConfigIndexBuildDevice(const std::string& value) {
Status s = CheckResourceConfigIndexBuildDevice(value);
if (!s.ok()) {
return s;
}
SetConfigValueInMem(CONFIG_DB, CONFIG_RESOURCE_INDEX_BUILD_DEVICE, value);
return Status::OK();
}
} // namespace server
} // namespace milvus
......@@ -53,8 +53,6 @@ static const char* CONFIG_DB_ARCHIVE_DAYS_THRESHOLD = "archive_days_threshold";
static const char* CONFIG_DB_ARCHIVE_DAYS_THRESHOLD_DEFAULT = "0";
static const char* CONFIG_DB_INSERT_BUFFER_SIZE = "insert_buffer_size";
static const char* CONFIG_DB_INSERT_BUFFER_SIZE_DEFAULT = "4";
static const char* CONFIG_DB_BUILD_INDEX_GPU = "build_index_gpu";
static const char* CONFIG_DB_BUILD_INDEX_GPU_DEFAULT = "0";
static const char* CONFIG_DB_PRELOAD_TABLE = "preload_table";
/* cache config */
......@@ -91,7 +89,9 @@ static const char* CONFIG_ENGINE_OMP_THREAD_NUM_DEFAULT = "0";
static const char* CONFIG_RESOURCE = "resource_config";
static const char* CONFIG_RESOURCE_MODE = "mode";
static const char* CONFIG_RESOURCE_MODE_DEFAULT = "simple";
static const char* CONFIG_RESOURCE_POOL = "resource_pool";
static const char* CONFIG_RESOURCE_SEARCH_RESOURCES = "search_resources";
static const char* CONFIG_RESOURCE_INDEX_BUILD_DEVICE = "index_build_device";
static const char* CONFIG_RESOURCE_INDEX_BUILD_DEVICE_DEFAULT = "gpu0";
class Config {
public:
......@@ -140,8 +140,6 @@ class Config {
CheckDBConfigArchiveDaysThreshold(const std::string& value);
Status
CheckDBConfigInsertBufferSize(const std::string& value);
Status
CheckDBConfigBuildIndexGPU(const std::string& value);
/* metric config */
Status
......@@ -173,7 +171,9 @@ class Config {
Status
CheckResourceConfigMode(const std::string& value);
Status
CheckResourceConfigPool(const std::vector<std::string>& value);
CheckResourceConfigSearchResources(const std::vector<std::string>& value);
Status
CheckResourceConfigIndexBuildDevice(const std::string& value);
std::string
GetConfigStr(const std::string& parent_key, const std::string& child_key, const std::string& default_value = "");
......@@ -203,8 +203,6 @@ class Config {
Status
GetDBConfigInsertBufferSize(int32_t& value);
Status
GetDBConfigBuildIndexGPU(int32_t& value);
Status
GetDBConfigPreloadTable(std::string& value);
/* metric config */
......@@ -237,7 +235,9 @@ class Config {
Status
GetResourceConfigMode(std::string& value);
Status
GetResourceConfigPool(std::vector<std::string>& value);
GetResourceConfigSearchResources(std::vector<std::string>& value);
Status
GetResourceConfigIndexBuildDevice(int32_t& value);
public:
/* server config */
......@@ -263,8 +263,6 @@ class Config {
SetDBConfigArchiveDaysThreshold(const std::string& value);
Status
SetDBConfigInsertBufferSize(const std::string& value);
Status
SetDBConfigBuildIndexGPU(const std::string& value);
/* metric config */
Status
......@@ -295,6 +293,8 @@ class Config {
/* resource config */
Status
SetResourceConfigMode(const std::string& value);
Status
SetResourceConfigIndexBuildDevice(const std::string& value);
private:
std::unordered_map<std::string, std::unordered_map<std::string, std::string>> config_map_;
......
......@@ -45,7 +45,7 @@ KnowhereResource::Initialize() {
server::Config& config = server::Config::GetInstance();
int32_t build_index_gpu;
s = config.GetDBConfigBuildIndexGPU(build_index_gpu);
s = config.GetResourceConfigIndexBuildDevice(build_index_gpu);
if (!s.ok())
return s;
......@@ -53,7 +53,7 @@ KnowhereResource::Initialize() {
// get search gpu resource
std::vector<std::string> pool;
s = config.GetResourceConfigPool(pool);
s = config.GetResourceConfigSearchResources(pool);
if (!s.ok())
return s;
......
# All the following configurations are default values.
# Default values are used when you make no changes to the following parameters.
server_config:
address: 0.0.0.0 # milvus server ip address (IPv4)
......@@ -11,27 +11,33 @@ db_config:
secondary_path: # path used to store data only, split by semicolon
backend_url: sqlite://:@:/ # URI format: dialect://username:password@host:port/database
# Keep 'dialect://:@:/', and replace other texts with real values.
# Keep 'dialect://:@:/', and replace other texts with real values
# Replace 'dialect' with 'mysql' or 'sqlite'
insert_buffer_size: 4 # GB, maximum insert buffer size allowed
build_index_gpu: 0 # gpu id used for building index
# sum of insert_buffer_size and cpu_cache_capacity cannot exceed total memory
preload_table: # preload data at startup, '*' means load all tables, empty value means no preload
# you can specify preload tables like this: table1,table2,table3
metric_config:
enable_monitor: false # enable monitoring or not
collector: prometheus # prometheus
prometheus_config:
port: 8080 # port prometheus used to fetch metrics
port: 8080 # port prometheus uses to fetch metrics
cache_config:
cpu_mem_capacity: 16 # GB, CPU memory used for cache
cpu_mem_threshold: 0.85 # percentage of data kept when cache cleanup triggered
cache_insert_data: false # whether load inserted data into cache
cpu_cache_capacity: 16 # GB, CPU memory used for cache
cpu_cache_threshold: 0.85 # percentage of data that will be kept when cache cleanup is triggered
gpu_cache_capacity: 4 # GB, GPU memory used for cache
gpu_cache_threshold: 0.85 # percentage of data that will be kept when cache cleanup is triggered
cache_insert_data: false # whether to load inserted data into cache
engine_config:
blas_threshold: 20
use_blas_threshold: 20 # if nq < use_blas_threshold, use SSE, faster with fluctuated response times
# if nq >= use_blas_threshold, use OpenBlas, slower with stable response times
resource_config:
resource_pool:
- cpu
search_resources: # define the GPUs used for search computation, valid value: gpux
- gpu0
index_build_device: gpu0 # GPU used for building index
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册