diff --git a/CHANGELOG.md b/CHANGELOG.md index 98a7c949bd4eeca220e5ae6beac888775f8460bf..58b2d0837f03240bc6ac04c7d6b2913cd37697bb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,9 +27,12 @@ Please mark all change in change log and use the ticket from JIRA. - MS-654 - Describe index timeout when building index - MS-658 - Fix SQ8 Hybrid can't search - \#9 Change default gpu_cache_capacity to 4 +- MS-665 - IVF_SQ8H search crash when no GPU resource in search_resources - \#20 - C++ sdk example get grpc error - \#23 - Add unittest to improve code coverage - \#31 - make clang-format failed after run build.sh -l +- \#39 - Create SQ8H index hang if using github server version +- \#30 - Some troubleshoot messages in Milvus do not provide enough information ## Improvement - MS-552 - Add and change the easylogging library @@ -51,6 +54,7 @@ Please mark all change in change log and use the ticket from JIRA. - MS-626 - Refactor DataObj to support cache any type data - MS-648 - Improve unittest - MS-655 - Upgrade SPTAG +- \#42 - Put union of index_build_device and search resources to gpu_pool ## New Feature - MS-614 - Preload table at startup diff --git a/core/conf/server_config.template b/core/conf/server_config.template index 27c2b12cfe20baefeaf35cecc456ec9f06b51b04..7abfb8b05549e1024c3d8e194bae0f8e11c34f89 100644 --- a/core/conf/server_config.template +++ b/core/conf/server_config.template @@ -16,7 +16,6 @@ db_config: insert_buffer_size: 4 # GB, maximum insert buffer size allowed # sum of insert_buffer_size and cpu_cache_capacity cannot exceed total memory - build_index_gpu: 0 # gpu id used for building index preload_table: # preload data at startup, '*' means load all tables, empty value means no preload # you can specify preload tables like this: table1,table2,table3 @@ -39,6 +38,6 @@ engine_config: # if nq >= use_blas_threshold, use OpenBlas, slower with stable response times resource_config: - resource_pool: - - cpu + search_resources: # define the GPUs used for search computation, valid value: gpux - gpu0 + index_build_device: gpu0 # GPU used for building index \ No newline at end of file diff --git a/core/src/db/engine/ExecutionEngineImpl.cpp b/core/src/db/engine/ExecutionEngineImpl.cpp index bde45cf11268695a2ef14a72c7eecef94029c493..862c1026d2955d39abb4f6c9e61c1fbe98c1baba 100644 --- a/core/src/db/engine/ExecutionEngineImpl.cpp +++ b/core/src/db/engine/ExecutionEngineImpl.cpp @@ -65,7 +65,7 @@ ExecutionEngineImpl::ExecutionEngineImpl(uint16_t dimension, const std::string& : location_(location), dim_(dimension), index_type_(index_type), metric_type_(metric_type), nlist_(nlist) { index_ = CreatetVecIndex(EngineType::FAISS_IDMAP); if (!index_) { - throw Exception(DB_ERROR, "Could not create VecIndex"); + throw Exception(DB_ERROR, "Unsupported index type"); } TempMetaConf temp_conf; @@ -111,7 +111,7 @@ ExecutionEngineImpl::CreatetVecIndex(EngineType type) { break; } default: { - ENGINE_LOG_ERROR << "Invalid engine type"; + ENGINE_LOG_ERROR << "Unsupported index type"; return nullptr; } } @@ -373,7 +373,7 @@ ExecutionEngineImpl::BuildIndex(const std::string& location, EngineType engine_t auto to_index = CreatetVecIndex(engine_type); if (!to_index) { - throw Exception(DB_ERROR, "Could not create VecIndex"); + throw Exception(DB_ERROR, "Unsupported index type"); } TempMetaConf temp_conf; @@ -503,7 +503,7 @@ ExecutionEngineImpl::GpuCache(uint64_t gpu_id) { Status ExecutionEngineImpl::Init() { server::Config& config = server::Config::GetInstance(); - Status s = config.GetDBConfigBuildIndexGPU(gpu_num_); + Status s = config.GetResourceConfigIndexBuildDevice(gpu_num_); if (!s.ok()) { return s; } diff --git a/core/src/scheduler/SchedInst.cpp b/core/src/scheduler/SchedInst.cpp index 0053332746f84acaa5fbb04baab852855abeb0fd..1bd1ca00cfd274fd60c65eff51c97124227605a4 100644 --- a/core/src/scheduler/SchedInst.cpp +++ b/core/src/scheduler/SchedInst.cpp @@ -50,29 +50,38 @@ load_simple_config() { std::string mode; config.GetResourceConfigMode(mode); std::vector pool; - config.GetResourceConfigPool(pool); + config.GetResourceConfigSearchResources(pool); // get resources - bool use_cpu_to_compute = false; - for (auto& resource : pool) { - if (resource == "cpu") { - use_cpu_to_compute = true; - break; - } - } auto gpu_ids = get_gpu_pool(); + int32_t build_gpu_id; + config.GetResourceConfigIndexBuildDevice(build_gpu_id); + // create and connect ResMgrInst::GetInstance()->Add(ResourceFactory::Create("disk", "DISK", 0, true, false)); auto io = Connection("io", 500); - ResMgrInst::GetInstance()->Add(ResourceFactory::Create("cpu", "CPU", 0, true, use_cpu_to_compute)); + ResMgrInst::GetInstance()->Add(ResourceFactory::Create("cpu", "CPU", 0, true, true)); ResMgrInst::GetInstance()->Connect("disk", "cpu", io); auto pcie = Connection("pcie", 12000); + bool find_build_gpu_id = false; for (auto& gpu_id : gpu_ids) { ResMgrInst::GetInstance()->Add(ResourceFactory::Create(std::to_string(gpu_id), "GPU", gpu_id, true, true)); ResMgrInst::GetInstance()->Connect("cpu", std::to_string(gpu_id), pcie); + if (build_gpu_id == gpu_id) { + find_build_gpu_id = true; + } + } + + if (not find_build_gpu_id) { + ResMgrInst::GetInstance()->Add(ResourceFactory::Create(std::to_string(build_gpu_id), + "GPU", + build_gpu_id, + true, + true)); + ResMgrInst::GetInstance()->Connect("cpu", std::to_string(build_gpu_id), pcie); } } diff --git a/core/src/scheduler/Utils.cpp b/core/src/scheduler/Utils.cpp index 18f6fc249d72dbf387e7cbcdcbb44c19a16076ef..998e545ba5d00bcb0589478d352c1a0710d53dc1 100644 --- a/core/src/scheduler/Utils.cpp +++ b/core/src/scheduler/Utils.cpp @@ -48,7 +48,7 @@ get_gpu_pool() { server::Config& config = server::Config::GetInstance(); std::vector pool; - Status s = config.GetResourceConfigPool(pool); + Status s = config.GetResourceConfigSearchResources(pool); if (!s.ok()) { SERVER_LOG_ERROR << s.message(); } diff --git a/core/src/scheduler/action/PushTaskToNeighbour.cpp b/core/src/scheduler/action/PushTaskToNeighbour.cpp index 4e7dbf984fe747676b8ac1e71c6f5fbfebbb9602..c64e81dcfa8be202f340f06f5e2363cd93c3207f 100644 --- a/core/src/scheduler/action/PushTaskToNeighbour.cpp +++ b/core/src/scheduler/action/PushTaskToNeighbour.cpp @@ -184,11 +184,11 @@ Action::SpecifiedResourceLabelTaskScheduler(ResourceMgrWPtr res_mgr, ResourcePtr // get build index gpu resource server::Config& config = server::Config::GetInstance(); int32_t build_index_gpu; - Status stat = config.GetDBConfigBuildIndexGPU(build_index_gpu); + Status stat = config.GetResourceConfigIndexBuildDevice(build_index_gpu); bool find_gpu_res = false; - for (uint64_t i = 0; i < compute_resources.size(); ++i) { - if (res_mgr.lock()->GetResource(ResourceType::GPU, build_index_gpu) != nullptr) { + if (res_mgr.lock()->GetResource(ResourceType::GPU, build_index_gpu) != nullptr) { + for (uint64_t i = 0; i < compute_resources.size(); ++i) { if (compute_resources[i]->name() == res_mgr.lock()->GetResource(ResourceType::GPU, build_index_gpu)->name()) { find_gpu_res = true; diff --git a/core/src/scheduler/optimizer/Pass.h b/core/src/scheduler/optimizer/Pass.h index 959c3ea5ee83d63b2b8a044eeb7a102654826421..016b05e4570f958bedb54a215d1aeb099b735332 100644 --- a/core/src/scheduler/optimizer/Pass.h +++ b/core/src/scheduler/optimizer/Pass.h @@ -34,9 +34,9 @@ namespace scheduler { class Pass { public: - virtual void - Init() { - } + // virtual void + // Init() { + // } virtual bool Run(const TaskPtr& task) = 0; diff --git a/core/src/scheduler/task/BuildIndexTask.cpp b/core/src/scheduler/task/BuildIndexTask.cpp index 26a2a44efce75bbc77632e5547d657d2e51beb09..d8602c141e166510ffe9f1c8a9a9bd83470b6c39 100644 --- a/core/src/scheduler/task/BuildIndexTask.cpp +++ b/core/src/scheduler/task/BuildIndexTask.cpp @@ -134,6 +134,7 @@ XBuildIndexTask::Execute() { ENGINE_LOG_DEBUG << "Failed to update file to index, mark file: " << table_file.file_id_ << " to to_delete"; + build_index_job->BuildIndexDone(to_index_id_); to_index_engine_ = nullptr; return; } @@ -148,6 +149,7 @@ XBuildIndexTask::Execute() { std::cout << "ERROR: failed to build index, index file is too large or gpu memory is not enough" << std::endl; + build_index_job->BuildIndexDone(to_index_id_); build_index_job->GetStatus() = Status(DB_ERROR, msg); to_index_engine_ = nullptr; return; @@ -158,6 +160,9 @@ XBuildIndexTask::Execute() { meta_ptr->HasTable(file_->table_id_, has_table); if (!has_table) { meta_ptr->DeleteTableFiles(file_->table_id_); + + build_index_job->BuildIndexDone(to_index_id_); + build_index_job->GetStatus() = Status(DB_ERROR, "Table has been deleted, discard index file."); to_index_engine_ = nullptr; return; } @@ -177,6 +182,7 @@ XBuildIndexTask::Execute() { std::cout << "ERROR: failed to persist index file: " << table_file.location_ << ", possible out of disk space" << std::endl; + build_index_job->BuildIndexDone(to_index_id_); build_index_job->GetStatus() = Status(DB_ERROR, msg); to_index_engine_ = nullptr; return; diff --git a/core/src/server/Config.cpp b/core/src/server/Config.cpp index 189070eb2be3a7047cdfccfc8dd156a81b3b8dff..7de84cbccc6944b4845c5fceaee751fc27e9d923 100644 --- a/core/src/server/Config.cpp +++ b/core/src/server/Config.cpp @@ -18,6 +18,7 @@ #include #include #include +#include #include #include @@ -129,12 +130,6 @@ Config::ValidateConfig() { return s; } - int32_t db_build_index_gpu; - s = GetDBConfigBuildIndexGPU(db_build_index_gpu); - if (!s.ok()) { - return s; - } - /* metric config */ bool metric_enable_monitor; s = GetMetricConfigEnableMonitor(metric_enable_monitor); @@ -205,8 +200,14 @@ Config::ValidateConfig() { return s; } - std::vector resource_pool; - s = GetResourceConfigPool(resource_pool); + std::vector search_resources; + s = GetResourceConfigSearchResources(search_resources); + if (!s.ok()) { + return s; + } + + int32_t resource_index_build_device; + s = GetResourceConfigIndexBuildDevice(resource_index_build_device); if (!s.ok()) { return s; } @@ -270,11 +271,6 @@ Config::ResetDefaultConfig() { return s; } - s = SetDBConfigBuildIndexGPU(CONFIG_DB_BUILD_INDEX_GPU_DEFAULT); - if (!s.ok()) { - return s; - } - /* metric config */ s = SetMetricConfigEnableMonitor(CONFIG_METRIC_ENABLE_MONITOR_DEFAULT); if (!s.ok()) { @@ -334,6 +330,11 @@ Config::ResetDefaultConfig() { return s; } + s = SetResourceConfigIndexBuildDevice(CONFIG_RESOURCE_INDEX_BUILD_DEVICE_DEFAULT); + if (!s.ok()) { + return s; + } + return Status::OK(); } @@ -459,19 +460,6 @@ Config::CheckDBConfigInsertBufferSize(const std::string& value) { return Status::OK(); } -Status -Config::CheckDBConfigBuildIndexGPU(const std::string& value) { - if (!ValidationUtil::ValidateStringIsNumber(value).ok()) { - return Status(SERVER_INVALID_ARGUMENT, "Invalid DB config build_index_gpu: " + value); - } else { - int32_t gpu_index = std::stoi(value); - if (!ValidationUtil::ValidateGpuIndex(gpu_index).ok()) { - return Status(SERVER_INVALID_ARGUMENT, "Invalid DB config build_index_gpu: " + value); - } - } - return Status::OK(); -} - Status Config::CheckMetricConfigEnableMonitor(const std::string& value) { if (!ValidationUtil::ValidateStringIsBool(value).ok()) { @@ -544,7 +532,7 @@ Config::CheckCacheConfigGpuCacheCapacity(const std::string& value) { } else { uint64_t gpu_cache_capacity = std::stoi(value) * GB; int gpu_index; - Status s = GetDBConfigBuildIndexGPU(gpu_index); + Status s = GetResourceConfigIndexBuildDevice(gpu_index); if (!s.ok()) { return s; } @@ -616,9 +604,38 @@ Config::CheckResourceConfigMode(const std::string& value) { } Status -Config::CheckResourceConfigPool(const std::vector& value) { +CheckGpuDevice(const std::string& value) { + const std::regex pat("gpu(\\d+)"); + std::cmatch m; + if (!std::regex_match(value.c_str(), m, pat)) { + return Status(SERVER_INVALID_ARGUMENT, "Invalid gpu device: " + value); + } + + int32_t gpu_index = std::stoi(value.substr(3)); + if (!ValidationUtil::ValidateGpuIndex(gpu_index).ok()) { + return Status(SERVER_INVALID_ARGUMENT, "Invalid gpu device: " + value); + } + return Status::OK(); +} + +Status +Config::CheckResourceConfigSearchResources(const std::vector& value) { if (value.empty()) { - return Status(SERVER_INVALID_ARGUMENT, "Invalid resource config pool"); + return Status(SERVER_INVALID_ARGUMENT, "Empty resource config search_resources"); + } + + for (auto& gpu_device : value) { + if (!CheckGpuDevice(gpu_device).ok()) { + return Status(SERVER_INVALID_ARGUMENT, "Invalid resource config search_resources: " + gpu_device); + } + } + return Status::OK(); +} + +Status +Config::CheckResourceConfigIndexBuildDevice(const std::string& value) { + if (!CheckGpuDevice(value).ok()) { + return Status(SERVER_INVALID_ARGUMENT, "Invalid resource config index_build_device: " + value); } return Status::OK(); } @@ -739,18 +756,6 @@ Config::GetDBConfigInsertBufferSize(int32_t& value) { return Status::OK(); } -Status -Config::GetDBConfigBuildIndexGPU(int32_t& value) { - std::string str = GetConfigStr(CONFIG_DB, CONFIG_DB_BUILD_INDEX_GPU, CONFIG_DB_BUILD_INDEX_GPU_DEFAULT); - Status s = CheckDBConfigBuildIndexGPU(str); - if (!s.ok()) { - return s; - } - - value = std::stoi(str); - return Status::OK(); -} - Status Config::GetDBConfigPreloadTable(std::string& value) { value = GetConfigStr(CONFIG_DB, CONFIG_DB_PRELOAD_TABLE); @@ -880,10 +885,23 @@ Config::GetResourceConfigMode(std::string& value) { } Status -Config::GetResourceConfigPool(std::vector& value) { +Config::GetResourceConfigSearchResources(std::vector& value) { ConfigNode resource_config = GetConfigNode(CONFIG_RESOURCE); - value = resource_config.GetSequence(CONFIG_RESOURCE_POOL); - return CheckResourceConfigPool(value); + value = resource_config.GetSequence(CONFIG_RESOURCE_SEARCH_RESOURCES); + return CheckResourceConfigSearchResources(value); +} + +Status +Config::GetResourceConfigIndexBuildDevice(int32_t& value) { + std::string str = + GetConfigStr(CONFIG_RESOURCE, CONFIG_RESOURCE_INDEX_BUILD_DEVICE, CONFIG_RESOURCE_INDEX_BUILD_DEVICE_DEFAULT); + Status s = CheckResourceConfigIndexBuildDevice(str); + if (!s.ok()) { + return s; + } + + value = std::stoi(str.substr(3)); + return Status::OK(); } /////////////////////////////////////////////////////////////////////////////// @@ -999,17 +1017,6 @@ Config::SetDBConfigInsertBufferSize(const std::string& value) { return Status::OK(); } -Status -Config::SetDBConfigBuildIndexGPU(const std::string& value) { - Status s = CheckDBConfigBuildIndexGPU(value); - if (!s.ok()) { - return s; - } - - SetConfigValueInMem(CONFIG_DB, CONFIG_DB_BUILD_INDEX_GPU, value); - return Status::OK(); -} - /* metric config */ Status Config::SetMetricConfigEnableMonitor(const std::string& value) { @@ -1135,5 +1142,16 @@ Config::SetResourceConfigMode(const std::string& value) { return Status::OK(); } +Status +Config::SetResourceConfigIndexBuildDevice(const std::string& value) { + Status s = CheckResourceConfigIndexBuildDevice(value); + if (!s.ok()) { + return s; + } + + SetConfigValueInMem(CONFIG_DB, CONFIG_RESOURCE_INDEX_BUILD_DEVICE, value); + return Status::OK(); +} + } // namespace server } // namespace milvus diff --git a/core/src/server/Config.h b/core/src/server/Config.h index f40c9f2224cb6688562d43e400482bfb5669c852..4cab25a1c64da77977077dd401e9b5ee4516f9c5 100644 --- a/core/src/server/Config.h +++ b/core/src/server/Config.h @@ -53,8 +53,6 @@ static const char* CONFIG_DB_ARCHIVE_DAYS_THRESHOLD = "archive_days_threshold"; static const char* CONFIG_DB_ARCHIVE_DAYS_THRESHOLD_DEFAULT = "0"; static const char* CONFIG_DB_INSERT_BUFFER_SIZE = "insert_buffer_size"; static const char* CONFIG_DB_INSERT_BUFFER_SIZE_DEFAULT = "4"; -static const char* CONFIG_DB_BUILD_INDEX_GPU = "build_index_gpu"; -static const char* CONFIG_DB_BUILD_INDEX_GPU_DEFAULT = "0"; static const char* CONFIG_DB_PRELOAD_TABLE = "preload_table"; /* cache config */ @@ -91,7 +89,9 @@ static const char* CONFIG_ENGINE_OMP_THREAD_NUM_DEFAULT = "0"; static const char* CONFIG_RESOURCE = "resource_config"; static const char* CONFIG_RESOURCE_MODE = "mode"; static const char* CONFIG_RESOURCE_MODE_DEFAULT = "simple"; -static const char* CONFIG_RESOURCE_POOL = "resource_pool"; +static const char* CONFIG_RESOURCE_SEARCH_RESOURCES = "search_resources"; +static const char* CONFIG_RESOURCE_INDEX_BUILD_DEVICE = "index_build_device"; +static const char* CONFIG_RESOURCE_INDEX_BUILD_DEVICE_DEFAULT = "gpu0"; class Config { public: @@ -140,8 +140,6 @@ class Config { CheckDBConfigArchiveDaysThreshold(const std::string& value); Status CheckDBConfigInsertBufferSize(const std::string& value); - Status - CheckDBConfigBuildIndexGPU(const std::string& value); /* metric config */ Status @@ -173,7 +171,9 @@ class Config { Status CheckResourceConfigMode(const std::string& value); Status - CheckResourceConfigPool(const std::vector& value); + CheckResourceConfigSearchResources(const std::vector& value); + Status + CheckResourceConfigIndexBuildDevice(const std::string& value); std::string GetConfigStr(const std::string& parent_key, const std::string& child_key, const std::string& default_value = ""); @@ -203,8 +203,6 @@ class Config { Status GetDBConfigInsertBufferSize(int32_t& value); Status - GetDBConfigBuildIndexGPU(int32_t& value); - Status GetDBConfigPreloadTable(std::string& value); /* metric config */ @@ -237,7 +235,9 @@ class Config { Status GetResourceConfigMode(std::string& value); Status - GetResourceConfigPool(std::vector& value); + GetResourceConfigSearchResources(std::vector& value); + Status + GetResourceConfigIndexBuildDevice(int32_t& value); public: /* server config */ @@ -263,8 +263,6 @@ class Config { SetDBConfigArchiveDaysThreshold(const std::string& value); Status SetDBConfigInsertBufferSize(const std::string& value); - Status - SetDBConfigBuildIndexGPU(const std::string& value); /* metric config */ Status @@ -295,6 +293,8 @@ class Config { /* resource config */ Status SetResourceConfigMode(const std::string& value); + Status + SetResourceConfigIndexBuildDevice(const std::string& value); private: std::unordered_map> config_map_; diff --git a/core/src/server/grpc_impl/GrpcRequestTask.cpp b/core/src/server/grpc_impl/GrpcRequestTask.cpp index 02cb24175aa20352e349f404dc4877e59378abef..86ff23b3d07e78c70d7860353a6433892f679ff1 100644 --- a/core/src/server/grpc_impl/GrpcRequestTask.cpp +++ b/core/src/server/grpc_impl/GrpcRequestTask.cpp @@ -113,6 +113,14 @@ ConvertTimeRangeToDBDates(const std::vector<::milvus::grpc::Range>& range_array, return Status::OK(); } + +std::string +TableNotExistMsg(const std::string& table_name) { + return "Table " + table_name + + " not exist. Use milvus.has_table to verify whether the table exists. You also can check if the table name " + "exists."; +} + } // namespace //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -255,7 +263,7 @@ CreateIndexTask::OnExecute() { } if (!has_table) { - return Status(SERVER_TABLE_NOT_EXIST, "Table " + table_name_ + " not exists"); + return Status(SERVER_TABLE_NOT_EXIST, TableNotExistMsg(table_name_)); } auto& grpc_index = index_param_->index(); @@ -348,7 +356,7 @@ DropTableTask::OnExecute() { status = DBWrapper::DB()->DescribeTable(table_info); if (!status.ok()) { if (status.code() == DB_NOT_FOUND) { - return Status(SERVER_TABLE_NOT_EXIST, "Table " + table_name_ + " not exists"); + return Status(SERVER_TABLE_NOT_EXIST, TableNotExistMsg(table_name_)); } else { return status; } @@ -420,12 +428,14 @@ InsertTask::OnExecute() { return status; } if (insert_param_->row_record_array().empty()) { - return Status(SERVER_INVALID_ROWRECORD_ARRAY, "Row record array is empty"); + return Status(SERVER_INVALID_ROWRECORD_ARRAY, + "The vector array is empty. Make sure you have entered vector records."); } if (!insert_param_->row_id_array().empty()) { if (insert_param_->row_id_array().size() != insert_param_->row_record_array_size()) { - return Status(SERVER_ILLEGAL_VECTOR_ID, "Size of vector ids is not equal to row record array size"); + return Status(SERVER_ILLEGAL_VECTOR_ID, + "The size of vector ID array must be equal to the size of the vector."); } } @@ -435,7 +445,7 @@ InsertTask::OnExecute() { status = DBWrapper::DB()->DescribeTable(table_info); if (!status.ok()) { if (status.code() == DB_NOT_FOUND) { - return Status(SERVER_TABLE_NOT_EXIST, "Table " + insert_param_->table_name() + " not exists"); + return Status(SERVER_TABLE_NOT_EXIST, TableNotExistMsg(insert_param_->table_name())); } else { return status; } @@ -447,13 +457,14 @@ InsertTask::OnExecute() { // user already provided id before, all insert action require user id if ((table_info.flag_ & engine::meta::FLAG_MASK_HAS_USERID) != 0 && !user_provide_ids) { return Status(SERVER_ILLEGAL_VECTOR_ID, - "Table vector ids are user defined, please provide id for this batch"); + "Table vector IDs are user-defined. Please provide IDs for all vectors of this table."); } // user didn't provided id before, no need to provide user id if ((table_info.flag_ & engine::meta::FLAG_MASK_NO_USERID) != 0 && user_provide_ids) { - return Status(SERVER_ILLEGAL_VECTOR_ID, - "Table vector ids are auto generated, no need to provide id for this batch"); + return Status( + SERVER_ILLEGAL_VECTOR_ID, + "Table vector IDs are auto-generated. All vectors of this table must use auto-generated IDs."); } rc.RecordSection("check validation"); @@ -470,13 +481,13 @@ InsertTask::OnExecute() { // TODO(yk): change to one dimension array or use multiple-thread to copy the data for (size_t i = 0; i < insert_param_->row_record_array_size(); i++) { if (insert_param_->row_record_array(i).vector_data().empty()) { - return Status(SERVER_INVALID_ROWRECORD_ARRAY, "Row record array data is empty"); + return Status(SERVER_INVALID_ROWRECORD_ARRAY, + "The vector dimension must be equal to the table dimension."); } uint64_t vec_dim = insert_param_->row_record_array(i).vector_data().size(); if (vec_dim != table_info.dimension_) { ErrorCode error_code = SERVER_INVALID_VECTOR_DIMENSION; - std::string error_msg = "Invalid row record dimension: " + std::to_string(vec_dim) + - " vs. table dimension:" + std::to_string(table_info.dimension_); + std::string error_msg = "The vector dimension must be equal to the table dimension."; return Status(error_code, error_msg); } memcpy(&vec_f[i * table_info.dimension_], insert_param_->row_record_array(i).vector_data().data(), @@ -569,7 +580,7 @@ SearchTask::OnExecute() { status = DBWrapper::DB()->DescribeTable(table_info); if (!status.ok()) { if (status.code() == DB_NOT_FOUND) { - return Status(SERVER_TABLE_NOT_EXIST, "Table " + table_name_ + " not exists"); + return Status(SERVER_TABLE_NOT_EXIST, TableNotExistMsg(table_name_)); } else { return status; } @@ -587,7 +598,8 @@ SearchTask::OnExecute() { } if (search_param_->query_record_array().empty()) { - return Status(SERVER_INVALID_ROWRECORD_ARRAY, "Row record array is empty"); + return Status(SERVER_INVALID_ROWRECORD_ARRAY, + "The vector array is empty. Make sure you have entered vector records."); } // step 4: check date range, and convert to db dates @@ -609,13 +621,13 @@ SearchTask::OnExecute() { std::vector vec_f(record_array_size * table_info.dimension_, 0); for (size_t i = 0; i < record_array_size; i++) { if (search_param_->query_record_array(i).vector_data().empty()) { - return Status(SERVER_INVALID_ROWRECORD_ARRAY, "Row record array data is empty"); + return Status(SERVER_INVALID_ROWRECORD_ARRAY, + "The vector dimension must be equal to the table dimension."); } uint64_t query_vec_dim = search_param_->query_record_array(i).vector_data().size(); if (query_vec_dim != table_info.dimension_) { ErrorCode error_code = SERVER_INVALID_VECTOR_DIMENSION; - std::string error_msg = "Invalid row record dimension: " + std::to_string(query_vec_dim) + - " vs. table dimension:" + std::to_string(table_info.dimension_); + std::string error_msg = "The vector dimension must be equal to the table dimension."; return Status(error_code, error_msg); } @@ -707,7 +719,7 @@ CountTableTask::OnExecute() { status = DBWrapper::DB()->GetTableRowCount(table_name_, row_count); if (!status.ok()) { if (status.code(), DB_NOT_FOUND) { - return Status(SERVER_TABLE_NOT_EXIST, "Table " + table_name_ + " not exists"); + return Status(SERVER_TABLE_NOT_EXIST, TableNotExistMsg(table_name_)); } else { return status; } @@ -779,7 +791,7 @@ DeleteByRangeTask::OnExecute() { status = DBWrapper::DB()->DescribeTable(table_info); if (!status.ok()) { if (status.code(), DB_NOT_FOUND) { - return Status(SERVER_TABLE_NOT_EXIST, "Table " + table_name + " not exists"); + return Status(SERVER_TABLE_NOT_EXIST, TableNotExistMsg(table_name)); } else { return status; } @@ -917,7 +929,7 @@ DropIndexTask::OnExecute() { } if (!has_table) { - return Status(SERVER_TABLE_NOT_EXIST, "Table " + table_name_ + " not exists"); + return Status(SERVER_TABLE_NOT_EXIST, TableNotExistMsg(table_name_)); } // step 2: check table existence diff --git a/core/src/utils/ValidationUtil.cpp b/core/src/utils/ValidationUtil.cpp index b982a31f5e18cfa36bea170ae068715d5476f37b..68088d6c932359b111b3d1e5a389fac0110d652a 100644 --- a/core/src/utils/ValidationUtil.cpp +++ b/core/src/utils/ValidationUtil.cpp @@ -37,14 +37,15 @@ Status ValidationUtil::ValidateTableName(const std::string& table_name) { // Table name shouldn't be empty. if (table_name.empty()) { - std::string msg = "Empty table name"; + std::string msg = "Table name should not be empty."; SERVER_LOG_ERROR << msg; return Status(SERVER_INVALID_TABLE_NAME, msg); } + std::string invalid_msg = "Invalid table name: " + table_name + ". "; // Table name size shouldn't exceed 16384. if (table_name.size() > TABLE_NAME_SIZE_LIMIT) { - std::string msg = "Table name size exceed the limitation"; + std::string msg = invalid_msg + "The length of a table name must be less than 255 characters."; SERVER_LOG_ERROR << msg; return Status(SERVER_INVALID_TABLE_NAME, msg); } @@ -52,7 +53,7 @@ ValidationUtil::ValidateTableName(const std::string& table_name) { // Table name first character should be underscore or character. char first_char = table_name[0]; if (first_char != '_' && std::isalpha(first_char) == 0) { - std::string msg = "Table name first character isn't underscore or character"; + std::string msg = invalid_msg + "The first character of a table name must be an underscore or letter."; SERVER_LOG_ERROR << msg; return Status(SERVER_INVALID_TABLE_NAME, msg); } @@ -61,7 +62,7 @@ ValidationUtil::ValidateTableName(const std::string& table_name) { for (int64_t i = 1; i < table_name_size; ++i) { char name_char = table_name[i]; if (name_char != '_' && std::isalnum(name_char) == 0) { - std::string msg = "Table name character isn't underscore or alphanumber"; + std::string msg = invalid_msg + "Table name can only contain numbers, letters, and underscores."; SERVER_LOG_ERROR << msg; return Status(SERVER_INVALID_TABLE_NAME, msg); } @@ -72,12 +73,9 @@ ValidationUtil::ValidateTableName(const std::string& table_name) { Status ValidationUtil::ValidateTableDimension(int64_t dimension) { - if (dimension <= 0) { - std::string msg = "Dimension value should be greater than 0"; - SERVER_LOG_ERROR << msg; - return Status(SERVER_INVALID_VECTOR_DIMENSION, msg); - } else if (dimension > TABLE_DIMENSION_LIMIT) { - std::string msg = "Table dimension excceed the limitation: " + std::to_string(TABLE_DIMENSION_LIMIT); + if (dimension <= 0 || dimension > TABLE_DIMENSION_LIMIT) { + std::string msg = "Invalid table dimension: " + std::to_string(dimension) + ". " + + "The table dimension must be within the range of 1 ~ 16384."; SERVER_LOG_ERROR << msg; return Status(SERVER_INVALID_VECTOR_DIMENSION, msg); } else { @@ -89,10 +87,20 @@ Status ValidationUtil::ValidateTableIndexType(int32_t index_type) { int engine_type = static_cast(engine::EngineType(index_type)); if (engine_type <= 0 || engine_type > static_cast(engine::EngineType::MAX_VALUE)) { - std::string msg = "Invalid index type: " + std::to_string(index_type); + std::string msg = "Invalid index type: " + std::to_string(index_type) + ". " + + "Make sure the index type is in IndexType list."; + SERVER_LOG_ERROR << msg; + return Status(SERVER_INVALID_INDEX_TYPE, msg); + } + +#ifndef CUSTOMIZATION + // special case, hybird index only available in customize faiss library + if (engine_type == static_cast(engine::EngineType::FAISS_IVFSQ8H)) { + std::string msg = "Unsupported index type: " + std::to_string(index_type); SERVER_LOG_ERROR << msg; return Status(SERVER_INVALID_INDEX_TYPE, msg); } +#endif return Status::OK(); } @@ -100,7 +108,8 @@ ValidationUtil::ValidateTableIndexType(int32_t index_type) { Status ValidationUtil::ValidateTableIndexNlist(int32_t nlist) { if (nlist <= 0) { - std::string msg = "nlist value should be greater than 0"; + std::string msg = + "Invalid index nlist: " + std::to_string(nlist) + ". " + "The index nlist must be greater than 0."; SERVER_LOG_ERROR << msg; return Status(SERVER_INVALID_INDEX_NLIST, msg); } @@ -111,7 +120,9 @@ ValidationUtil::ValidateTableIndexNlist(int32_t nlist) { Status ValidationUtil::ValidateTableIndexFileSize(int64_t index_file_size) { if (index_file_size <= 0 || index_file_size > INDEX_FILE_SIZE_LIMIT) { - std::string msg = "Invalid index file size: " + std::to_string(index_file_size); + std::string msg = "Invalid index file size: " + std::to_string(index_file_size) + ". " + + "The index file size must be within the range of 1 ~ " + + std::to_string(INDEX_FILE_SIZE_LIMIT) + "."; SERVER_LOG_ERROR << msg; return Status(SERVER_INVALID_INDEX_FILE_SIZE, msg); } @@ -123,7 +134,8 @@ Status ValidationUtil::ValidateTableIndexMetricType(int32_t metric_type) { if (metric_type != static_cast(engine::MetricType::L2) && metric_type != static_cast(engine::MetricType::IP)) { - std::string msg = "Invalid metric type: " + std::to_string(metric_type); + std::string msg = "Invalid index metric type: " + std::to_string(metric_type) + ". " + + "Make sure the metric type is either MetricType.L2 or MetricType.IP."; SERVER_LOG_ERROR << msg; return Status(SERVER_INVALID_INDEX_METRIC_TYPE, msg); } @@ -133,7 +145,8 @@ ValidationUtil::ValidateTableIndexMetricType(int32_t metric_type) { Status ValidationUtil::ValidateSearchTopk(int64_t top_k, const engine::meta::TableSchema& table_schema) { if (top_k <= 0 || top_k > 2048) { - std::string msg = "Invalid top k value: " + std::to_string(top_k) + ", rational range [1, 2048]"; + std::string msg = + "Invalid topk: " + std::to_string(top_k) + ". " + "The topk must be within the range of 1 ~ 2048."; SERVER_LOG_ERROR << msg; return Status(SERVER_INVALID_TOPK, msg); } @@ -144,8 +157,8 @@ ValidationUtil::ValidateSearchTopk(int64_t top_k, const engine::meta::TableSchem Status ValidationUtil::ValidateSearchNprobe(int64_t nprobe, const engine::meta::TableSchema& table_schema) { if (nprobe <= 0 || nprobe > table_schema.nlist_) { - std::string msg = "Invalid nprobe value: " + std::to_string(nprobe) + ", rational range [1, " + - std::to_string(table_schema.nlist_) + "]"; + std::string msg = "Invalid nprobe: " + std::to_string(nprobe) + ". " + + "The nprobe must be within the range of 1 ~ index nlist."; SERVER_LOG_ERROR << msg; return Status(SERVER_INVALID_NPROBE, msg); } diff --git a/core/src/wrapper/KnowhereResource.cpp b/core/src/wrapper/KnowhereResource.cpp index d291bb92998bd3d20b1c04351b71af8d2fe7f20e..650ae727c1ce735d2bd0e5ea2d784b21546c1f57 100644 --- a/core/src/wrapper/KnowhereResource.cpp +++ b/core/src/wrapper/KnowhereResource.cpp @@ -45,7 +45,7 @@ KnowhereResource::Initialize() { server::Config& config = server::Config::GetInstance(); int32_t build_index_gpu; - s = config.GetDBConfigBuildIndexGPU(build_index_gpu); + s = config.GetResourceConfigIndexBuildDevice(build_index_gpu); if (!s.ok()) return s; @@ -53,7 +53,7 @@ KnowhereResource::Initialize() { // get search gpu resource std::vector pool; - s = config.GetResourceConfigPool(pool); + s = config.GetResourceConfigSearchResources(pool); if (!s.ok()) return s; diff --git a/core/unittest/server/appendix/server_config.yaml b/core/unittest/server/appendix/server_config.yaml index f92b2f1a1891d4f3959fecd7ca811935776b08fb..c5aef825116868b37105333e4c7d8f06cb14e655 100644 --- a/core/unittest/server/appendix/server_config.yaml +++ b/core/unittest/server/appendix/server_config.yaml @@ -1,4 +1,4 @@ -# All the following configurations are default values. +# Default values are used when you make no changes to the following parameters. server_config: address: 0.0.0.0 # milvus server ip address (IPv4) @@ -11,27 +11,33 @@ db_config: secondary_path: # path used to store data only, split by semicolon backend_url: sqlite://:@:/ # URI format: dialect://username:password@host:port/database - # Keep 'dialect://:@:/', and replace other texts with real values. + # Keep 'dialect://:@:/', and replace other texts with real values # Replace 'dialect' with 'mysql' or 'sqlite' insert_buffer_size: 4 # GB, maximum insert buffer size allowed - build_index_gpu: 0 # gpu id used for building index + # sum of insert_buffer_size and cpu_cache_capacity cannot exceed total memory + + preload_table: # preload data at startup, '*' means load all tables, empty value means no preload + # you can specify preload tables like this: table1,table2,table3 metric_config: enable_monitor: false # enable monitoring or not collector: prometheus # prometheus prometheus_config: - port: 8080 # port prometheus used to fetch metrics + port: 8080 # port prometheus uses to fetch metrics cache_config: - cpu_mem_capacity: 16 # GB, CPU memory used for cache - cpu_mem_threshold: 0.85 # percentage of data kept when cache cleanup triggered - cache_insert_data: false # whether load inserted data into cache + cpu_cache_capacity: 16 # GB, CPU memory used for cache + cpu_cache_threshold: 0.85 # percentage of data that will be kept when cache cleanup is triggered + gpu_cache_capacity: 4 # GB, GPU memory used for cache + gpu_cache_threshold: 0.85 # percentage of data that will be kept when cache cleanup is triggered + cache_insert_data: false # whether to load inserted data into cache engine_config: - blas_threshold: 20 + use_blas_threshold: 20 # if nq < use_blas_threshold, use SSE, faster with fluctuated response times + # if nq >= use_blas_threshold, use OpenBlas, slower with stable response times resource_config: - resource_pool: - - cpu + search_resources: # define the GPUs used for search computation, valid value: gpux - gpu0 + index_build_device: gpu0 # GPU used for building index diff --git a/core/unittest/wrapper/test_wrapper.cpp b/core/unittest/wrapper/test_wrapper.cpp index e16a4e2bb243c5d12ed1f9be62d4ac3aed95e218..f112fc7e65a07012f55062a244f4f5786bd67dda 100644 --- a/core/unittest/wrapper/test_wrapper.cpp +++ b/core/unittest/wrapper/test_wrapper.cpp @@ -164,3 +164,14 @@ TEST_P(KnowhereWrapperTest, SERIALIZE_TEST) { AssertResult(res_ids, res_dis); } } + +#include "wrapper/ConfAdapter.h" +TEST(whatever, test_config) { + milvus::engine::TempMetaConf conf; + auto nsg_conf = std::make_shared(); + nsg_conf->Match(conf); + nsg_conf->MatchSearch(conf, milvus::engine::IndexType::FAISS_IVFPQ_GPU); + + auto pq_conf = std::make_shared(); + pq_conf->Match(conf); +} diff --git a/core/unittest/wrapper/utils.h b/core/unittest/wrapper/utils.h index b895b599e026a5cf77717f485cdb0744ddfa4617..5a614543c9da819fe3e76105b5b33164eee1d865 100644 --- a/core/unittest/wrapper/utils.h +++ b/core/unittest/wrapper/utils.h @@ -104,28 +104,28 @@ class ParamGenerator { tempconf->metric_type = knowhere::METRICTYPE::L2; return tempconf; } - case milvus::engine::IndexType::FAISS_IVFPQ_CPU: - case milvus::engine::IndexType::FAISS_IVFPQ_GPU: { - auto tempconf = std::make_shared(); - tempconf->nlist = 100; - tempconf->nprobe = 16; - tempconf->nbits = 8; - tempconf->m = 8; - tempconf->metric_type = knowhere::METRICTYPE::L2; - return tempconf; - } - case milvus::engine::IndexType::NSG_MIX: { - auto tempconf = std::make_shared(); - tempconf->nlist = 100; - tempconf->nprobe = 16; - tempconf->search_length = 8; - tempconf->knng = 200; - tempconf->search_length = 40; // TODO(linxj): be 20 when search - tempconf->out_degree = 60; - tempconf->candidate_pool_size = 200; - tempconf->metric_type = knowhere::METRICTYPE::L2; - return tempconf; - } +// case milvus::engine::IndexType::FAISS_IVFPQ_CPU: +// case milvus::engine::IndexType::FAISS_IVFPQ_GPU: { +// auto tempconf = std::make_shared(); +// tempconf->nlist = 100; +// tempconf->nprobe = 16; +// tempconf->nbits = 8; +// tempconf->m = 8; +// tempconf->metric_type = knowhere::METRICTYPE::L2; +// return tempconf; +// } +// case milvus::engine::IndexType::NSG_MIX: { +// auto tempconf = std::make_shared(); +// tempconf->nlist = 100; +// tempconf->nprobe = 16; +// tempconf->search_length = 8; +// tempconf->knng = 200; +// tempconf->search_length = 40; // TODO(linxj): be 20 when search +// tempconf->out_degree = 60; +// tempconf->candidate_pool_size = 200; +// tempconf->metric_type = knowhere::METRICTYPE::L2; +// return tempconf; +// } } } };