Merge remote-tracking branch 'upstream/0.5.0' into 0.5.0

Former-commit-id: f90856809e386ecdce5bb88210d895e141e53d82

CHANGELOG.md

@@ -27,9 +27,12 @@ Please mark all change in change log and use the ticket from JIRA.
 - MS-654 - Describe index timeout when building index
 - MS-658 - Fix SQ8 Hybrid can't search
 - \#9 Change default gpu_cache_capacity to 4
+- MS-665 - IVF_SQ8H search crash when no GPU resource in search_resources
 - \#20 - C++ sdk example get grpc error 
 - \#23 - Add unittest to improve code coverage
 - \#31 - make clang-format failed after run build.sh -l
+- \#39 - Create SQ8H index hang if using github server version
+- \#30 - Some troubleshoot messages in Milvus do not provide enough information
 
 ## Improvement
 - MS-552 - Add and change the easylogging library
@@ -51,6 +54,7 @@ Please mark all change in change log and use the ticket from JIRA.
 - MS-626 - Refactor DataObj to support cache any type data
 - MS-648 - Improve unittest
 - MS-655 - Upgrade SPTAG
+- \#42 - Put union of index_build_device and search resources to gpu_pool
 
 ## New Feature
 - MS-614 - Preload table at startup

core/conf/server_config.template

@@ -16,7 +16,6 @@ db_config:
 
   insert_buffer_size: 4             # GB, maximum insert buffer size allowed
                                     # sum of insert_buffer_size and cpu_cache_capacity cannot exceed total memory
-  build_index_gpu: 0                # gpu id used for building index
 
   preload_table:                    # preload data at startup, '*' means load all tables, empty value means no preload
                                     # you can specify preload tables like this: table1,table2,table3
@@ -39,6 +38,6 @@ engine_config:
                                     # if nq >= use_blas_threshold, use OpenBlas, slower with stable response times
 
 resource_config:
-  resource_pool:
-    - cpu
+  search_resources:                 # define the GPUs used for search computation, valid value: gpux
     - gpu0
+  index_build_device: gpu0          # GPU used for building index
\ No newline at end of file

core/src/db/engine/ExecutionEngineImpl.cpp

@@ -65,7 +65,7 @@ ExecutionEngineImpl::ExecutionEngineImpl(uint16_t dimension, const std::string&
     : location_(location), dim_(dimension), index_type_(index_type), metric_type_(metric_type), nlist_(nlist) {
     index_ = CreatetVecIndex(EngineType::FAISS_IDMAP);
     if (!index_) {
-        throw Exception(DB_ERROR, "Could not create VecIndex");
+        throw Exception(DB_ERROR, "Unsupported index type");
     }
 
     TempMetaConf temp_conf;
@@ -111,7 +111,7 @@ ExecutionEngineImpl::CreatetVecIndex(EngineType type) {
             break;
         }
         default: {
-            ENGINE_LOG_ERROR << "Invalid engine type";
+            ENGINE_LOG_ERROR << "Unsupported index type";
             return nullptr;
         }
     }
@@ -373,7 +373,7 @@ ExecutionEngineImpl::BuildIndex(const std::string& location, EngineType engine_t
 
     auto to_index = CreatetVecIndex(engine_type);
     if (!to_index) {
-        throw Exception(DB_ERROR, "Could not create VecIndex");
+        throw Exception(DB_ERROR, "Unsupported index type");
     }
 
     TempMetaConf temp_conf;
@@ -503,7 +503,7 @@ ExecutionEngineImpl::GpuCache(uint64_t gpu_id) {
 Status
 ExecutionEngineImpl::Init() {
     server::Config& config = server::Config::GetInstance();
-    Status s = config.GetDBConfigBuildIndexGPU(gpu_num_);
+    Status s = config.GetResourceConfigIndexBuildDevice(gpu_num_);
     if (!s.ok()) {
         return s;
     }

core/src/scheduler/SchedInst.cpp

@@ -50,29 +50,38 @@ load_simple_config() {
     std::string mode;
     config.GetResourceConfigMode(mode);
     std::vector<std::string> pool;
-    config.GetResourceConfigPool(pool);
+    config.GetResourceConfigSearchResources(pool);
 
     // get resources
-    bool use_cpu_to_compute = false;
-    for (auto& resource : pool) {
-        if (resource == "cpu") {
-            use_cpu_to_compute = true;
-            break;
-        }
-    }
     auto gpu_ids = get_gpu_pool();
 
+    int32_t build_gpu_id;
+    config.GetResourceConfigIndexBuildDevice(build_gpu_id);
+
     // create and connect
     ResMgrInst::GetInstance()->Add(ResourceFactory::Create("disk", "DISK", 0, true, false));
 
     auto io = Connection("io", 500);
-    ResMgrInst::GetInstance()->Add(ResourceFactory::Create("cpu", "CPU", 0, true, use_cpu_to_compute));
+    ResMgrInst::GetInstance()->Add(ResourceFactory::Create("cpu", "CPU", 0, true, true));
     ResMgrInst::GetInstance()->Connect("disk", "cpu", io);
 
     auto pcie = Connection("pcie", 12000);
+    bool find_build_gpu_id = false;
     for (auto& gpu_id : gpu_ids) {
         ResMgrInst::GetInstance()->Add(ResourceFactory::Create(std::to_string(gpu_id), "GPU", gpu_id, true, true));
         ResMgrInst::GetInstance()->Connect("cpu", std::to_string(gpu_id), pcie);
+        if (build_gpu_id == gpu_id) {
+            find_build_gpu_id = true;
+        }
+    }
+
+    if (not find_build_gpu_id) {
+        ResMgrInst::GetInstance()->Add(ResourceFactory::Create(std::to_string(build_gpu_id),
+                                                               "GPU",
+                                                               build_gpu_id,
+                                                               true,
+                                                               true));
+        ResMgrInst::GetInstance()->Connect("cpu", std::to_string(build_gpu_id), pcie);
     }
 }
 

core/src/scheduler/Utils.cpp

@@ -48,7 +48,7 @@ get_gpu_pool() {
 
     server::Config& config = server::Config::GetInstance();
     std::vector<std::string> pool;
-    Status s = config.GetResourceConfigPool(pool);
+    Status s = config.GetResourceConfigSearchResources(pool);
     if (!s.ok()) {
         SERVER_LOG_ERROR << s.message();
     }

core/src/scheduler/action/PushTaskToNeighbour.cpp

@@ -184,11 +184,11 @@ Action::SpecifiedResourceLabelTaskScheduler(ResourceMgrWPtr res_mgr, ResourcePtr
             // get build index gpu resource
             server::Config& config = server::Config::GetInstance();
             int32_t build_index_gpu;
-            Status stat = config.GetDBConfigBuildIndexGPU(build_index_gpu);
+            Status stat = config.GetResourceConfigIndexBuildDevice(build_index_gpu);
 
             bool find_gpu_res = false;
-            for (uint64_t i = 0; i < compute_resources.size(); ++i) {
-                if (res_mgr.lock()->GetResource(ResourceType::GPU, build_index_gpu) != nullptr) {
+            if (res_mgr.lock()->GetResource(ResourceType::GPU, build_index_gpu) != nullptr) {
+                for (uint64_t i = 0; i < compute_resources.size(); ++i) {
                     if (compute_resources[i]->name() ==
                         res_mgr.lock()->GetResource(ResourceType::GPU, build_index_gpu)->name()) {
                         find_gpu_res = true;

core/src/scheduler/optimizer/Pass.h

@@ -34,9 +34,9 @@ namespace scheduler {
 
 class Pass {
  public:
-    virtual void
-    Init() {
-    }
+    //    virtual void
+    //    Init() {
+    //    }
 
     virtual bool
     Run(const TaskPtr& task) = 0;

core/src/scheduler/task/BuildIndexTask.cpp

@@ -134,6 +134,7 @@ XBuildIndexTask::Execute() {
                 ENGINE_LOG_DEBUG << "Failed to update file to index, mark file: " << table_file.file_id_
                                  << " to to_delete";
 
+                build_index_job->BuildIndexDone(to_index_id_);
                 to_index_engine_ = nullptr;
                 return;
             }
@@ -148,6 +149,7 @@ XBuildIndexTask::Execute() {
             std::cout << "ERROR: failed to build index, index file is too large or gpu memory is not enough"
                       << std::endl;
 
+            build_index_job->BuildIndexDone(to_index_id_);
             build_index_job->GetStatus() = Status(DB_ERROR, msg);
             to_index_engine_ = nullptr;
             return;
@@ -158,6 +160,9 @@ XBuildIndexTask::Execute() {
         meta_ptr->HasTable(file_->table_id_, has_table);
         if (!has_table) {
             meta_ptr->DeleteTableFiles(file_->table_id_);
+
+            build_index_job->BuildIndexDone(to_index_id_);
+            build_index_job->GetStatus() = Status(DB_ERROR, "Table has been deleted, discard index file.");
             to_index_engine_ = nullptr;
             return;
         }
@@ -177,6 +182,7 @@ XBuildIndexTask::Execute() {
             std::cout << "ERROR: failed to persist index file: " << table_file.location_
                       << ", possible out of disk space" << std::endl;
 
+            build_index_job->BuildIndexDone(to_index_id_);
             build_index_job->GetStatus() = Status(DB_ERROR, msg);
             to_index_engine_ = nullptr;
             return;

core/src/server/Config.cpp

@@ -18,6 +18,7 @@
 #include <sys/stat.h>
 #include <algorithm>
 #include <iostream>
+#include <regex>
 #include <string>
 #include <vector>
 
@@ -129,12 +130,6 @@ Config::ValidateConfig() {
         return s;
     }
 
-    int32_t db_build_index_gpu;
-    s = GetDBConfigBuildIndexGPU(db_build_index_gpu);
-    if (!s.ok()) {
-        return s;
-    }
-
     /* metric config */
     bool metric_enable_monitor;
     s = GetMetricConfigEnableMonitor(metric_enable_monitor);
@@ -205,8 +200,14 @@ Config::ValidateConfig() {
         return s;
     }
 
-    std::vector<std::string> resource_pool;
-    s = GetResourceConfigPool(resource_pool);
+    std::vector<std::string> search_resources;
+    s = GetResourceConfigSearchResources(search_resources);
+    if (!s.ok()) {
+        return s;
+    }
+
+    int32_t resource_index_build_device;
+    s = GetResourceConfigIndexBuildDevice(resource_index_build_device);
     if (!s.ok()) {
         return s;
     }
@@ -270,11 +271,6 @@ Config::ResetDefaultConfig() {
         return s;
     }
 
-    s = SetDBConfigBuildIndexGPU(CONFIG_DB_BUILD_INDEX_GPU_DEFAULT);
-    if (!s.ok()) {
-        return s;
-    }
-
     /* metric config */
     s = SetMetricConfigEnableMonitor(CONFIG_METRIC_ENABLE_MONITOR_DEFAULT);
     if (!s.ok()) {
@@ -334,6 +330,11 @@ Config::ResetDefaultConfig() {
         return s;
     }
 
+    s = SetResourceConfigIndexBuildDevice(CONFIG_RESOURCE_INDEX_BUILD_DEVICE_DEFAULT);
+    if (!s.ok()) {
+        return s;
+    }
+
     return Status::OK();
 }
 
@@ -459,19 +460,6 @@ Config::CheckDBConfigInsertBufferSize(const std::string& value) {
     return Status::OK();
 }
 
-Status
-Config::CheckDBConfigBuildIndexGPU(const std::string& value) {
-    if (!ValidationUtil::ValidateStringIsNumber(value).ok()) {
-        return Status(SERVER_INVALID_ARGUMENT, "Invalid DB config build_index_gpu: " + value);
-    } else {
-        int32_t gpu_index = std::stoi(value);
-        if (!ValidationUtil::ValidateGpuIndex(gpu_index).ok()) {
-            return Status(SERVER_INVALID_ARGUMENT, "Invalid DB config build_index_gpu: " + value);
-        }
-    }
-    return Status::OK();
-}
-
 Status
 Config::CheckMetricConfigEnableMonitor(const std::string& value) {
     if (!ValidationUtil::ValidateStringIsBool(value).ok()) {
@@ -544,7 +532,7 @@ Config::CheckCacheConfigGpuCacheCapacity(const std::string& value) {
     } else {
         uint64_t gpu_cache_capacity = std::stoi(value) * GB;
         int gpu_index;
-        Status s = GetDBConfigBuildIndexGPU(gpu_index);
+        Status s = GetResourceConfigIndexBuildDevice(gpu_index);
         if (!s.ok()) {
             return s;
         }
@@ -616,9 +604,38 @@ Config::CheckResourceConfigMode(const std::string& value) {
 }
 
 Status
-Config::CheckResourceConfigPool(const std::vector<std::string>& value) {
+CheckGpuDevice(const std::string& value) {
+    const std::regex pat("gpu(\\d+)");
+    std::cmatch m;
+    if (!std::regex_match(value.c_str(), m, pat)) {
+        return Status(SERVER_INVALID_ARGUMENT, "Invalid gpu device: " + value);
+    }
+
+    int32_t gpu_index = std::stoi(value.substr(3));
+    if (!ValidationUtil::ValidateGpuIndex(gpu_index).ok()) {
+        return Status(SERVER_INVALID_ARGUMENT, "Invalid gpu device: " + value);
+    }
+    return Status::OK();
+}
+
+Status
+Config::CheckResourceConfigSearchResources(const std::vector<std::string>& value) {
     if (value.empty()) {
-        return Status(SERVER_INVALID_ARGUMENT, "Invalid resource config pool");
+        return Status(SERVER_INVALID_ARGUMENT, "Empty resource config search_resources");
+    }
+
+    for (auto& gpu_device : value) {
+        if (!CheckGpuDevice(gpu_device).ok()) {
+            return Status(SERVER_INVALID_ARGUMENT, "Invalid resource config search_resources: " + gpu_device);
+        }
+    }
+    return Status::OK();
+}
+
+Status
+Config::CheckResourceConfigIndexBuildDevice(const std::string& value) {
+    if (!CheckGpuDevice(value).ok()) {
+        return Status(SERVER_INVALID_ARGUMENT, "Invalid resource config index_build_device: " + value);
     }
     return Status::OK();
 }
@@ -739,18 +756,6 @@ Config::GetDBConfigInsertBufferSize(int32_t& value) {
     return Status::OK();
 }
 
-Status
-Config::GetDBConfigBuildIndexGPU(int32_t& value) {
-    std::string str = GetConfigStr(CONFIG_DB, CONFIG_DB_BUILD_INDEX_GPU, CONFIG_DB_BUILD_INDEX_GPU_DEFAULT);
-    Status s = CheckDBConfigBuildIndexGPU(str);
-    if (!s.ok()) {
-        return s;
-    }
-
-    value = std::stoi(str);
-    return Status::OK();
-}
-
 Status
 Config::GetDBConfigPreloadTable(std::string& value) {
     value = GetConfigStr(CONFIG_DB, CONFIG_DB_PRELOAD_TABLE);
@@ -880,10 +885,23 @@ Config::GetResourceConfigMode(std::string& value) {
 }
 
 Status
-Config::GetResourceConfigPool(std::vector<std::string>& value) {
+Config::GetResourceConfigSearchResources(std::vector<std::string>& value) {
     ConfigNode resource_config = GetConfigNode(CONFIG_RESOURCE);
-    value = resource_config.GetSequence(CONFIG_RESOURCE_POOL);
-    return CheckResourceConfigPool(value);
+    value = resource_config.GetSequence(CONFIG_RESOURCE_SEARCH_RESOURCES);
+    return CheckResourceConfigSearchResources(value);
+}
+
+Status
+Config::GetResourceConfigIndexBuildDevice(int32_t& value) {
+    std::string str =
+        GetConfigStr(CONFIG_RESOURCE, CONFIG_RESOURCE_INDEX_BUILD_DEVICE, CONFIG_RESOURCE_INDEX_BUILD_DEVICE_DEFAULT);
+    Status s = CheckResourceConfigIndexBuildDevice(str);
+    if (!s.ok()) {
+        return s;
+    }
+
+    value = std::stoi(str.substr(3));
+    return Status::OK();
 }
 
 ///////////////////////////////////////////////////////////////////////////////
@@ -999,17 +1017,6 @@ Config::SetDBConfigInsertBufferSize(const std::string& value) {
     return Status::OK();
 }
 
-Status
-Config::SetDBConfigBuildIndexGPU(const std::string& value) {
-    Status s = CheckDBConfigBuildIndexGPU(value);
-    if (!s.ok()) {
-        return s;
-    }
-
-    SetConfigValueInMem(CONFIG_DB, CONFIG_DB_BUILD_INDEX_GPU, value);
-    return Status::OK();
-}
-
 /* metric config */
 Status
 Config::SetMetricConfigEnableMonitor(const std::string& value) {
@@ -1135,5 +1142,16 @@ Config::SetResourceConfigMode(const std::string& value) {
     return Status::OK();
 }
 
+Status
+Config::SetResourceConfigIndexBuildDevice(const std::string& value) {
+    Status s = CheckResourceConfigIndexBuildDevice(value);
+    if (!s.ok()) {
+        return s;
+    }
+
+    SetConfigValueInMem(CONFIG_DB, CONFIG_RESOURCE_INDEX_BUILD_DEVICE, value);
+    return Status::OK();
+}
+
 }  // namespace server
 }  // namespace milvus

core/src/server/Config.h

@@ -53,8 +53,6 @@ static const char* CONFIG_DB_ARCHIVE_DAYS_THRESHOLD = "archive_days_threshold";
 static const char* CONFIG_DB_ARCHIVE_DAYS_THRESHOLD_DEFAULT = "0";
 static const char* CONFIG_DB_INSERT_BUFFER_SIZE = "insert_buffer_size";
 static const char* CONFIG_DB_INSERT_BUFFER_SIZE_DEFAULT = "4";
-static const char* CONFIG_DB_BUILD_INDEX_GPU = "build_index_gpu";
-static const char* CONFIG_DB_BUILD_INDEX_GPU_DEFAULT = "0";
 static const char* CONFIG_DB_PRELOAD_TABLE = "preload_table";
 
 /* cache config */
@@ -91,7 +89,9 @@ static const char* CONFIG_ENGINE_OMP_THREAD_NUM_DEFAULT = "0";
 static const char* CONFIG_RESOURCE = "resource_config";
 static const char* CONFIG_RESOURCE_MODE = "mode";
 static const char* CONFIG_RESOURCE_MODE_DEFAULT = "simple";
-static const char* CONFIG_RESOURCE_POOL = "resource_pool";
+static const char* CONFIG_RESOURCE_SEARCH_RESOURCES = "search_resources";
+static const char* CONFIG_RESOURCE_INDEX_BUILD_DEVICE = "index_build_device";
+static const char* CONFIG_RESOURCE_INDEX_BUILD_DEVICE_DEFAULT = "gpu0";
 
 class Config {
  public:
@@ -140,8 +140,6 @@ class Config {
     CheckDBConfigArchiveDaysThreshold(const std::string& value);
     Status
     CheckDBConfigInsertBufferSize(const std::string& value);
-    Status
-    CheckDBConfigBuildIndexGPU(const std::string& value);
 
     /* metric config */
     Status
@@ -173,7 +171,9 @@ class Config {
     Status
     CheckResourceConfigMode(const std::string& value);
     Status
-    CheckResourceConfigPool(const std::vector<std::string>& value);
+    CheckResourceConfigSearchResources(const std::vector<std::string>& value);
+    Status
+    CheckResourceConfigIndexBuildDevice(const std::string& value);
 
     std::string
     GetConfigStr(const std::string& parent_key, const std::string& child_key, const std::string& default_value = "");
@@ -203,8 +203,6 @@ class Config {
     Status
     GetDBConfigInsertBufferSize(int32_t& value);
     Status
-    GetDBConfigBuildIndexGPU(int32_t& value);
-    Status
     GetDBConfigPreloadTable(std::string& value);
 
     /* metric config */
@@ -237,7 +235,9 @@ class Config {
     Status
     GetResourceConfigMode(std::string& value);
     Status
-    GetResourceConfigPool(std::vector<std::string>& value);
+    GetResourceConfigSearchResources(std::vector<std::string>& value);
+    Status
+    GetResourceConfigIndexBuildDevice(int32_t& value);
 
  public:
     /* server config */
@@ -263,8 +263,6 @@ class Config {
     SetDBConfigArchiveDaysThreshold(const std::string& value);
     Status
     SetDBConfigInsertBufferSize(const std::string& value);
-    Status
-    SetDBConfigBuildIndexGPU(const std::string& value);
 
     /* metric config */
     Status
@@ -295,6 +293,8 @@ class Config {
     /* resource config */
     Status
     SetResourceConfigMode(const std::string& value);
+    Status
+    SetResourceConfigIndexBuildDevice(const std::string& value);
 
  private:
     std::unordered_map<std::string, std::unordered_map<std::string, std::string>> config_map_;

core/src/server/grpc_impl/GrpcRequestTask.cpp

@@ -113,6 +113,14 @@ ConvertTimeRangeToDBDates(const std::vector<::milvus::grpc::Range>& range_array,
 
     return Status::OK();
 }
+
+std::string
+TableNotExistMsg(const std::string& table_name) {
+    return "Table " + table_name +
+           " not exist. Use milvus.has_table to verify whether the table exists. You also can check if the table name "
+           "exists.";
+}
+
 }  // namespace
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -255,7 +263,7 @@ CreateIndexTask::OnExecute() {
         }
 
         if (!has_table) {
-            return Status(SERVER_TABLE_NOT_EXIST, "Table " + table_name_ + " not exists");
+            return Status(SERVER_TABLE_NOT_EXIST, TableNotExistMsg(table_name_));
         }
 
         auto& grpc_index = index_param_->index();
@@ -348,7 +356,7 @@ DropTableTask::OnExecute() {
         status = DBWrapper::DB()->DescribeTable(table_info);
         if (!status.ok()) {
             if (status.code() == DB_NOT_FOUND) {
-                return Status(SERVER_TABLE_NOT_EXIST, "Table " + table_name_ + " not exists");
+                return Status(SERVER_TABLE_NOT_EXIST, TableNotExistMsg(table_name_));
             } else {
                 return status;
             }
@@ -420,12 +428,14 @@ InsertTask::OnExecute() {
             return status;
         }
         if (insert_param_->row_record_array().empty()) {
-            return Status(SERVER_INVALID_ROWRECORD_ARRAY, "Row record array is empty");
+            return Status(SERVER_INVALID_ROWRECORD_ARRAY,
+                          "The vector array is empty. Make sure you have entered vector records.");
         }
 
         if (!insert_param_->row_id_array().empty()) {
             if (insert_param_->row_id_array().size() != insert_param_->row_record_array_size()) {
-                return Status(SERVER_ILLEGAL_VECTOR_ID, "Size of vector ids is not equal to row record array size");
+                return Status(SERVER_ILLEGAL_VECTOR_ID,
+                              "The size of vector ID array must be equal to the size of the vector.");
             }
         }
 
@@ -435,7 +445,7 @@ InsertTask::OnExecute() {
         status = DBWrapper::DB()->DescribeTable(table_info);
         if (!status.ok()) {
             if (status.code() == DB_NOT_FOUND) {
-                return Status(SERVER_TABLE_NOT_EXIST, "Table " + insert_param_->table_name() + " not exists");
+                return Status(SERVER_TABLE_NOT_EXIST, TableNotExistMsg(insert_param_->table_name()));
             } else {
                 return status;
             }
@@ -447,13 +457,14 @@ InsertTask::OnExecute() {
         // user already provided id before, all insert action require user id
         if ((table_info.flag_ & engine::meta::FLAG_MASK_HAS_USERID) != 0 && !user_provide_ids) {
             return Status(SERVER_ILLEGAL_VECTOR_ID,
-                          "Table vector ids are user defined, please provide id for this batch");
+                          "Table vector IDs are user-defined. Please provide IDs for all vectors of this table.");
         }
 
         // user didn't provided id before, no need to provide user id
         if ((table_info.flag_ & engine::meta::FLAG_MASK_NO_USERID) != 0 && user_provide_ids) {
-            return Status(SERVER_ILLEGAL_VECTOR_ID,
-                          "Table vector ids are auto generated, no need to provide id for this batch");
+            return Status(
+                SERVER_ILLEGAL_VECTOR_ID,
+                "Table vector IDs are auto-generated. All vectors of this table must use auto-generated IDs.");
         }
 
         rc.RecordSection("check validation");
@@ -470,13 +481,13 @@ InsertTask::OnExecute() {
         // TODO(yk): change to one dimension array or use multiple-thread to copy the data
         for (size_t i = 0; i < insert_param_->row_record_array_size(); i++) {
             if (insert_param_->row_record_array(i).vector_data().empty()) {
-                return Status(SERVER_INVALID_ROWRECORD_ARRAY, "Row record array data is empty");
+                return Status(SERVER_INVALID_ROWRECORD_ARRAY,
+                              "The vector dimension must be equal to the table dimension.");
             }
             uint64_t vec_dim = insert_param_->row_record_array(i).vector_data().size();
             if (vec_dim != table_info.dimension_) {
                 ErrorCode error_code = SERVER_INVALID_VECTOR_DIMENSION;
-                std::string error_msg = "Invalid row record dimension: " + std::to_string(vec_dim) +
-                                        " vs. table dimension:" + std::to_string(table_info.dimension_);
+                std::string error_msg = "The vector dimension must be equal to the table dimension.";
                 return Status(error_code, error_msg);
             }
             memcpy(&vec_f[i * table_info.dimension_], insert_param_->row_record_array(i).vector_data().data(),
@@ -569,7 +580,7 @@ SearchTask::OnExecute() {
         status = DBWrapper::DB()->DescribeTable(table_info);
         if (!status.ok()) {
             if (status.code() == DB_NOT_FOUND) {
-                return Status(SERVER_TABLE_NOT_EXIST, "Table " + table_name_ + " not exists");
+                return Status(SERVER_TABLE_NOT_EXIST, TableNotExistMsg(table_name_));
             } else {
                 return status;
             }
@@ -587,7 +598,8 @@ SearchTask::OnExecute() {
         }
 
         if (search_param_->query_record_array().empty()) {
-            return Status(SERVER_INVALID_ROWRECORD_ARRAY, "Row record array is empty");
+            return Status(SERVER_INVALID_ROWRECORD_ARRAY,
+                          "The vector array is empty. Make sure you have entered vector records.");
         }
 
         // step 4: check date range, and convert to db dates
@@ -609,13 +621,13 @@ SearchTask::OnExecute() {
         std::vector<float> vec_f(record_array_size * table_info.dimension_, 0);
         for (size_t i = 0; i < record_array_size; i++) {
             if (search_param_->query_record_array(i).vector_data().empty()) {
-                return Status(SERVER_INVALID_ROWRECORD_ARRAY, "Row record array data is empty");
+                return Status(SERVER_INVALID_ROWRECORD_ARRAY,
+                              "The vector dimension must be equal to the table dimension.");
             }
             uint64_t query_vec_dim = search_param_->query_record_array(i).vector_data().size();
             if (query_vec_dim != table_info.dimension_) {
                 ErrorCode error_code = SERVER_INVALID_VECTOR_DIMENSION;
-                std::string error_msg = "Invalid row record dimension: " + std::to_string(query_vec_dim) +
-                                        " vs. table dimension:" + std::to_string(table_info.dimension_);
+                std::string error_msg = "The vector dimension must be equal to the table dimension.";
                 return Status(error_code, error_msg);
             }
 
@@ -707,7 +719,7 @@ CountTableTask::OnExecute() {
         status = DBWrapper::DB()->GetTableRowCount(table_name_, row_count);
         if (!status.ok()) {
             if (status.code(), DB_NOT_FOUND) {
-                return Status(SERVER_TABLE_NOT_EXIST, "Table " + table_name_ + " not exists");
+                return Status(SERVER_TABLE_NOT_EXIST, TableNotExistMsg(table_name_));
             } else {
                 return status;
             }
@@ -779,7 +791,7 @@ DeleteByRangeTask::OnExecute() {
         status = DBWrapper::DB()->DescribeTable(table_info);
         if (!status.ok()) {
             if (status.code(), DB_NOT_FOUND) {
-                return Status(SERVER_TABLE_NOT_EXIST, "Table " + table_name + " not exists");
+                return Status(SERVER_TABLE_NOT_EXIST, TableNotExistMsg(table_name));
             } else {
                 return status;
             }
@@ -917,7 +929,7 @@ DropIndexTask::OnExecute() {
         }
 
         if (!has_table) {
-            return Status(SERVER_TABLE_NOT_EXIST, "Table " + table_name_ + " not exists");
+            return Status(SERVER_TABLE_NOT_EXIST, TableNotExistMsg(table_name_));
         }
 
         // step 2: check table existence

core/src/utils/ValidationUtil.cpp

@@ -37,14 +37,15 @@ Status
 ValidationUtil::ValidateTableName(const std::string& table_name) {
     // Table name shouldn't be empty.
     if (table_name.empty()) {
-        std::string msg = "Empty table name";
+        std::string msg = "Table name should not be empty.";
         SERVER_LOG_ERROR << msg;
         return Status(SERVER_INVALID_TABLE_NAME, msg);
     }
 
+    std::string invalid_msg = "Invalid table name: " + table_name + ". ";
     // Table name size shouldn't exceed 16384.
     if (table_name.size() > TABLE_NAME_SIZE_LIMIT) {
-        std::string msg = "Table name size exceed the limitation";
+        std::string msg = invalid_msg + "The length of a table name must be less than 255 characters.";
         SERVER_LOG_ERROR << msg;
         return Status(SERVER_INVALID_TABLE_NAME, msg);
     }
@@ -52,7 +53,7 @@ ValidationUtil::ValidateTableName(const std::string& table_name) {
     // Table name first character should be underscore or character.
     char first_char = table_name[0];
     if (first_char != '_' && std::isalpha(first_char) == 0) {
-        std::string msg = "Table name first character isn't underscore or character";
+        std::string msg = invalid_msg + "The first character of a table name must be an underscore or letter.";
         SERVER_LOG_ERROR << msg;
         return Status(SERVER_INVALID_TABLE_NAME, msg);
     }
@@ -61,7 +62,7 @@ ValidationUtil::ValidateTableName(const std::string& table_name) {
     for (int64_t i = 1; i < table_name_size; ++i) {
         char name_char = table_name[i];
         if (name_char != '_' && std::isalnum(name_char) == 0) {
-            std::string msg = "Table name character isn't underscore or alphanumber";
+            std::string msg = invalid_msg + "Table name can only contain numbers, letters, and underscores.";
             SERVER_LOG_ERROR << msg;
             return Status(SERVER_INVALID_TABLE_NAME, msg);
         }
@@ -72,12 +73,9 @@ ValidationUtil::ValidateTableName(const std::string& table_name) {
 
 Status
 ValidationUtil::ValidateTableDimension(int64_t dimension) {
-    if (dimension <= 0) {
-        std::string msg = "Dimension value should be greater than 0";
-        SERVER_LOG_ERROR << msg;
-        return Status(SERVER_INVALID_VECTOR_DIMENSION, msg);
-    } else if (dimension > TABLE_DIMENSION_LIMIT) {
-        std::string msg = "Table dimension excceed the limitation: " + std::to_string(TABLE_DIMENSION_LIMIT);
+    if (dimension <= 0 || dimension > TABLE_DIMENSION_LIMIT) {
+        std::string msg = "Invalid table dimension: " + std::to_string(dimension) + ". " +
+                          "The table dimension must be within the range of 1 ~ 16384.";
         SERVER_LOG_ERROR << msg;
         return Status(SERVER_INVALID_VECTOR_DIMENSION, msg);
     } else {
@@ -89,10 +87,20 @@ Status
 ValidationUtil::ValidateTableIndexType(int32_t index_type) {
     int engine_type = static_cast<int>(engine::EngineType(index_type));
     if (engine_type <= 0 || engine_type > static_cast<int>(engine::EngineType::MAX_VALUE)) {
-        std::string msg = "Invalid index type: " + std::to_string(index_type);
+        std::string msg = "Invalid index type: " + std::to_string(index_type) + ". " +
+                          "Make sure the index type is in IndexType list.";
+        SERVER_LOG_ERROR << msg;
+        return Status(SERVER_INVALID_INDEX_TYPE, msg);
+    }
+
+#ifndef CUSTOMIZATION
+    // special case, hybird index only available in customize faiss library
+    if (engine_type == static_cast<int>(engine::EngineType::FAISS_IVFSQ8H)) {
+        std::string msg = "Unsupported index type: " + std::to_string(index_type);
         SERVER_LOG_ERROR << msg;
         return Status(SERVER_INVALID_INDEX_TYPE, msg);
     }
+#endif
 
     return Status::OK();
 }
@@ -100,7 +108,8 @@ ValidationUtil::ValidateTableIndexType(int32_t index_type) {
 Status
 ValidationUtil::ValidateTableIndexNlist(int32_t nlist) {
     if (nlist <= 0) {
-        std::string msg = "nlist value should be greater than 0";
+        std::string msg =
+            "Invalid index nlist: " + std::to_string(nlist) + ". " + "The index nlist must be greater than 0.";
         SERVER_LOG_ERROR << msg;
         return Status(SERVER_INVALID_INDEX_NLIST, msg);
     }
@@ -111,7 +120,9 @@ ValidationUtil::ValidateTableIndexNlist(int32_t nlist) {
 Status
 ValidationUtil::ValidateTableIndexFileSize(int64_t index_file_size) {
     if (index_file_size <= 0 || index_file_size > INDEX_FILE_SIZE_LIMIT) {
-        std::string msg = "Invalid index file size: " + std::to_string(index_file_size);
+        std::string msg = "Invalid index file size: " + std::to_string(index_file_size) + ". " +
+                          "The index file size must be within the range of 1 ~ " +
+                          std::to_string(INDEX_FILE_SIZE_LIMIT) + ".";
         SERVER_LOG_ERROR << msg;
         return Status(SERVER_INVALID_INDEX_FILE_SIZE, msg);
     }
@@ -123,7 +134,8 @@ Status
 ValidationUtil::ValidateTableIndexMetricType(int32_t metric_type) {
     if (metric_type != static_cast<int32_t>(engine::MetricType::L2) &&
         metric_type != static_cast<int32_t>(engine::MetricType::IP)) {
-        std::string msg = "Invalid metric type: " + std::to_string(metric_type);
+        std::string msg = "Invalid index metric type: " + std::to_string(metric_type) + ". " +
+                          "Make sure the metric type is either MetricType.L2 or MetricType.IP.";
         SERVER_LOG_ERROR << msg;
         return Status(SERVER_INVALID_INDEX_METRIC_TYPE, msg);
     }
@@ -133,7 +145,8 @@ ValidationUtil::ValidateTableIndexMetricType(int32_t metric_type) {
 Status
 ValidationUtil::ValidateSearchTopk(int64_t top_k, const engine::meta::TableSchema& table_schema) {
     if (top_k <= 0 || top_k > 2048) {
-        std::string msg = "Invalid top k value: " + std::to_string(top_k) + ", rational range [1, 2048]";
+        std::string msg =
+            "Invalid topk: " + std::to_string(top_k) + ". " + "The topk must be within the range of 1 ~ 2048.";
         SERVER_LOG_ERROR << msg;
         return Status(SERVER_INVALID_TOPK, msg);
     }
@@ -144,8 +157,8 @@ ValidationUtil::ValidateSearchTopk(int64_t top_k, const engine::meta::TableSchem
 Status
 ValidationUtil::ValidateSearchNprobe(int64_t nprobe, const engine::meta::TableSchema& table_schema) {
     if (nprobe <= 0 || nprobe > table_schema.nlist_) {
-        std::string msg = "Invalid nprobe value: " + std::to_string(nprobe) + ", rational range [1, " +
-                          std::to_string(table_schema.nlist_) + "]";
+        std::string msg = "Invalid nprobe: " + std::to_string(nprobe) + ". " +
+                          "The nprobe must be within the range of 1 ~ index nlist.";
         SERVER_LOG_ERROR << msg;
         return Status(SERVER_INVALID_NPROBE, msg);
     }

core/src/wrapper/KnowhereResource.cpp

@@ -45,7 +45,7 @@ KnowhereResource::Initialize() {
     server::Config& config = server::Config::GetInstance();
 
     int32_t build_index_gpu;
-    s = config.GetDBConfigBuildIndexGPU(build_index_gpu);
+    s = config.GetResourceConfigIndexBuildDevice(build_index_gpu);
     if (!s.ok())
         return s;
 
@@ -53,7 +53,7 @@ KnowhereResource::Initialize() {
 
     // get search gpu resource
     std::vector<std::string> pool;
-    s = config.GetResourceConfigPool(pool);
+    s = config.GetResourceConfigSearchResources(pool);
     if (!s.ok())
         return s;
 

core/unittest/server/appendix/server_config.yaml

-# All the following configurations are default values.
+# Default values are used when you make no changes to the following parameters.
 
 server_config:
   address: 0.0.0.0                  # milvus server ip address (IPv4)
@@ -11,27 +11,33 @@ db_config:
   secondary_path:                   # path used to store data only, split by semicolon
 
   backend_url: sqlite://:@:/        # URI format: dialect://username:password@host:port/database
-                                    # Keep 'dialect://:@:/', and replace other texts with real values.
+                                    # Keep 'dialect://:@:/', and replace other texts with real values
                                     # Replace 'dialect' with 'mysql' or 'sqlite'
 
   insert_buffer_size: 4             # GB, maximum insert buffer size allowed
-  build_index_gpu: 0                # gpu id used for building index
+                                    # sum of insert_buffer_size and cpu_cache_capacity cannot exceed total memory
+
+  preload_table:                    # preload data at startup, '*' means load all tables, empty value means no preload
+                                    # you can specify preload tables like this: table1,table2,table3
 
 metric_config:
   enable_monitor: false             # enable monitoring or not
   collector: prometheus             # prometheus
   prometheus_config:
-    port: 8080                      # port prometheus used to fetch metrics
+    port: 8080                      # port prometheus uses to fetch metrics
 
 cache_config:
-  cpu_mem_capacity: 16              # GB, CPU memory used for cache
-  cpu_mem_threshold: 0.85           # percentage of data kept when cache cleanup triggered
-  cache_insert_data: false          # whether load inserted data into cache
+  cpu_cache_capacity: 16            # GB, CPU memory used for cache
+  cpu_cache_threshold: 0.85         # percentage of data that will be kept when cache cleanup is triggered
+  gpu_cache_capacity: 4             # GB, GPU memory used for cache
+  gpu_cache_threshold: 0.85         # percentage of data that will be kept when cache cleanup is triggered
+  cache_insert_data: false          # whether to load inserted data into cache
 
 engine_config:
-  blas_threshold: 20
+  use_blas_threshold: 20            # if nq <  use_blas_threshold, use SSE, faster with fluctuated response times
+                                    # if nq >= use_blas_threshold, use OpenBlas, slower with stable response times
 
 resource_config:
-  resource_pool:
-    - cpu
+  search_resources:                 # define the GPUs used for search computation, valid value: gpux
     - gpu0
+  index_build_device: gpu0          # GPU used for building index

core/unittest/wrapper/test_wrapper.cpp

@@ -164,3 +164,14 @@ TEST_P(KnowhereWrapperTest, SERIALIZE_TEST) {
         AssertResult(res_ids, res_dis);
     }
 }
+
+#include "wrapper/ConfAdapter.h"
+TEST(whatever, test_config) {
+    milvus::engine::TempMetaConf conf;
+    auto nsg_conf = std::make_shared<milvus::engine::NSGConfAdapter>();
+    nsg_conf->Match(conf);
+    nsg_conf->MatchSearch(conf, milvus::engine::IndexType::FAISS_IVFPQ_GPU);
+
+    auto pq_conf = std::make_shared<milvus::engine::IVFPQConfAdapter>();
+    pq_conf->Match(conf);
+}

core/unittest/wrapper/utils.h

@@ -104,28 +104,28 @@ class ParamGenerator {
                 tempconf->metric_type = knowhere::METRICTYPE::L2;
                 return tempconf;
             }
-            case milvus::engine::IndexType::FAISS_IVFPQ_CPU:
-            case milvus::engine::IndexType::FAISS_IVFPQ_GPU: {
-                auto tempconf = std::make_shared<knowhere::IVFPQCfg>();
-                tempconf->nlist = 100;
-                tempconf->nprobe = 16;
-                tempconf->nbits = 8;
-                tempconf->m = 8;
-                tempconf->metric_type = knowhere::METRICTYPE::L2;
-                return tempconf;
-            }
-            case milvus::engine::IndexType::NSG_MIX: {
-                auto tempconf = std::make_shared<knowhere::NSGCfg>();
-                tempconf->nlist = 100;
-                tempconf->nprobe = 16;
-                tempconf->search_length = 8;
-                tempconf->knng = 200;
-                tempconf->search_length = 40; // TODO(linxj): be 20 when search
-                tempconf->out_degree = 60;
-                tempconf->candidate_pool_size = 200;
-                tempconf->metric_type = knowhere::METRICTYPE::L2;
-                return tempconf;
-            }
+//            case milvus::engine::IndexType::FAISS_IVFPQ_CPU:
+//            case milvus::engine::IndexType::FAISS_IVFPQ_GPU: {
+//                auto tempconf = std::make_shared<knowhere::IVFPQCfg>();
+//                tempconf->nlist = 100;
+//                tempconf->nprobe = 16;
+//                tempconf->nbits = 8;
+//                tempconf->m = 8;
+//                tempconf->metric_type = knowhere::METRICTYPE::L2;
+//                return tempconf;
+//            }
+//            case milvus::engine::IndexType::NSG_MIX: {
+//                auto tempconf = std::make_shared<knowhere::NSGCfg>();
+//                tempconf->nlist = 100;
+//                tempconf->nprobe = 16;
+//                tempconf->search_length = 8;
+//                tempconf->knng = 200;
+//                tempconf->search_length = 40; // TODO(linxj): be 20 when search
+//                tempconf->out_degree = 60;
+//                tempconf->candidate_pool_size = 200;
+//                tempconf->metric_type = knowhere::METRICTYPE::L2;
+//                return tempconf;
+//            }
         }
     }
 };