diff --git a/cpp/CHANGELOG.md b/cpp/CHANGELOG.md index 99d55ea524f4a29e1df9702a22c805c254cd579f..879094b8d1bdffef30b8b9a1f7fb33096e9e5269 100644 --- a/cpp/CHANGELOG.md +++ b/cpp/CHANGELOG.md @@ -10,6 +10,10 @@ Please mark all change in change log and use the ticket from JIRA. - MS-416 - ExecutionEngineImpl::GpuCache has not return value cause crash - MS-417 - YAML sequence load disable cause scheduler startup failed - MS-413 - Create index failed and server exited +- MS-427 - Describe index error after drop index +- MS-432 - Search vectors params nprobe need to check max number +- MS-431 - Search vectors params nprobe: 0/-1, expected result: raise exception +- MS-331 - Crate Table : when table exists, error code is META_FAILED(code=15) rather than ILLEGAL TABLE NAME(code=9)) ## Improvement - MS-327 - Clean code for milvus diff --git a/cpp/src/db/DB.h b/cpp/src/db/DB.h index f42154174166e4741c32bc2c0e57bf20e23a3f5c..908714a08a61a8a7e50a379b49bbf8f413f09b12 100644 --- a/cpp/src/db/DB.h +++ b/cpp/src/db/DB.h @@ -46,7 +46,6 @@ public: virtual Status Size(uint64_t& result) = 0; - virtual Status BuildIndex(const std::string& table_id) = 0; virtual Status CreateIndex(const std::string& table_id, const TableIndex& index) = 0; virtual Status DescribeIndex(const std::string& table_id, TableIndex& index) = 0; virtual Status DropIndex(const std::string& table_id) = 0; diff --git a/cpp/src/db/DBImpl.cpp b/cpp/src/db/DBImpl.cpp index a9386ff1eb8e865ccfd3579be2c6e930621c3fb7..168522e21d0287a35d636af06f197881d2c5fd31 100644 --- a/cpp/src/db/DBImpl.cpp +++ b/cpp/src/db/DBImpl.cpp @@ -424,7 +424,14 @@ Status DBImpl::MergeFiles(const std::string& table_id, const meta::DateT& date, } //step 4: update table files state - table_file.file_type_ = meta::TableFileSchema::RAW; + //if index type isn't IDMAP, set file type to TO_INDEX if file size execeed index_file_size + //else set file type to RAW, no need to build index + if (table_file.engine_type_ != (int)EngineType::FAISS_IDMAP) { + table_file.file_type_ = (index->PhysicalSize() >= table_file.index_file_size_) ? + meta::TableFileSchema::TO_INDEX : meta::TableFileSchema::RAW; + } else { + table_file.file_type_ = meta::TableFileSchema::RAW; + } table_file.file_size_ = index->PhysicalSize(); table_file.row_count_ = index->Count(); updated.push_back(table_file); @@ -516,22 +523,6 @@ void DBImpl::StartBuildIndexTask(bool force) { } } -Status DBImpl::BuildIndex(const std::string& table_id) { - bool has = false; - meta_ptr_->HasNonIndexFiles(table_id, has); - int times = 1; - - while (has) { - ENGINE_LOG_DEBUG << "Non index files detected in " << table_id << "! Will build index " << times; - meta_ptr_->UpdateTableFilesToIndex(table_id); - /* StartBuildIndexTask(true); */ - std::this_thread::sleep_for(std::chrono::milliseconds(std::min(10*1000, times*100))); - meta_ptr_->HasNonIndexFiles(table_id, has); - times++; - } - return Status::OK(); -} - Status DBImpl::CreateIndex(const std::string& table_id, const TableIndex& index) { { std::unique_lock lock(build_index_mutex_); @@ -552,18 +543,17 @@ Status DBImpl::CreateIndex(const std::string& table_id, const TableIndex& index) //step 2: drop old index files DropIndex(table_id); - if(index.engine_type_ == (int)EngineType::FAISS_IDMAP) { - ENGINE_LOG_DEBUG << "index type = IDMAP, no need to build index"; - return Status::OK(); - } - //step 3: update index info - status = meta_ptr_->UpdateTableIndexParam(table_id, index); if (!status.ok()) { ENGINE_LOG_ERROR << "Failed to update table index info"; return status; } + + if(index.engine_type_ == (int)EngineType::FAISS_IDMAP) { + ENGINE_LOG_DEBUG << "index type = IDMAP, no need to build index"; + return Status::OK(); + } } bool has = false; diff --git a/cpp/src/db/DBImpl.h b/cpp/src/db/DBImpl.h index 65ac489c49ac3e6f9e9e658575b2e8692f04e16d..a6e52bd3c3bce3aa0cb8398662f0ac65e78e2830 100644 --- a/cpp/src/db/DBImpl.h +++ b/cpp/src/db/DBImpl.h @@ -82,8 +82,6 @@ class DBImpl : public DB { Status Size(uint64_t &result) override; - Status BuildIndex(const std::string& table_id) override; - Status CreateIndex(const std::string& table_id, const TableIndex& index) override; Status DescribeIndex(const std::string& table_id, TableIndex& index) override; diff --git a/cpp/src/db/Status.cpp b/cpp/src/db/Status.cpp index 76c72fac62e9326fe05f7e68e9b456ee5a01ab76..65524c62a96878fd6837a2473768274acf42795c 100644 --- a/cpp/src/db/Status.cpp +++ b/cpp/src/db/Status.cpp @@ -48,6 +48,21 @@ std::string Status::ToString() const { case kNotFound: type = "NotFound: "; break; + case kError: + type = "Error: "; + break; + case kInvalidDBPath: + type = "InvalidDBPath: "; + break; + case kGroupError: + type = "GroupError: "; + break; + case kDBTransactionError: + type = "DBTransactionError: "; + break; + case kAlreadyExist: + type = "AlreadyExist: "; + break; default: snprintf(tmp, sizeof(tmp), "Unkown code(%d): ", static_cast(code())); diff --git a/cpp/src/db/insert/MemTableFile.cpp b/cpp/src/db/insert/MemTableFile.cpp index 48e52d0d9b333639bff0e61d2109849cf174e77d..2054a5047e30cba72166583f860b0d1430736d5e 100644 --- a/cpp/src/db/insert/MemTableFile.cpp +++ b/cpp/src/db/insert/MemTableFile.cpp @@ -87,8 +87,14 @@ Status MemTableFile::Serialize() { table_file_schema_.file_size_ = execution_engine_->PhysicalSize(); table_file_schema_.row_count_ = execution_engine_->Count(); - table_file_schema_.file_type_ = (size >= table_file_schema_.index_file_size_) ? - meta::TableFileSchema::TO_INDEX : meta::TableFileSchema::RAW; + //if index type isn't IDMAP, set file type to TO_INDEX if file size execeed index_file_size + //else set file type to RAW, no need to build index + if (table_file_schema_.engine_type_ != (int)EngineType::FAISS_IDMAP) { + table_file_schema_.file_type_ = (size >= table_file_schema_.index_file_size_) ? + meta::TableFileSchema::TO_INDEX : meta::TableFileSchema::RAW; + } else { + table_file_schema_.file_type_ = meta::TableFileSchema::RAW; + } auto status = meta_->UpdateTableFile(table_file_schema_); diff --git a/cpp/src/db/meta/MetaTypes.h b/cpp/src/db/meta/MetaTypes.h index 0727528fe935f5401c2acce79a71ad9736869e8a..a75bcc07dc7652ce244e55e0e4c99dbebb9e0981 100644 --- a/cpp/src/db/meta/MetaTypes.h +++ b/cpp/src/db/meta/MetaTypes.h @@ -19,8 +19,8 @@ namespace meta { constexpr int32_t DEFAULT_ENGINE_TYPE = (int)EngineType::FAISS_IDMAP; constexpr int32_t DEFAULT_NLIST = 16384; -constexpr int32_t DEFAULT_INDEX_FILE_SIZE = ONE_GB; constexpr int32_t DEFAULT_METRIC_TYPE = (int)MetricType::L2; +constexpr int32_t DEFAULT_INDEX_FILE_SIZE = ONE_GB; constexpr int64_t FLAG_MASK_USERID = 1; diff --git a/cpp/src/db/meta/MySQLMetaImpl.cpp b/cpp/src/db/meta/MySQLMetaImpl.cpp index 92af8e2a677d9697c7dd9a03302e0908c25d17e4..cb1ec199cf08a8914bce88f46279179d5f2c75f7 100644 --- a/cpp/src/db/meta/MySQLMetaImpl.cpp +++ b/cpp/src/db/meta/MySQLMetaImpl.cpp @@ -536,6 +536,7 @@ Status MySQLMetaImpl::DropTableIndex(const std::string &table_id) { Query dropTableIndexQuery = connectionPtr->query(); + //soft delete index files dropTableIndexQuery << "UPDATE TableFiles " << "SET file_type = " << std::to_string(TableFileSchema::TO_DELETE) << "," << "updated_time = " << utils::GetMicroSecTimeStamp() << " " << @@ -550,6 +551,7 @@ Status MySQLMetaImpl::DropTableIndex(const std::string &table_id) { dropTableIndexQuery.error()); } + //set all backup file to raw dropTableIndexQuery << "UPDATE TableFiles " << "SET file_type = " << std::to_string(TableFileSchema::RAW) << "," << "updated_time = " << utils::GetMicroSecTimeStamp() << " " << @@ -564,6 +566,21 @@ Status MySQLMetaImpl::DropTableIndex(const std::string &table_id) { dropTableIndexQuery.error()); } + //set table index type to raw + dropTableIndexQuery << "UPDATE Tables " << + "SET engine_type = " << std::to_string(DEFAULT_ENGINE_TYPE) << "," << + "nlist = " << std::to_string(DEFAULT_NLIST) << " " << + "metric_type = " << std::to_string(DEFAULT_METRIC_TYPE) << " " << + "WHERE table_id = " << quote << table_id << ";"; + + ENGINE_LOG_DEBUG << "MySQLMetaImpl::DropTableIndex: " << dropTableIndexQuery.str(); + + if (!dropTableIndexQuery.exec()) { + ENGINE_LOG_ERROR << "QUERY ERROR WHEN DROP TABLE INDEX"; + return Status::DBTransactionError("QUERY ERROR WHEN DROP TABLE INDEX", + dropTableIndexQuery.error()); + } + } //Scoped Connection } catch (const BadQuery &er) { diff --git a/cpp/src/db/meta/SqliteMetaImpl.cpp b/cpp/src/db/meta/SqliteMetaImpl.cpp index a06c391ca5f2c68003313d7fb63413a540e8553c..3f427130ae80e4c0c0140c4833597581d7668a5a 100644 --- a/cpp/src/db/meta/SqliteMetaImpl.cpp +++ b/cpp/src/db/meta/SqliteMetaImpl.cpp @@ -461,6 +461,17 @@ Status SqliteMetaImpl::DropTableIndex(const std::string &table_id) { c(&TableFileSchema::file_type_) == (int) TableFileSchema::BACKUP )); + //set table index type to raw + ConnectorPtr->update_all( + set( + c(&TableSchema::engine_type_) = DEFAULT_ENGINE_TYPE, + c(&TableSchema::nlist_) = DEFAULT_NLIST, + c(&TableSchema::metric_type_) = DEFAULT_METRIC_TYPE + ), + where( + c(&TableSchema::table_id_) == table_id + )); + } catch (std::exception &e) { return HandleException("Encounter exception when delete table index files", e); } @@ -798,11 +809,15 @@ Status SqliteMetaImpl::FilesToSearch(const std::string &table_id, } files[table_file.date_].push_back(table_file); } - + if(files.empty()) { + std::cout << "ERROR" << std::endl; + } } catch (std::exception &e) { return HandleException("Encounter exception when iterate index files", e); } + + return Status::OK(); } diff --git a/cpp/src/server/grpc_impl/GrpcRequestScheduler.cpp b/cpp/src/server/grpc_impl/GrpcRequestScheduler.cpp index ef60aba8a3b16e8c0672a56a689aa5ce228e3bbd..637b4c3b34154fd9c7307c2cf5653ad188f97be2 100644 --- a/cpp/src/server/grpc_impl/GrpcRequestScheduler.cpp +++ b/cpp/src/server/grpc_impl/GrpcRequestScheduler.cpp @@ -39,6 +39,9 @@ namespace { {SERVER_INVALID_ROWRECORD, ::milvus::grpc::ErrorCode::ILLEGAL_ROWRECORD}, {SERVER_INVALID_ROWRECORD_ARRAY, ::milvus::grpc::ErrorCode::ILLEGAL_ROWRECORD}, {SERVER_INVALID_TOPK, ::milvus::grpc::ErrorCode::ILLEGAL_TOPK}, + {SERVER_INVALID_NPROBE, ::milvus::grpc::ErrorCode::ILLEGAL_ARGUMENT}, + {SERVER_INVALID_INDEX_NLIST, ::milvus::grpc::ErrorCode::ILLEGAL_ARGUMENT}, + {SERVER_INVALID_INDEX_METRIC_TYPE,::milvus::grpc::ErrorCode::ILLEGAL_ARGUMENT}, {SERVER_ILLEGAL_VECTOR_ID, ::milvus::grpc::ErrorCode::ILLEGAL_VECTOR_ID}, {SERVER_ILLEGAL_SEARCH_RESULT, ::milvus::grpc::ErrorCode::ILLEGAL_SEARCH_RESULT}, {SERVER_CACHE_ERROR, ::milvus::grpc::ErrorCode::CACHE_FAILED}, diff --git a/cpp/src/server/grpc_impl/GrpcRequestTask.cpp b/cpp/src/server/grpc_impl/GrpcRequestTask.cpp index a4b8d68c296e4066f263ecf41f148e18845e67bc..5489089033e789f05282ebf01e82f7f7d9221a80 100644 --- a/cpp/src/server/grpc_impl/GrpcRequestTask.cpp +++ b/cpp/src/server/grpc_impl/GrpcRequestTask.cpp @@ -153,7 +153,10 @@ CreateTableTask::OnExecute() { engine::Status stat = DBWrapper::DB()->CreateTable(table_info); if (!stat.ok()) { //table could exist - return SetError(DB_META_TRANSACTION_FAILED, "Engine failed: " + stat.ToString()); + if(stat.IsAlreadyExist()) { + return SetError(SERVER_INVALID_TABLE_NAME, stat.ToString()); + } + return SetError(DB_META_TRANSACTION_FAILED, stat.ToString()); } } catch (std::exception &ex) { @@ -193,7 +196,7 @@ DescribeTableTask::OnExecute() { table_info.table_id_ = table_name_; engine::Status stat = DBWrapper::DB()->DescribeTable(table_info); if (!stat.ok()) { - return SetError(DB_META_TRANSACTION_FAILED, "Engine failed: " + stat.ToString()); + return SetError(DB_META_TRANSACTION_FAILED, stat.ToString()); } schema_->mutable_table_name()->set_table_name(table_info.table_id_); @@ -238,7 +241,7 @@ CreateIndexTask::OnExecute() { bool has_table = false; engine::Status stat = DBWrapper::DB()->HasTable(table_name_, has_table); if (!stat.ok()) { - return SetError(DB_META_TRANSACTION_FAILED, "Engine failed: " + stat.ToString()); + return SetError(DB_META_TRANSACTION_FAILED, stat.ToString()); } if (!has_table) { @@ -268,7 +271,7 @@ CreateIndexTask::OnExecute() { index.metric_type_ = grpc_index.metric_type(); stat = DBWrapper::DB()->CreateIndex(table_name_, index); if (!stat.ok()) { - return SetError(SERVER_BUILD_INDEX_ERROR, "Engine failed: " + stat.ToString()); + return SetError(SERVER_BUILD_INDEX_ERROR, stat.ToString()); } rc.ElapseFromBegin("totally cost"); @@ -306,7 +309,7 @@ HasTableTask::OnExecute() { //step 2: check table existence engine::Status stat = DBWrapper::DB()->HasTable(table_name_, has_table_); if (!stat.ok()) { - return SetError(DB_META_TRANSACTION_FAILED, "Engine failed: " + stat.ToString()); + return SetError(DB_META_TRANSACTION_FAILED, stat.ToString()); } rc.ElapseFromBegin("totally cost"); @@ -348,7 +351,7 @@ DropTableTask::OnExecute() { if (stat.IsNotFound()) { return SetError(SERVER_TABLE_NOT_EXIST, "Table " + table_name_ + " not exists"); } else { - return SetError(DB_META_TRANSACTION_FAILED, "Engine failed: " + stat.ToString()); + return SetError(DB_META_TRANSACTION_FAILED, stat.ToString()); } } @@ -358,7 +361,7 @@ DropTableTask::OnExecute() { std::vector dates; stat = DBWrapper::DB()->DeleteTable(table_name_, dates); if (!stat.ok()) { - return SetError(DB_META_TRANSACTION_FAILED, "Engine failed: " + stat.ToString()); + return SetError(DB_META_TRANSACTION_FAILED, stat.ToString()); } rc.ElapseFromBegin("total cost"); @@ -386,7 +389,7 @@ ShowTablesTask::OnExecute() { std::vector schema_array; engine::Status stat = DBWrapper::DB()->AllTables(schema_array); if (!stat.ok()) { - return SetError(DB_META_TRANSACTION_FAILED, "Engine failed: " + stat.ToString()); + return SetError(DB_META_TRANSACTION_FAILED, stat.ToString()); } for (auto &schema : schema_array) { @@ -448,7 +451,7 @@ InsertTask::OnExecute() { return SetError(SERVER_TABLE_NOT_EXIST, "Table " + insert_param_->table_name() + " not exists"); } else { - return SetError(DB_META_TRANSACTION_FAILED, "Engine failed: " + stat.ToString()); + return SetError(DB_META_TRANSACTION_FAILED, stat.ToString()); } } @@ -573,28 +576,13 @@ SearchTask::OnExecute() { try { TimeRecorder rc("SearchTask"); - //step 1: check arguments + //step 1: check table name std::string table_name_ = search_param_->table_name(); ServerError res = ValidationUtil::ValidateTableName(table_name_); if (res != SERVER_SUCCESS) { return SetError(res, "Invalid table name: " + table_name_); } - int64_t top_k_ = search_param_->topk(); - - if (top_k_ <= 0 || top_k_ > 1024) { - return SetError(SERVER_INVALID_TOPK, "Invalid topk: " + std::to_string(top_k_)); - } - - int64_t nprobe = search_param_->nprobe(); - if (nprobe <= 0) { - return SetError(SERVER_INVALID_NPROBE, "Invalid nprobe: " + std::to_string(nprobe)); - } - - if (search_param_->query_record_array().empty()) { - return SetError(SERVER_INVALID_ROWRECORD_ARRAY, "Row record array is empty"); - } - //step 2: check table existence engine::meta::TableSchema table_info; table_info.table_id_ = table_name_; @@ -603,11 +591,28 @@ SearchTask::OnExecute() { if (stat.IsNotFound()) { return SetError(SERVER_TABLE_NOT_EXIST, "Table " + table_name_ + " not exists"); } else { - return SetError(DB_META_TRANSACTION_FAILED, "Engine failed: " + stat.ToString()); + return SetError(DB_META_TRANSACTION_FAILED, stat.ToString()); } } - //step 3: check date range, and convert to db dates + //step 3: check search parameter + int64_t top_k = search_param_->topk(); + res = ValidationUtil::ValidateSearchTopk(top_k, table_info); + if (res != SERVER_SUCCESS) { + return SetError(res, "Invalid topk: " + std::to_string(top_k)); + } + + int64_t nprobe = search_param_->nprobe(); + res = ValidationUtil::ValidateSearchNprobe(nprobe, table_info); + if (res != SERVER_SUCCESS) { + return SetError(res, "Invalid nprobe: " + std::to_string(nprobe)); + } + + if (search_param_->query_record_array().empty()) { + return SetError(SERVER_INVALID_ROWRECORD_ARRAY, "Row record array is empty"); + } + + //step 4: check date range, and convert to db dates std::vector dates; ServerError error_code = SERVER_SUCCESS; std::string error_msg; @@ -630,7 +635,7 @@ SearchTask::OnExecute() { ProfilerStart(fname.c_str()); #endif - //step 3: prepare float data + //step 5: prepare float data auto record_array_size = search_param_->query_record_array_size(); std::vector vec_f(record_array_size * table_info.dimension_, 0); for (size_t i = 0; i < record_array_size; i++) { @@ -651,21 +656,21 @@ SearchTask::OnExecute() { } rc.ElapseFromBegin("prepare vector data"); - //step 4: search vectors + //step 6: search vectors engine::QueryResults results; auto record_count = (uint64_t) search_param_->query_record_array().size(); if (file_id_array_.empty()) { - stat = DBWrapper::DB()->Query(table_name_, (size_t) top_k_, record_count, nprobe, vec_f.data(), + stat = DBWrapper::DB()->Query(table_name_, (size_t) top_k, record_count, nprobe, vec_f.data(), dates, results); } else { - stat = DBWrapper::DB()->Query(table_name_, file_id_array_, (size_t) top_k_, + stat = DBWrapper::DB()->Query(table_name_, file_id_array_, (size_t) top_k, record_count, nprobe, vec_f.data(), dates, results); } rc.ElapseFromBegin("search vectors from engine"); if (!stat.ok()) { - return SetError(DB_META_TRANSACTION_FAILED, "Engine failed: " + stat.ToString()); + return SetError(DB_META_TRANSACTION_FAILED, stat.ToString()); } if (results.empty()) { @@ -680,7 +685,7 @@ SearchTask::OnExecute() { rc.ElapseFromBegin("do search"); - //step 5: construct result array + //step 7: construct result array for (uint64_t i = 0; i < record_count; i++) { auto &result = results[i]; const auto &record = search_param_->query_record_array(i); @@ -699,10 +704,10 @@ SearchTask::OnExecute() { ProfilerStop(); #endif + //step 8: print time cost percent double span_result = rc.RecordSection("construct result"); rc.ElapseFromBegin("totally cost"); - //step 6: print time cost percent } catch (std::exception &ex) { return SetError(SERVER_UNEXPECTED_ERROR, ex.what()); @@ -740,7 +745,7 @@ CountTableTask::OnExecute() { uint64_t row_count = 0; engine::Status stat = DBWrapper::DB()->GetTableRowCount(table_name_, row_count); if (!stat.ok()) { - return SetError(DB_META_TRANSACTION_FAILED, "Engine failed: " + stat.ToString()); + return SetError(DB_META_TRANSACTION_FAILED, stat.ToString()); } row_count_ = (int64_t) row_count; @@ -816,7 +821,7 @@ DeleteByRangeTask::OnExecute() { if (stat.IsNotFound()) { return SetError(SERVER_TABLE_NOT_EXIST, "Table " + table_name + " not exists"); } else { - return SetError(DB_META_TRANSACTION_FAILED, "Engine failed: " + stat.ToString()); + return SetError(DB_META_TRANSACTION_FAILED, stat.ToString()); } } @@ -842,7 +847,7 @@ DeleteByRangeTask::OnExecute() { #endif engine::Status status = DBWrapper::DB()->DeleteTable(table_name, dates); if (!status.ok()) { - return SetError(DB_META_TRANSACTION_FAILED, "Engine failed: " + stat.ToString()); + return SetError(DB_META_TRANSACTION_FAILED, stat.ToString()); } } catch (std::exception &ex) { @@ -878,7 +883,7 @@ PreloadTableTask::OnExecute() { //step 2: check table existence engine::Status stat = DBWrapper::DB()->PreloadTable(table_name_); if (!stat.ok()) { - return SetError(DB_META_TRANSACTION_FAILED, "Engine failed: " + stat.ToString()); + return SetError(DB_META_TRANSACTION_FAILED, stat.ToString()); } rc.ElapseFromBegin("totally cost"); @@ -919,7 +924,7 @@ DescribeIndexTask::OnExecute() { engine::TableIndex index; engine::Status stat = DBWrapper::DB()->DescribeIndex(table_name_, index); if (!stat.ok()) { - return SetError(DB_META_TRANSACTION_FAILED, "Engine failed: " + stat.ToString()); + return SetError(DB_META_TRANSACTION_FAILED, stat.ToString()); } index_param_->mutable_table_name()->set_table_name(table_name_); @@ -961,7 +966,7 @@ DropIndexTask::OnExecute() { //step 2: check table existence engine::Status stat = DBWrapper::DB()->DropIndex(table_name_); if (!stat.ok()) { - return SetError(DB_META_TRANSACTION_FAILED, "Engine failed: " + stat.ToString()); + return SetError(DB_META_TRANSACTION_FAILED, stat.ToString()); } rc.ElapseFromBegin("totally cost"); diff --git a/cpp/src/utils/ValidationUtil.cpp b/cpp/src/utils/ValidationUtil.cpp index 0e2e1c2b4f0dfffeb88cb94ac7704930b14859ac..2245496903aca085b8d28bfc10afe93c902c7506 100644 --- a/cpp/src/utils/ValidationUtil.cpp +++ b/cpp/src/utils/ValidationUtil.cpp @@ -92,6 +92,24 @@ ValidationUtil::ValidateTableIndexMetricType(int32_t metric_type) { return SERVER_SUCCESS; } +ServerError +ValidationUtil::ValidateSearchTopk(int64_t top_k, const engine::meta::TableSchema& table_schema) { + if (top_k <= 0 || top_k > 1024) { + return SERVER_INVALID_TOPK; + } + + return SERVER_SUCCESS; +} + +ServerError +ValidationUtil::ValidateSearchNprobe(int64_t nprobe, const engine::meta::TableSchema& table_schema) { + if (nprobe <= 0 || nprobe > table_schema.nlist_) { + return SERVER_INVALID_NPROBE; + } + + return SERVER_SUCCESS; +} + ServerError ValidationUtil::ValidateGpuIndex(uint32_t gpu_index) { int num_devices = 0; diff --git a/cpp/src/utils/ValidationUtil.h b/cpp/src/utils/ValidationUtil.h index 4792500f6719089e49b447f65b1b84f95f1d4a6a..d1d84f710465a6111d093c8953dd95cb8c08256a 100644 --- a/cpp/src/utils/ValidationUtil.h +++ b/cpp/src/utils/ValidationUtil.h @@ -1,5 +1,6 @@ #pragma once +#include "db/meta/MetaTypes.h" #include "Error.h" namespace zilliz { @@ -26,6 +27,12 @@ public: static ServerError ValidateTableIndexMetricType(int32_t metric_type); + static ServerError + ValidateSearchTopk(int64_t top_k, const engine::meta::TableSchema& table_schema); + + static ServerError + ValidateSearchNprobe(int64_t nprobe, const engine::meta::TableSchema& table_schema); + static ServerError ValidateGpuIndex(uint32_t gpu_index); diff --git a/cpp/unittest/db/db_tests.cpp b/cpp/unittest/db/db_tests.cpp index 668f5b512ae0fafcc762a973d8db7080eba4064d..be8748ce5cffc8465dfd3b882c772fc94942bf5d 100644 --- a/cpp/unittest/db/db_tests.cpp +++ b/cpp/unittest/db/db_tests.cpp @@ -34,7 +34,6 @@ namespace { engine::meta::TableSchema table_info; table_info.dimension_ = TABLE_DIM; table_info.table_id_ = TABLE_NAME; - table_info.engine_type_ = (int)engine::EngineType::FAISS_IDMAP; return table_info; } @@ -263,7 +262,9 @@ TEST_F(DBTest, SEARCH_TEST) { ASSERT_STATS(stat); } - db_->BuildIndex(TABLE_NAME); // wait until build index finish + engine::TableIndex index; + index.engine_type_ = (int)engine::EngineType::FAISS_IDMAP; + db_->CreateIndex(TABLE_NAME, index); // wait until build index finish { engine::QueryResults results; @@ -273,7 +274,7 @@ TEST_F(DBTest, SEARCH_TEST) { {//search by specify index file engine::meta::DatesT dates; - std::vector file_ids = {"4", "5", "6"}; + std::vector file_ids = {"1", "2", "3", "4", "5", "6"}; engine::QueryResults results; stat = db_->Query(TABLE_NAME, file_ids, k, nq, 10, xq.data(), dates, results); ASSERT_STATS(stat); @@ -305,7 +306,12 @@ TEST_F(DBTest, PRELOADTABLE_TEST) { db_->InsertVectors(TABLE_NAME, nb, xb.data(), target_ids); ASSERT_EQ(target_ids.size(), nb); } - db_->BuildIndex(TABLE_NAME); + + sleep(2); + + engine::TableIndex index; + index.engine_type_ = (int)engine::EngineType::FAISS_IDMAP; + db_->CreateIndex(TABLE_NAME, index); // wait until build index finish int64_t prev_cache_usage = cache::CpuCacheMgr::GetInstance()->CacheUsage(); stat = db_->PreloadTable(TABLE_NAME); diff --git a/cpp/unittest/server/util_test.cpp b/cpp/unittest/server/util_test.cpp index c0b1c83cf8a505db9d62d4c11b6519e49e86ada3..290655f565547a53077ffdbae5b9483101ed4c09 100644 --- a/cpp/unittest/server/util_test.cpp +++ b/cpp/unittest/server/util_test.cpp @@ -217,6 +217,21 @@ TEST(UtilTest, VALIDATE_INDEX_TEST) { ASSERT_EQ(server::ValidationUtil::ValidateTableIndexMetricType(2), server::SERVER_SUCCESS); } +TEST(ValidationUtilTest, ValidateTopkTest) { + engine::meta::TableSchema schema; + ASSERT_EQ(server::ValidationUtil::ValidateSearchTopk(10, schema), server::SERVER_SUCCESS); + ASSERT_NE(server::ValidationUtil::ValidateSearchTopk(65536, schema), server::SERVER_SUCCESS); + ASSERT_NE(server::ValidationUtil::ValidateSearchTopk(0, schema), server::SERVER_SUCCESS); +} + +TEST(ValidationUtilTest, ValidateNprobeTest) { + engine::meta::TableSchema schema; + schema.nlist_ = 100; + ASSERT_EQ(server::ValidationUtil::ValidateSearchNprobe(10, schema), server::SERVER_SUCCESS); + ASSERT_NE(server::ValidationUtil::ValidateSearchNprobe(0, schema), server::SERVER_SUCCESS); + ASSERT_NE(server::ValidationUtil::ValidateSearchNprobe(101, schema), server::SERVER_SUCCESS); +} + TEST(ValidationUtilTest, ValidateGpuTest) { ASSERT_EQ(server::ValidationUtil::ValidateGpuIndex(0), server::SERVER_SUCCESS); ASSERT_NE(server::ValidationUtil::ValidateGpuIndex(100), server::SERVER_SUCCESS);