diff --git a/CHANGELOG.md b/CHANGELOG.md index 69e40ea6fb2425111a4e4ba80c06b31868c7ec86..6ffed248afff66ca8b6392cfc8f403aa06d77d99 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -45,6 +45,7 @@ Please mark all change in change log and use the issue from GitHub - \#1535 Degradation searching performance with metric_type: binary_idmap - \#1556 Index file not created after table and index created - \#1560 Search crashed with Super-high dimensional binary vector +- \#1571 Meta engine type become IDMAP after drop index for BINARY table ## Feature - \#216 Add CLI to get server info diff --git a/core/src/db/DBImpl.cpp b/core/src/db/DBImpl.cpp index 29ed17308afd776c3c2367ce80008cc467f947da..c32819a8302ba339db0d8f92be2d60f9ca813856 100644 --- a/core/src/db/DBImpl.cpp +++ b/core/src/db/DBImpl.cpp @@ -366,8 +366,8 @@ DBImpl::PreloadTable(const std::string& table_id) { if (file.file_type_ == meta::TableFileSchema::FILE_TYPE::RAW || file.file_type_ == meta::TableFileSchema::FILE_TYPE::TO_INDEX || file.file_type_ == meta::TableFileSchema::FILE_TYPE::BACKUP) { - engine_type = server::ValidationUtil::IsBinaryMetricType(file.metric_type_) ? EngineType::FAISS_BIN_IDMAP - : EngineType::FAISS_IDMAP; + engine_type = + utils::IsBinaryMetricType(file.metric_type_) ? EngineType::FAISS_BIN_IDMAP : EngineType::FAISS_IDMAP; } else { engine_type = (EngineType)file.engine_type_; } @@ -1019,7 +1019,7 @@ DBImpl::GetVectorByIdHelper(const std::string& table_id, IDNumber vector_id, Vec auto deleted = std::find(deleted_docs.begin(), deleted_docs.end(), offset); if (deleted == deleted_docs.end()) { // Load raw vector - bool is_binary = server::ValidationUtil::IsBinaryMetricType(file.metric_type_); + bool is_binary = utils::IsBinaryMetricType(file.metric_type_); size_t single_vector_bytes = is_binary ? file.dimension_ / 8 : file.dimension_ * sizeof(float); std::vector raw_vector; status = segment_reader.LoadVectors(offset * single_vector_bytes, single_vector_bytes, raw_vector); diff --git a/core/src/db/Utils.cpp b/core/src/db/Utils.cpp index f9ac5dfe4086857c16553d42554b95e9fe8be6e4..d184cea0857d67bd86d75d509b7ccf4077affc10 100644 --- a/core/src/db/Utils.cpp +++ b/core/src/db/Utils.cpp @@ -220,6 +220,19 @@ IsRawIndexType(int32_t type) { return (type == (int32_t)EngineType::FAISS_IDMAP) || (type == (int32_t)EngineType::FAISS_BIN_IDMAP); } +bool +IsBinaryIndexType(int32_t index_type) { + return (index_type == (int32_t)engine::EngineType::FAISS_BIN_IDMAP) || + (index_type == (int32_t)engine::EngineType::FAISS_BIN_IVFFLAT); +} + +bool +IsBinaryMetricType(int32_t metric_type) { + return (metric_type == (int32_t)engine::MetricType::HAMMING) || + (metric_type == (int32_t)engine::MetricType::JACCARD) || + (metric_type == (int32_t)engine::MetricType::TANIMOTO); +} + meta::DateT GetDate(const std::time_t& t, int day_delta) { struct tm ltm; diff --git a/core/src/db/Utils.h b/core/src/db/Utils.h index 88197d7d5c9ce470eadf33eb68dbe78728af0fdf..c78b4fd7176bae5aee8c0831da308f944847378e 100644 --- a/core/src/db/Utils.h +++ b/core/src/db/Utils.h @@ -48,6 +48,12 @@ IsSameIndex(const TableIndex& index1, const TableIndex& index2); bool IsRawIndexType(int32_t type); +static bool +IsBinaryIndexType(int32_t index_type); + +bool +IsBinaryMetricType(int32_t metric_type); + meta::DateT GetDate(const std::time_t& t, int day_delta = 0); meta::DateT diff --git a/core/src/db/engine/ExecutionEngineImpl.cpp b/core/src/db/engine/ExecutionEngineImpl.cpp index 1fe820aa7a6ea8f6f12948106a72fa0edbb9acc5..9cb1dfb09fe32afa00f7d542793dd33d53e8ba04 100644 --- a/core/src/db/engine/ExecutionEngineImpl.cpp +++ b/core/src/db/engine/ExecutionEngineImpl.cpp @@ -100,9 +100,8 @@ ExecutionEngineImpl::ExecutionEngineImpl(uint16_t dimension, const std::string& index_type_(index_type), metric_type_(metric_type), index_params_(index_params) { - EngineType tmp_index_type = server::ValidationUtil::IsBinaryMetricType((int32_t)metric_type) - ? EngineType::FAISS_BIN_IDMAP - : EngineType::FAISS_IDMAP; + EngineType tmp_index_type = + utils::IsBinaryMetricType((int32_t)metric_type) ? EngineType::FAISS_BIN_IDMAP : EngineType::FAISS_IDMAP; index_ = CreatetVecIndex(tmp_index_type); if (!index_) { throw Exception(DB_ERROR, "Unsupported index type"); diff --git a/core/src/db/meta/SqliteMetaImpl.cpp b/core/src/db/meta/SqliteMetaImpl.cpp index ca37f2c2584850c6fee5ee678aefa43c921a3b65..ce5ff99742a27e32eae24cb72d8eb2940ce3f18d 100644 --- a/core/src/db/meta/SqliteMetaImpl.cpp +++ b/core/src/db/meta/SqliteMetaImpl.cpp @@ -804,8 +804,18 @@ SqliteMetaImpl::DropTableIndex(const std::string& table_id) { c(&TableFileSchema::file_type_) == (int)TableFileSchema::BACKUP)); // set table index type to raw + auto groups = ConnectorPtr->select(columns(&TableSchema::metric_type_), + where(c(&TableSchema::table_id_) == table_id)); + + int32_t raw_engine_type = DEFAULT_ENGINE_TYPE; + if (groups.size() == 1) { + int32_t metric_type_ = std::get<0>(groups[0]); + if (engine::utils::IsBinaryMetricType(metric_type_)) { + raw_engine_type = (int32_t)EngineType::FAISS_BIN_IDMAP; + } + } ConnectorPtr->update_all( - set(c(&TableSchema::engine_type_) = DEFAULT_ENGINE_TYPE, c(&TableSchema::index_params_) = "{}"), + set(c(&TableSchema::engine_type_) = raw_engine_type, c(&TableSchema::index_params_) = "{}"), where(c(&TableSchema::table_id_) == table_id)); ENGINE_LOG_DEBUG << "Successfully drop table index, table id = " << table_id; @@ -1189,29 +1199,21 @@ SqliteMetaImpl::FilesByType(const std::string& table_id, const std::vector& file_schema.metric_type_ = table_schema.metric_type_; switch (file_schema.file_type_) { - case (int)TableFileSchema::RAW: - ++raw_count; - break; - case (int)TableFileSchema::NEW: - ++new_count; + case (int)TableFileSchema::RAW:++raw_count; break; - case (int)TableFileSchema::NEW_MERGE: - ++new_merge_count; + case (int)TableFileSchema::NEW:++new_count; break; - case (int)TableFileSchema::NEW_INDEX: - ++new_index_count; + case (int)TableFileSchema::NEW_MERGE:++new_merge_count; break; - case (int)TableFileSchema::TO_INDEX: - ++to_index_count; + case (int)TableFileSchema::NEW_INDEX:++new_index_count; break; - case (int)TableFileSchema::INDEX: - ++index_count; + case (int)TableFileSchema::TO_INDEX:++to_index_count; break; - case (int)TableFileSchema::BACKUP: - ++backup_count; + case (int)TableFileSchema::INDEX:++index_count; break; - default: + case (int)TableFileSchema::BACKUP:++backup_count; break; + default:break; } auto status = utils::GetTableFilePath(options_, file_schema); @@ -1225,29 +1227,23 @@ SqliteMetaImpl::FilesByType(const std::string& table_id, const std::vector& std::string msg = "Get table files by type."; for (int file_type : file_types) { switch (file_type) { - case (int)TableFileSchema::RAW: - msg = msg + " raw files:" + std::to_string(raw_count); - break; - case (int)TableFileSchema::NEW: - msg = msg + " new files:" + std::to_string(new_count); + case (int)TableFileSchema::RAW:msg = msg + " raw files:" + std::to_string(raw_count); break; - case (int)TableFileSchema::NEW_MERGE: - msg = msg + " new_merge files:" + std::to_string(new_merge_count); + case (int)TableFileSchema::NEW:msg = msg + " new files:" + std::to_string(new_count); break; - case (int)TableFileSchema::NEW_INDEX: - msg = msg + " new_index files:" + std::to_string(new_index_count); + case (int)TableFileSchema::NEW_MERGE:msg = msg + " new_merge files:" + + std::to_string(new_merge_count); break; - case (int)TableFileSchema::TO_INDEX: - msg = msg + " to_index files:" + std::to_string(to_index_count); + case (int)TableFileSchema::NEW_INDEX:msg = msg + " new_index files:" + + std::to_string(new_index_count); break; - case (int)TableFileSchema::INDEX: - msg = msg + " index files:" + std::to_string(index_count); + case (int)TableFileSchema::TO_INDEX:msg = msg + " to_index files:" + std::to_string(to_index_count); break; - case (int)TableFileSchema::BACKUP: - msg = msg + " backup files:" + std::to_string(backup_count); + case (int)TableFileSchema::INDEX:msg = msg + " index files:" + std::to_string(index_count); break; - default: + case (int)TableFileSchema::BACKUP:msg = msg + " backup files:" + std::to_string(backup_count); break; + default:break; } } ENGINE_LOG_DEBUG << msg; diff --git a/core/src/scheduler/task/BuildIndexTask.cpp b/core/src/scheduler/task/BuildIndexTask.cpp index 13d3b4a6119137e14ddc2376ead074c557997c74..666b9777316616f9ca506eb0e06417236c9da8f6 100644 --- a/core/src/scheduler/task/BuildIndexTask.cpp +++ b/core/src/scheduler/task/BuildIndexTask.cpp @@ -17,6 +17,7 @@ #include #include +#include "db/Utils.h" #include "db/engine/EngineFactory.h" #include "metrics/Metrics.h" #include "scheduler/job/BuildIndexJob.h" @@ -35,8 +36,8 @@ XBuildIndexTask::XBuildIndexTask(TableFileSchemaPtr file, TaskLabelPtr label) if (file->file_type_ == TableFileSchema::FILE_TYPE::RAW || file->file_type_ == TableFileSchema::FILE_TYPE::TO_INDEX || file->file_type_ == TableFileSchema::FILE_TYPE::BACKUP) { - engine_type = server::ValidationUtil::IsBinaryMetricType(file->metric_type_) ? EngineType::FAISS_BIN_IDMAP - : EngineType::FAISS_IDMAP; + engine_type = engine::utils::IsBinaryMetricType(file->metric_type_) ? EngineType::FAISS_BIN_IDMAP + : EngineType::FAISS_IDMAP; } else { engine_type = (EngineType)file->engine_type_; } diff --git a/core/src/scheduler/task/SearchTask.cpp b/core/src/scheduler/task/SearchTask.cpp index 0b32f35ab56e838fb3f0f77d954c95a72a705094..6331c727c75eb5ac105ffab5e85cea2a12b3b0a0 100644 --- a/core/src/scheduler/task/SearchTask.cpp +++ b/core/src/scheduler/task/SearchTask.cpp @@ -109,8 +109,8 @@ XSearchTask::XSearchTask(const std::shared_ptr& context, TableF if (file->file_type_ == TableFileSchema::FILE_TYPE::RAW || file->file_type_ == TableFileSchema::FILE_TYPE::TO_INDEX || file->file_type_ == TableFileSchema::FILE_TYPE::BACKUP) { - engine_type = server::ValidationUtil::IsBinaryMetricType(file->metric_type_) ? EngineType::FAISS_BIN_IDMAP - : EngineType::FAISS_IDMAP; + engine_type = engine::utils::IsBinaryMetricType(file->metric_type_) ? EngineType::FAISS_BIN_IDMAP + : EngineType::FAISS_IDMAP; } else { engine_type = (EngineType)file->engine_type_; } diff --git a/core/src/server/delivery/request/CreateIndexRequest.cpp b/core/src/server/delivery/request/CreateIndexRequest.cpp index 5836b3cc095278773e3ba20fada429835098a5be..521fe7aa58892476a8ddb8f0d6ad96c80e1dcbdd 100644 --- a/core/src/server/delivery/request/CreateIndexRequest.cpp +++ b/core/src/server/delivery/request/CreateIndexRequest.cpp @@ -10,6 +10,7 @@ // or implied. See the License for the specific language governing permissions and limitations under the License. #include "server/delivery/request/CreateIndexRequest.h" +#include "db/Utils.h" #include "server/Config.h" #include "server/DBWrapper.h" #include "utils/Log.h" @@ -83,7 +84,7 @@ CreateIndexRequest::OnExecute() { status = DBWrapper::DB()->DescribeTable(table_info); int32_t adapter_index_type = index_type_; - if (ValidationUtil::IsBinaryMetricType(table_info.metric_type_)) { // binary vector not allow + if (engine::utils::IsBinaryMetricType(table_info.metric_type_)) { // binary vector not allow if (adapter_index_type == static_cast(engine::EngineType::FAISS_IDMAP)) { adapter_index_type = static_cast(engine::EngineType::FAISS_BIN_IDMAP); } else if (adapter_index_type == static_cast(engine::EngineType::FAISS_IVFFLAT)) { diff --git a/core/src/server/delivery/request/CreateTableRequest.cpp b/core/src/server/delivery/request/CreateTableRequest.cpp index 15d07b8cfd9afc3aabc66e325b00fbecb34b2eb3..d92db5a94d303ceb50a1606213d099c2a3010f03 100644 --- a/core/src/server/delivery/request/CreateTableRequest.cpp +++ b/core/src/server/delivery/request/CreateTableRequest.cpp @@ -10,6 +10,7 @@ // or implied. See the License for the specific language governing permissions and limitations under the License. #include "server/delivery/request/CreateTableRequest.h" +#include "db/Utils.h" #include "server/DBWrapper.h" #include "server/delivery/request/BaseRequest.h" #include "utils/Log.h" @@ -78,7 +79,7 @@ CreateTableRequest::OnExecute() { table_info.metric_type_ = metric_type_; // some metric type only support binary vector, adapt the index type - if (ValidationUtil::IsBinaryMetricType(metric_type_)) { + if (engine::utils::IsBinaryMetricType(metric_type_)) { if (table_info.engine_type_ == static_cast(engine::EngineType::FAISS_IDMAP)) { table_info.engine_type_ = static_cast(engine::EngineType::FAISS_BIN_IDMAP); } else if (table_info.engine_type_ == static_cast(engine::EngineType::FAISS_IVFFLAT)) { diff --git a/core/src/server/delivery/request/InsertRequest.cpp b/core/src/server/delivery/request/InsertRequest.cpp index 729dc4d292d74e9e56927c38d81ba59cb7506a91..a739d86b9f113b57cf6ea899ed727883fc09f5e7 100644 --- a/core/src/server/delivery/request/InsertRequest.cpp +++ b/core/src/server/delivery/request/InsertRequest.cpp @@ -10,6 +10,7 @@ // or implied. See the License for the specific language governing permissions and limitations under the License. #include "server/delivery/request/InsertRequest.h" +#include "db/Utils.h" #include "server/DBWrapper.h" #include "utils/CommonUtil.h" #include "utils/Log.h" @@ -115,7 +116,7 @@ InsertRequest::OnExecute() { #endif // step 4: some metric type doesn't support float vectors if (!vectors_data_.float_data_.empty()) { // insert float vectors - if (ValidationUtil::IsBinaryMetricType(table_schema.metric_type_)) { + if (engine::utils::IsBinaryMetricType(table_schema.metric_type_)) { return Status(SERVER_INVALID_ROWRECORD_ARRAY, "Table metric type doesn't support float vectors."); } @@ -131,7 +132,7 @@ InsertRequest::OnExecute() { "The vector dimension must be equal to the table dimension."); } } else if (!vectors_data_.binary_data_.empty()) { // insert binary vectors - if (!ValidationUtil::IsBinaryMetricType(table_schema.metric_type_)) { + if (!engine::utils::IsBinaryMetricType(table_schema.metric_type_)) { return Status(SERVER_INVALID_ROWRECORD_ARRAY, "Table metric type doesn't support binary vectors."); } diff --git a/core/src/server/delivery/request/SearchRequest.cpp b/core/src/server/delivery/request/SearchRequest.cpp index 0b12f91f5d372bcce557c634ab733e7606fd59b8..e31be87b403aefbb50926394ec60a163312ef14c 100644 --- a/core/src/server/delivery/request/SearchRequest.cpp +++ b/core/src/server/delivery/request/SearchRequest.cpp @@ -10,6 +10,7 @@ // or implied. See the License for the specific language governing permissions and limitations under the License. #include "server/delivery/request/SearchRequest.h" +#include "db/Utils.h" #include "server/DBWrapper.h" #include "utils/CommonUtil.h" #include "utils/Log.h" @@ -103,7 +104,7 @@ SearchRequest::OnExecute() { rc.RecordSection("check validation"); // step 4: check metric type - if (ValidationUtil::IsBinaryMetricType(table_schema.metric_type_)) { + if (engine::utils::IsBinaryMetricType(table_schema.metric_type_)) { // check prepared binary data if (vectors_data_.binary_data_.size() % vector_count != 0) { return Status(SERVER_INVALID_ROWRECORD_ARRAY, diff --git a/core/src/utils/ValidationUtil.cpp b/core/src/utils/ValidationUtil.cpp index 84f260c5089184632e3d5e1d0e91b08d17e02db6..616375f08e4066a8f0904a8ae0195487dbf96310 100644 --- a/core/src/utils/ValidationUtil.cpp +++ b/core/src/utils/ValidationUtil.cpp @@ -264,12 +264,6 @@ ValidationUtil::ValidateSearchParams(const milvus::json& search_params, const en return Status::OK(); } -bool -ValidationUtil::IsBinaryIndexType(int32_t index_type) { - return (index_type == static_cast(engine::EngineType::FAISS_BIN_IDMAP)) || - (index_type == static_cast(engine::EngineType::FAISS_BIN_IVFFLAT)); -} - Status ValidationUtil::ValidateTableIndexFileSize(int64_t index_file_size) { if (index_file_size <= 0 || index_file_size > INDEX_FILE_SIZE_LIMIT) { @@ -294,13 +288,6 @@ ValidationUtil::ValidateTableIndexMetricType(int32_t metric_type) { return Status::OK(); } -bool -ValidationUtil::IsBinaryMetricType(int32_t metric_type) { - return (metric_type == static_cast(engine::MetricType::HAMMING)) || - (metric_type == static_cast(engine::MetricType::JACCARD)) || - (metric_type == static_cast(engine::MetricType::TANIMOTO)); -} - Status ValidationUtil::ValidateSearchTopk(int64_t top_k, const engine::meta::TableSchema& table_schema) { if (top_k <= 0 || top_k > 2048) { diff --git a/core/src/utils/ValidationUtil.h b/core/src/utils/ValidationUtil.h index 4fa2df1619ded489a014b1b1c3d50091e42a3a68..481cb31f1b2f27c37e6970661cceb906a9aa7d21 100644 --- a/core/src/utils/ValidationUtil.h +++ b/core/src/utils/ValidationUtil.h @@ -43,18 +43,12 @@ class ValidationUtil { ValidateSearchParams(const milvus::json& search_params, const engine::meta::TableSchema& table_schema, int64_t topk); - static bool - IsBinaryIndexType(int32_t index_type); - static Status ValidateTableIndexFileSize(int64_t index_file_size); static Status ValidateTableIndexMetricType(int32_t metric_type); - static bool - IsBinaryMetricType(int32_t metric_type); - static Status ValidateSearchTopk(int64_t top_k, const engine::meta::TableSchema& table_schema);