#1571 Meta engine type become IDMAP after drop index for BINARY table

Signed-off-by: Ngroot <yihua.mo@zilliz.com>

CHANGELOG.md

@@ -45,6 +45,7 @@ Please mark all change in change log and use the issue from GitHub
 -   \#1535 Degradation searching performance with metric_type: binary_idmap
 -   \#1556 Index file not created after table and index created
 -   \#1560 Search crashed with Super-high dimensional binary vector
+-   \#1571 Meta engine type become IDMAP after drop index for BINARY table
 
 ## Feature
 -   \#216 Add CLI to get server info

core/src/db/DBImpl.cpp

@@ -366,8 +366,8 @@ DBImpl::PreloadTable(const std::string& table_id) {
         if (file.file_type_ == meta::TableFileSchema::FILE_TYPE::RAW ||
             file.file_type_ == meta::TableFileSchema::FILE_TYPE::TO_INDEX ||
             file.file_type_ == meta::TableFileSchema::FILE_TYPE::BACKUP) {
-            engine_type = server::ValidationUtil::IsBinaryMetricType(file.metric_type_) ? EngineType::FAISS_BIN_IDMAP
-                                                                                        : EngineType::FAISS_IDMAP;
+            engine_type =
+                utils::IsBinaryMetricType(file.metric_type_) ? EngineType::FAISS_BIN_IDMAP : EngineType::FAISS_IDMAP;
         } else {
             engine_type = (EngineType)file.engine_type_;
         }
@@ -1019,7 +1019,7 @@ DBImpl::GetVectorByIdHelper(const std::string& table_id, IDNumber vector_id, Vec
                 auto deleted = std::find(deleted_docs.begin(), deleted_docs.end(), offset);
                 if (deleted == deleted_docs.end()) {
                     // Load raw vector
-                    bool is_binary = server::ValidationUtil::IsBinaryMetricType(file.metric_type_);
+                    bool is_binary = utils::IsBinaryMetricType(file.metric_type_);
                     size_t single_vector_bytes = is_binary ? file.dimension_ / 8 : file.dimension_ * sizeof(float);
                     std::vector<uint8_t> raw_vector;
                     status = segment_reader.LoadVectors(offset * single_vector_bytes, single_vector_bytes, raw_vector);

core/src/db/Utils.cpp

@@ -220,6 +220,19 @@ IsRawIndexType(int32_t type) {
     return (type == (int32_t)EngineType::FAISS_IDMAP) || (type == (int32_t)EngineType::FAISS_BIN_IDMAP);
 }
 
+bool
+IsBinaryIndexType(int32_t index_type) {
+    return (index_type == (int32_t)engine::EngineType::FAISS_BIN_IDMAP) ||
+           (index_type == (int32_t)engine::EngineType::FAISS_BIN_IVFFLAT);
+}
+
+bool
+IsBinaryMetricType(int32_t metric_type) {
+    return (metric_type == (int32_t)engine::MetricType::HAMMING) ||
+           (metric_type == (int32_t)engine::MetricType::JACCARD) ||
+           (metric_type == (int32_t)engine::MetricType::TANIMOTO);
+}
+
 meta::DateT
 GetDate(const std::time_t& t, int day_delta) {
     struct tm ltm;

core/src/db/Utils.h

@@ -48,6 +48,12 @@ IsSameIndex(const TableIndex& index1, const TableIndex& index2);
 bool
 IsRawIndexType(int32_t type);
 
+static bool
+IsBinaryIndexType(int32_t index_type);
+
+bool
+IsBinaryMetricType(int32_t metric_type);
+
 meta::DateT
 GetDate(const std::time_t& t, int day_delta = 0);
 meta::DateT

core/src/db/engine/ExecutionEngineImpl.cpp

@@ -100,9 +100,8 @@ ExecutionEngineImpl::ExecutionEngineImpl(uint16_t dimension, const std::string&
       index_type_(index_type),
       metric_type_(metric_type),
       index_params_(index_params) {
-    EngineType tmp_index_type = server::ValidationUtil::IsBinaryMetricType((int32_t)metric_type)
-                                    ? EngineType::FAISS_BIN_IDMAP
-                                    : EngineType::FAISS_IDMAP;
+    EngineType tmp_index_type =
+        utils::IsBinaryMetricType((int32_t)metric_type) ? EngineType::FAISS_BIN_IDMAP : EngineType::FAISS_IDMAP;
     index_ = CreatetVecIndex(tmp_index_type);
     if (!index_) {
         throw Exception(DB_ERROR, "Unsupported index type");

core/src/db/meta/SqliteMetaImpl.cpp

@@ -804,8 +804,18 @@ SqliteMetaImpl::DropTableIndex(const std::string& table_id) {
                                        c(&TableFileSchema::file_type_) == (int)TableFileSchema::BACKUP));
 
         // set table index type to raw
+        auto groups = ConnectorPtr->select(columns(&TableSchema::metric_type_),
+                                           where(c(&TableSchema::table_id_) == table_id));
+
+        int32_t raw_engine_type = DEFAULT_ENGINE_TYPE;
+        if (groups.size() == 1) {
+            int32_t metric_type_ = std::get<0>(groups[0]);
+            if (engine::utils::IsBinaryMetricType(metric_type_)) {
+                raw_engine_type = (int32_t)EngineType::FAISS_BIN_IDMAP;
+            }
+        }
         ConnectorPtr->update_all(
-            set(c(&TableSchema::engine_type_) = DEFAULT_ENGINE_TYPE, c(&TableSchema::index_params_) = "{}"),
+            set(c(&TableSchema::engine_type_) = raw_engine_type, c(&TableSchema::index_params_) = "{}"),
             where(c(&TableSchema::table_id_) == table_id));
 
         ENGINE_LOG_DEBUG << "Successfully drop table index, table id = " << table_id;
@@ -1189,29 +1199,21 @@ SqliteMetaImpl::FilesByType(const std::string& table_id, const std::vector<int>&
                 file_schema.metric_type_ = table_schema.metric_type_;
 
                 switch (file_schema.file_type_) {
-                    case (int)TableFileSchema::RAW:
-                        ++raw_count;
-                        break;
-                    case (int)TableFileSchema::NEW:
-                        ++new_count;
+                    case (int)TableFileSchema::RAW:++raw_count;
                         break;
-                    case (int)TableFileSchema::NEW_MERGE:
-                        ++new_merge_count;
+                    case (int)TableFileSchema::NEW:++new_count;
                         break;
-                    case (int)TableFileSchema::NEW_INDEX:
-                        ++new_index_count;
+                    case (int)TableFileSchema::NEW_MERGE:++new_merge_count;
                         break;
-                    case (int)TableFileSchema::TO_INDEX:
-                        ++to_index_count;
+                    case (int)TableFileSchema::NEW_INDEX:++new_index_count;
                         break;
-                    case (int)TableFileSchema::INDEX:
-                        ++index_count;
+                    case (int)TableFileSchema::TO_INDEX:++to_index_count;
                         break;
-                    case (int)TableFileSchema::BACKUP:
-                        ++backup_count;
+                    case (int)TableFileSchema::INDEX:++index_count;
                         break;
-                    default:
+                    case (int)TableFileSchema::BACKUP:++backup_count;
                         break;
+                    default:break;
                 }
 
                 auto status = utils::GetTableFilePath(options_, file_schema);
@@ -1225,29 +1227,23 @@ SqliteMetaImpl::FilesByType(const std::string& table_id, const std::vector<int>&
             std::string msg = "Get table files by type.";
             for (int file_type : file_types) {
                 switch (file_type) {
-                    case (int)TableFileSchema::RAW:
-                        msg = msg + " raw files:" + std::to_string(raw_count);
-                        break;
-                    case (int)TableFileSchema::NEW:
-                        msg = msg + " new files:" + std::to_string(new_count);
+                    case (int)TableFileSchema::RAW:msg = msg + " raw files:" + std::to_string(raw_count);
                         break;
-                    case (int)TableFileSchema::NEW_MERGE:
-                        msg = msg + " new_merge files:" + std::to_string(new_merge_count);
+                    case (int)TableFileSchema::NEW:msg = msg + " new files:" + std::to_string(new_count);
                         break;
-                    case (int)TableFileSchema::NEW_INDEX:
-                        msg = msg + " new_index files:" + std::to_string(new_index_count);
+                    case (int)TableFileSchema::NEW_MERGE:msg = msg + " new_merge files:"
+                                                               + std::to_string(new_merge_count);
                         break;
-                    case (int)TableFileSchema::TO_INDEX:
-                        msg = msg + " to_index files:" + std::to_string(to_index_count);
+                    case (int)TableFileSchema::NEW_INDEX:msg = msg + " new_index files:"
+                                                               + std::to_string(new_index_count);
                         break;
-                    case (int)TableFileSchema::INDEX:
-                        msg = msg + " index files:" + std::to_string(index_count);
+                    case (int)TableFileSchema::TO_INDEX:msg = msg + " to_index files:" + std::to_string(to_index_count);
                         break;
-                    case (int)TableFileSchema::BACKUP:
-                        msg = msg + " backup files:" + std::to_string(backup_count);
+                    case (int)TableFileSchema::INDEX:msg = msg + " index files:" + std::to_string(index_count);
                         break;
-                    default:
+                    case (int)TableFileSchema::BACKUP:msg = msg + " backup files:" + std::to_string(backup_count);
                         break;
+                    default:break;
                 }
             }
             ENGINE_LOG_DEBUG << msg;

core/src/scheduler/task/BuildIndexTask.cpp

@@ -17,6 +17,7 @@
 #include <thread>
 #include <utility>
 
+#include "db/Utils.h"
 #include "db/engine/EngineFactory.h"
 #include "metrics/Metrics.h"
 #include "scheduler/job/BuildIndexJob.h"
@@ -35,8 +36,8 @@ XBuildIndexTask::XBuildIndexTask(TableFileSchemaPtr file, TaskLabelPtr label)
         if (file->file_type_ == TableFileSchema::FILE_TYPE::RAW ||
             file->file_type_ == TableFileSchema::FILE_TYPE::TO_INDEX ||
             file->file_type_ == TableFileSchema::FILE_TYPE::BACKUP) {
-            engine_type = server::ValidationUtil::IsBinaryMetricType(file->metric_type_) ? EngineType::FAISS_BIN_IDMAP
-                                                                                         : EngineType::FAISS_IDMAP;
+            engine_type = engine::utils::IsBinaryMetricType(file->metric_type_) ? EngineType::FAISS_BIN_IDMAP
+                                                                                : EngineType::FAISS_IDMAP;
         } else {
             engine_type = (EngineType)file->engine_type_;
         }

core/src/scheduler/task/SearchTask.cpp

@@ -109,8 +109,8 @@ XSearchTask::XSearchTask(const std::shared_ptr<server::Context>& context, TableF
         if (file->file_type_ == TableFileSchema::FILE_TYPE::RAW ||
             file->file_type_ == TableFileSchema::FILE_TYPE::TO_INDEX ||
             file->file_type_ == TableFileSchema::FILE_TYPE::BACKUP) {
-            engine_type = server::ValidationUtil::IsBinaryMetricType(file->metric_type_) ? EngineType::FAISS_BIN_IDMAP
-                                                                                         : EngineType::FAISS_IDMAP;
+            engine_type = engine::utils::IsBinaryMetricType(file->metric_type_) ? EngineType::FAISS_BIN_IDMAP
+                                                                                : EngineType::FAISS_IDMAP;
         } else {
             engine_type = (EngineType)file->engine_type_;
         }

core/src/server/delivery/request/CreateIndexRequest.cpp

@@ -10,6 +10,7 @@
 // or implied. See the License for the specific language governing permissions and limitations under the License.
 
 #include "server/delivery/request/CreateIndexRequest.h"
+#include "db/Utils.h"
 #include "server/Config.h"
 #include "server/DBWrapper.h"
 #include "utils/Log.h"
@@ -83,7 +84,7 @@ CreateIndexRequest::OnExecute() {
         status = DBWrapper::DB()->DescribeTable(table_info);
 
         int32_t adapter_index_type = index_type_;
-        if (ValidationUtil::IsBinaryMetricType(table_info.metric_type_)) {  // binary vector not allow
+        if (engine::utils::IsBinaryMetricType(table_info.metric_type_)) {  // binary vector not allow
             if (adapter_index_type == static_cast<int32_t>(engine::EngineType::FAISS_IDMAP)) {
                 adapter_index_type = static_cast<int32_t>(engine::EngineType::FAISS_BIN_IDMAP);
             } else if (adapter_index_type == static_cast<int32_t>(engine::EngineType::FAISS_IVFFLAT)) {

core/src/server/delivery/request/CreateTableRequest.cpp

@@ -10,6 +10,7 @@
 // or implied. See the License for the specific language governing permissions and limitations under the License.
 
 #include "server/delivery/request/CreateTableRequest.h"
+#include "db/Utils.h"
 #include "server/DBWrapper.h"
 #include "server/delivery/request/BaseRequest.h"
 #include "utils/Log.h"
@@ -78,7 +79,7 @@ CreateTableRequest::OnExecute() {
         table_info.metric_type_ = metric_type_;
 
         // some metric type only support binary vector, adapt the index type
-        if (ValidationUtil::IsBinaryMetricType(metric_type_)) {
+        if (engine::utils::IsBinaryMetricType(metric_type_)) {
             if (table_info.engine_type_ == static_cast<int32_t>(engine::EngineType::FAISS_IDMAP)) {
                 table_info.engine_type_ = static_cast<int32_t>(engine::EngineType::FAISS_BIN_IDMAP);
             } else if (table_info.engine_type_ == static_cast<int32_t>(engine::EngineType::FAISS_IVFFLAT)) {

core/src/server/delivery/request/InsertRequest.cpp

@@ -10,6 +10,7 @@
 // or implied. See the License for the specific language governing permissions and limitations under the License.
 
 #include "server/delivery/request/InsertRequest.h"
+#include "db/Utils.h"
 #include "server/DBWrapper.h"
 #include "utils/CommonUtil.h"
 #include "utils/Log.h"
@@ -115,7 +116,7 @@ InsertRequest::OnExecute() {
 #endif
         // step 4: some metric type doesn't support float vectors
         if (!vectors_data_.float_data_.empty()) {  // insert float vectors
-            if (ValidationUtil::IsBinaryMetricType(table_schema.metric_type_)) {
+            if (engine::utils::IsBinaryMetricType(table_schema.metric_type_)) {
                 return Status(SERVER_INVALID_ROWRECORD_ARRAY, "Table metric type doesn't support float vectors.");
             }
 
@@ -131,7 +132,7 @@ InsertRequest::OnExecute() {
                               "The vector dimension must be equal to the table dimension.");
             }
         } else if (!vectors_data_.binary_data_.empty()) {  // insert binary vectors
-            if (!ValidationUtil::IsBinaryMetricType(table_schema.metric_type_)) {
+            if (!engine::utils::IsBinaryMetricType(table_schema.metric_type_)) {
                 return Status(SERVER_INVALID_ROWRECORD_ARRAY, "Table metric type doesn't support binary vectors.");
             }
 

core/src/server/delivery/request/SearchRequest.cpp

@@ -10,6 +10,7 @@
 // or implied. See the License for the specific language governing permissions and limitations under the License.
 
 #include "server/delivery/request/SearchRequest.h"
+#include "db/Utils.h"
 #include "server/DBWrapper.h"
 #include "utils/CommonUtil.h"
 #include "utils/Log.h"
@@ -103,7 +104,7 @@ SearchRequest::OnExecute() {
         rc.RecordSection("check validation");
 
         // step 4: check metric type
-        if (ValidationUtil::IsBinaryMetricType(table_schema.metric_type_)) {
+        if (engine::utils::IsBinaryMetricType(table_schema.metric_type_)) {
             // check prepared binary data
             if (vectors_data_.binary_data_.size() % vector_count != 0) {
                 return Status(SERVER_INVALID_ROWRECORD_ARRAY,

core/src/utils/ValidationUtil.cpp

@@ -264,12 +264,6 @@ ValidationUtil::ValidateSearchParams(const milvus::json& search_params, const en
     return Status::OK();
 }
 
-bool
-ValidationUtil::IsBinaryIndexType(int32_t index_type) {
-    return (index_type == static_cast<int32_t>(engine::EngineType::FAISS_BIN_IDMAP)) ||
-           (index_type == static_cast<int32_t>(engine::EngineType::FAISS_BIN_IVFFLAT));
-}
-
 Status
 ValidationUtil::ValidateTableIndexFileSize(int64_t index_file_size) {
     if (index_file_size <= 0 || index_file_size > INDEX_FILE_SIZE_LIMIT) {
@@ -294,13 +288,6 @@ ValidationUtil::ValidateTableIndexMetricType(int32_t metric_type) {
     return Status::OK();
 }
 
-bool
-ValidationUtil::IsBinaryMetricType(int32_t metric_type) {
-    return (metric_type == static_cast<int32_t>(engine::MetricType::HAMMING)) ||
-           (metric_type == static_cast<int32_t>(engine::MetricType::JACCARD)) ||
-           (metric_type == static_cast<int32_t>(engine::MetricType::TANIMOTO));
-}
-
 Status
 ValidationUtil::ValidateSearchTopk(int64_t top_k, const engine::meta::TableSchema& table_schema) {
     if (top_k <= 0 || top_k > 2048) {

core/src/utils/ValidationUtil.h

@@ -43,18 +43,12 @@ class ValidationUtil {
     ValidateSearchParams(const milvus::json& search_params, const engine::meta::TableSchema& table_schema,
                          int64_t topk);
 
-    static bool
-    IsBinaryIndexType(int32_t index_type);
-
     static Status
     ValidateTableIndexFileSize(int64_t index_file_size);
 
     static Status
     ValidateTableIndexMetricType(int32_t metric_type);
 
-    static bool
-    IsBinaryMetricType(int32_t metric_type);
-
     static Status
     ValidateSearchTopk(int64_t top_k, const engine::meta::TableSchema& table_schema);