#1532 Search with ivf_flat failed with open-dataset: sift-256-hamming

Signed-off-by: Ngroot <yihua.mo@zilliz.com>

CHANGELOG.md

@@ -40,6 +40,7 @@ Please mark all change in change log and use the issue from GitHub
 -   \#1521 Make cache_insert_data take effect in-service
 -   \#1525 Add setter API for config preload_table
 -   \#1530 Set table file with correct engine type in meta
+-   \#1532 Search with ivf_flat failed with open-dataset: sift-256-hamming
 -   \#1535 Degradation searching performance with metric_type: binary_idmap
 
 ## Feature

core/src/db/DBImpl.cpp

@@ -812,7 +812,7 @@ DBImpl::CompactFile(const std::string& table_id, const meta::TableFileSchema& fi
     // Update table files state
     // if index type isn't IDMAP, set file type to TO_INDEX if file size exceed index_file_size
     // else set file type to RAW, no need to build index
-    if (compacted_file.engine_type_ != (int)EngineType::FAISS_IDMAP) {
+    if (!utils::IsRawIndexType(compacted_file.engine_type_)) {
         compacted_file.file_type_ = (segment_writer_ptr->Size() >= compacted_file.index_file_size_)
                                         ? meta::TableFileSchema::TO_INDEX
                                         : meta::TableFileSchema::RAW;
@@ -1468,7 +1468,7 @@ DBImpl::MergeFiles(const std::string& table_id, const meta::TableFilesSchema& fi
     // step 4: update table files state
     // if index type isn't IDMAP, set file type to TO_INDEX if file size exceed index_file_size
     // else set file type to RAW, no need to build index
-    if (table_file.engine_type_ != (int)EngineType::FAISS_IDMAP) {
+    if (!utils::IsRawIndexType(table_file.engine_type_)) {
         table_file.file_type_ = (segment_writer_ptr->Size() >= table_file.index_file_size_)
                                     ? meta::TableFileSchema::TO_INDEX
                                     : meta::TableFileSchema::RAW;
@@ -1770,7 +1770,7 @@ DBImpl::BuildTableIndexRecursively(const std::string& table_id, const TableIndex
     // for IDMAP type, only wait all NEW file converted to RAW file
     // for other type, wait NEW/RAW/NEW_MERGE/NEW_INDEX/TO_INDEX files converted to INDEX files
     std::vector<int> file_types;
-    if (index.engine_type_ == static_cast<int32_t>(EngineType::FAISS_IDMAP)) {
+    if (utils::IsRawIndexType(index.engine_type_)) {
         file_types = {
             static_cast<int32_t>(meta::TableFileSchema::NEW),
             static_cast<int32_t>(meta::TableFileSchema::NEW_MERGE),
@@ -1792,7 +1792,7 @@ DBImpl::BuildTableIndexRecursively(const std::string& table_id, const TableIndex
 
     while (!table_files.empty()) {
         ENGINE_LOG_DEBUG << "Non index files detected! Will build index " << times;
-        if (index.engine_type_ != (int)EngineType::FAISS_IDMAP) {
+        if (!utils::IsRawIndexType(index.engine_type_)) {
             status = meta_ptr_->UpdateTableFilesToIndex(table_id);
         }
 

core/src/db/Utils.cpp

@@ -215,6 +215,11 @@ IsSameIndex(const TableIndex& index1, const TableIndex& index2) {
            index1.metric_type_ == index2.metric_type_;
 }
 
+bool
+IsRawIndexType(int32_t type) {
+    return (type == (int32_t)EngineType::FAISS_IDMAP) || (type == (int32_t)EngineType::FAISS_BIN_IDMAP);
+}
+
 meta::DateT
 GetDate(const std::time_t& t, int day_delta) {
     struct tm ltm;

core/src/db/Utils.h

@@ -45,6 +45,9 @@ GetParentPath(const std::string& path, std::string& parent_path);
 bool
 IsSameIndex(const TableIndex& index1, const TableIndex& index2);
 
+bool
+IsRawIndexType(int32_t type);
+
 meta::DateT
 GetDate(const std::time_t& t, int day_delta = 0);
 meta::DateT

core/src/db/engine/ExecutionEngineImpl.cpp

@@ -406,7 +406,7 @@ ExecutionEngineImpl::Load(bool to_cache) {
         utils::GetParentPath(location_, segment_dir);
         auto segment_reader_ptr = std::make_shared<segment::SegmentReader>(segment_dir);
 
-        if (index_type_ == EngineType::FAISS_IDMAP || index_type_ == EngineType::FAISS_BIN_IDMAP) {
+        if (utils::IsRawIndexType((int32_t)index_type_)) {
             index_ = index_type_ == EngineType::FAISS_IDMAP ? GetVecIndexFactory(IndexType::FAISS_IDMAP)
                                                             : GetVecIndexFactory(IndexType::FAISS_BIN_IDMAP);
             milvus::json conf{{knowhere::meta::DEVICEID, gpu_num_}, {knowhere::meta::DIM, dim_}};

core/src/db/meta/MySQLMetaImpl.cpp

@@ -2086,8 +2086,7 @@ MySQLMetaImpl::CleanUpFilesWithTTL(uint64_t seconds /*, CleanUpFilter* filter*/)
                     // If we are deleting a raw table file, it means it's okay to delete the entire segment directory.
                     // Else, we can only delete the single file
                     // TODO(zhiru): We determine whether a table file is raw by its engine type. This is a bit hacky
-                    if (table_file.engine_type_ == (int32_t)EngineType::FAISS_IDMAP ||
-                        table_file.engine_type_ == (int32_t)EngineType::FAISS_BIN_IDMAP) {
+                    if (utils::IsRawIndexType(table_file.engine_type_)) {
                         utils::DeleteSegment(options_, table_file);
                         std::string segment_dir;
                         utils::GetParentPath(table_file.location_, segment_dir);

core/src/db/meta/SqliteMetaImpl.cpp

@@ -1425,8 +1425,7 @@ SqliteMetaImpl::CleanUpFilesWithTTL(uint64_t seconds /*, CleanUpFilter* filter*/
                     // If we are deleting a raw table file, it means it's okay to delete the entire segment directory.
                     // Else, we can only delete the single file
                     // TODO(zhiru): We determine whether a table file is raw by its engine type. This is a bit hacky
-                    if (table_file.engine_type_ == (int32_t)EngineType::FAISS_IDMAP ||
-                        table_file.engine_type_ == (int32_t)EngineType::FAISS_BIN_IDMAP) {
+                    if (utils::IsRawIndexType(table_file.engine_type_)) {
                         utils::DeleteSegment(options_, table_file);
                         std::string segment_dir;
                         utils::GetParentPath(table_file.location_, segment_dir);