diff --git a/CHANGELOG.md b/CHANGELOG.md index 6b3504adc91442e4cd389ef6f09f1c49eb393758..d7a2505c50d57111fbe524dfd697b74323c91074 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,6 +28,7 @@ Please mark all change in change log and use the issue from GitHub - \#1298 Unittest failed when on CPU2GPU case - \#1359 Negative distance value returned when searching with HNSW index type - \#1429 Server crashed when searching vectors using GPU +- \#1484 Index type changed to IDMAP after compacted ## Feature - \#216 Add CLI to get server info diff --git a/core/src/db/DBImpl.cpp b/core/src/db/DBImpl.cpp index eadf56d39c4f219af69655d369416e6ab28416e9..1bcd54ac429ce8f3e099e2bc32f4c80e82cdd8ee 100644 --- a/core/src/db/DBImpl.cpp +++ b/core/src/db/DBImpl.cpp @@ -658,12 +658,23 @@ DBImpl::Compact(const std::string& table_id) { const std::lock_guard lock(flush_merge_compact_mutex_); + // Save table index + TableIndex table_index; + status = DescribeIndex(table_id, table_index); + if (!status.ok()) { + return status; + } + // Drop all index status = DropIndex(table_id); if (!status.ok()) { - std::string err_msg = "Failed to drop index in compact: " + status.message(); - ENGINE_LOG_ERROR << err_msg; - return Status(DB_ERROR, err_msg); + return status; + } + + // Then update table index to the previous index + status = UpdateTableIndexRecursively(table_id, table_index); + if (!status.ok()) { + return status; } // Get files to compact from meta. @@ -679,23 +690,51 @@ DBImpl::Compact(const std::string& table_id) { ENGINE_LOG_DEBUG << "Found " << files_to_compact.size() << " segment to compact"; OngoingFileChecker::GetInstance().MarkOngoingFiles(files_to_compact); + + meta::TableFilesSchema files_to_update; + Status compact_status; for (auto& file : files_to_compact) { - status = CompactFile(table_id, file); + compact_status = CompactFile(table_id, file, files_to_update); - if (!status.ok()) { - OngoingFileChecker::GetInstance().UnmarkOngoingFiles(files_to_compact); - return status; + if (!compact_status.ok()) { + ENGINE_LOG_ERROR << "Compact failed for file " << file.file_id_ << ": " << compact_status.message(); + break; } } + + if (compact_status.ok()) { + ENGINE_LOG_DEBUG << "Finished compacting table: " << table_id; + } + + ENGINE_LOG_ERROR << "Updating meta after compaction..."; + + // Drop index again, in case some files were in the index building process during compacting + status = DropIndex(table_id); + if (!status.ok()) { + return status; + } + + // Update index + status = UpdateTableIndexRecursively(table_id, table_index); + if (!status.ok()) { + return status; + } + + status = meta_ptr_->UpdateTableFiles(files_to_update); + if (!status.ok()) { + return status; + } + OngoingFileChecker::GetInstance().UnmarkOngoingFiles(files_to_compact); - ENGINE_LOG_DEBUG << "Finished compacting table: " << table_id; + ENGINE_LOG_DEBUG << "Finished updating meta after compaction"; return status; } Status -DBImpl::CompactFile(const std::string& table_id, const milvus::engine::meta::TableFileSchema& file) { +DBImpl::CompactFile(const std::string& table_id, const meta::TableFileSchema& file, + meta::TableFilesSchema& files_to_update) { ENGINE_LOG_DEBUG << "Compacting segment " << file.segment_id_ << " for table: " << table_id; // Create new table file @@ -740,10 +779,6 @@ DBImpl::CompactFile(const std::string& table_id, const milvus::engine::meta::Tab return status; } - // Drop index again, in case some files were in the index building process during merging - // TODO: might be too frequent? - DropIndex(table_id); - // Update table files state // if index type isn't IDMAP, set file type to TO_INDEX if file size exceed index_file_size // else set file type to RAW, no need to build index @@ -763,7 +798,10 @@ DBImpl::CompactFile(const std::string& table_id, const milvus::engine::meta::Tab } updated.emplace_back(compacted_file); - status = meta_ptr_->UpdateTableFiles(updated); + + for (auto& f : updated) { + files_to_update.emplace_back(f); + } ENGINE_LOG_DEBUG << "Compacted segment " << compacted_file.segment_id_ << " from " << std::to_string(file_to_compact.file_size_) << " bytes to " diff --git a/core/src/db/DBImpl.h b/core/src/db/DBImpl.h index 4285a1cd0237dd577827973221d14ec2122f2f3e..759fe611090a0f4576b5ba63177f483c22b9c0cc 100644 --- a/core/src/db/DBImpl.h +++ b/core/src/db/DBImpl.h @@ -182,7 +182,8 @@ class DBImpl : public DB { BackgroundBuildIndex(); Status - CompactFile(const std::string& table_id, const milvus::engine::meta::TableFileSchema& file); + CompactFile(const std::string& table_id, const meta::TableFileSchema& file, + meta::TableFilesSchema& files_to_update); /* Status diff --git a/core/unittest/db/test_delete.cpp b/core/unittest/db/test_delete.cpp index 78d35d65394b57bff3a3bfa4c148b4ff90c6287f..09675f4b4a92e5f8a77509c0f9849834faf642a8 100644 --- a/core/unittest/db/test_delete.cpp +++ b/core/unittest/db/test_delete.cpp @@ -571,7 +571,11 @@ TEST_F(DeleteTest, compact_with_index) { stat = db_->Compact(GetTableName()); ASSERT_TRUE(stat.ok()); - // std::this_thread::sleep_for(std::chrono::seconds(5)); // wait for build index to finish + + milvus::engine::TableIndex table_index; + stat = db_->DescribeIndex(GetTableName(), table_index); + ASSERT_TRUE(stat.ok()); + ASSERT_FLOAT_EQ(table_index.engine_type_, index.engine_type_); int topk = 10, nprobe = 10; for (auto& pair : search_vectors) {