未验证 提交 01b6ad36 编写于 作者: Z Zhiru Zhu 提交者: GitHub

Fix#1484 (#1490)

* update
Signed-off-by: NZhiru Zhu <zzhu@fandm.edu>

* #1426 Support to configure whether to enabled autoflush (#1468)
Signed-off-by: Nshengjun.li <shengjun.li@zilliz.com>
Signed-off-by: NZhiru Zhu <zzhu@fandm.edu>

* fix cases: test_search_id/test_wal (#1486)
Signed-off-by: Ndel-zhenwu <zw@zilliz.com>
Signed-off-by: NZhiru Zhu <zzhu@fandm.edu>

* #1480 add return code for AVX512 selection (#1482)
Signed-off-by: Nyudong.cai <yudong.cai@zilliz.com>
Signed-off-by: NZhiru Zhu <zzhu@fandm.edu>

* update
Signed-off-by: NZhiru Zhu <zzhu@fandm.edu>

* update
Signed-off-by: NZhiru Zhu <zzhu@fandm.edu>
Co-authored-by: Nshengjun.li <49774184+shengjun1985@users.noreply.github.com>
Co-authored-by: Ndel-zhenwu <56623710+del-zhenwu@users.noreply.github.com>
Co-authored-by: NCai Yudong <yudong.cai@zilliz.com>
上级 d460f48c
......@@ -28,6 +28,7 @@ Please mark all change in change log and use the issue from GitHub
- \#1298 Unittest failed when on CPU2GPU case
- \#1359 Negative distance value returned when searching with HNSW index type
- \#1429 Server crashed when searching vectors using GPU
- \#1484 Index type changed to IDMAP after compacted
## Feature
- \#216 Add CLI to get server info
......
......@@ -658,12 +658,23 @@ DBImpl::Compact(const std::string& table_id) {
const std::lock_guard<std::mutex> lock(flush_merge_compact_mutex_);
// Save table index
TableIndex table_index;
status = DescribeIndex(table_id, table_index);
if (!status.ok()) {
return status;
}
// Drop all index
status = DropIndex(table_id);
if (!status.ok()) {
std::string err_msg = "Failed to drop index in compact: " + status.message();
ENGINE_LOG_ERROR << err_msg;
return Status(DB_ERROR, err_msg);
return status;
}
// Then update table index to the previous index
status = UpdateTableIndexRecursively(table_id, table_index);
if (!status.ok()) {
return status;
}
// Get files to compact from meta.
......@@ -679,23 +690,51 @@ DBImpl::Compact(const std::string& table_id) {
ENGINE_LOG_DEBUG << "Found " << files_to_compact.size() << " segment to compact";
OngoingFileChecker::GetInstance().MarkOngoingFiles(files_to_compact);
meta::TableFilesSchema files_to_update;
Status compact_status;
for (auto& file : files_to_compact) {
status = CompactFile(table_id, file);
compact_status = CompactFile(table_id, file, files_to_update);
if (!compact_status.ok()) {
ENGINE_LOG_ERROR << "Compact failed for file " << file.file_id_ << ": " << compact_status.message();
break;
}
}
if (compact_status.ok()) {
ENGINE_LOG_DEBUG << "Finished compacting table: " << table_id;
}
ENGINE_LOG_ERROR << "Updating meta after compaction...";
// Drop index again, in case some files were in the index building process during compacting
status = DropIndex(table_id);
if (!status.ok()) {
OngoingFileChecker::GetInstance().UnmarkOngoingFiles(files_to_compact);
return status;
}
// Update index
status = UpdateTableIndexRecursively(table_id, table_index);
if (!status.ok()) {
return status;
}
status = meta_ptr_->UpdateTableFiles(files_to_update);
if (!status.ok()) {
return status;
}
OngoingFileChecker::GetInstance().UnmarkOngoingFiles(files_to_compact);
ENGINE_LOG_DEBUG << "Finished compacting table: " << table_id;
ENGINE_LOG_DEBUG << "Finished updating meta after compaction";
return status;
}
Status
DBImpl::CompactFile(const std::string& table_id, const milvus::engine::meta::TableFileSchema& file) {
DBImpl::CompactFile(const std::string& table_id, const meta::TableFileSchema& file,
meta::TableFilesSchema& files_to_update) {
ENGINE_LOG_DEBUG << "Compacting segment " << file.segment_id_ << " for table: " << table_id;
// Create new table file
......@@ -740,10 +779,6 @@ DBImpl::CompactFile(const std::string& table_id, const milvus::engine::meta::Tab
return status;
}
// Drop index again, in case some files were in the index building process during merging
// TODO: might be too frequent?
DropIndex(table_id);
// Update table files state
// if index type isn't IDMAP, set file type to TO_INDEX if file size exceed index_file_size
// else set file type to RAW, no need to build index
......@@ -763,7 +798,10 @@ DBImpl::CompactFile(const std::string& table_id, const milvus::engine::meta::Tab
}
updated.emplace_back(compacted_file);
status = meta_ptr_->UpdateTableFiles(updated);
for (auto& f : updated) {
files_to_update.emplace_back(f);
}
ENGINE_LOG_DEBUG << "Compacted segment " << compacted_file.segment_id_ << " from "
<< std::to_string(file_to_compact.file_size_) << " bytes to "
......
......@@ -182,7 +182,8 @@ class DBImpl : public DB {
BackgroundBuildIndex();
Status
CompactFile(const std::string& table_id, const milvus::engine::meta::TableFileSchema& file);
CompactFile(const std::string& table_id, const meta::TableFileSchema& file,
meta::TableFilesSchema& files_to_update);
/*
Status
......
......@@ -571,7 +571,11 @@ TEST_F(DeleteTest, compact_with_index) {
stat = db_->Compact(GetTableName());
ASSERT_TRUE(stat.ok());
// std::this_thread::sleep_for(std::chrono::seconds(5)); // wait for build index to finish
milvus::engine::TableIndex table_index;
stat = db_->DescribeIndex(GetTableName(), table_index);
ASSERT_TRUE(stat.ok());
ASSERT_FLOAT_EQ(table_index.engine_type_, index.engine_type_);
int topk = 10, nprobe = 10;
for (auto& pair : search_vectors) {
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册