diff --git a/cpp/CHANGELOG.md b/cpp/CHANGELOG.md index 99d55ea524f4a29e1df9702a22c805c254cd579f..5e79f04137b17af03a92a188e7189945b49896a5 100644 --- a/cpp/CHANGELOG.md +++ b/cpp/CHANGELOG.md @@ -10,6 +10,7 @@ Please mark all change in change log and use the ticket from JIRA. - MS-416 - ExecutionEngineImpl::GpuCache has not return value cause crash - MS-417 - YAML sequence load disable cause scheduler startup failed - MS-413 - Create index failed and server exited +- MS-427 - Describe index error after drop index ## Improvement - MS-327 - Clean code for milvus diff --git a/cpp/src/db/DB.h b/cpp/src/db/DB.h index f42154174166e4741c32bc2c0e57bf20e23a3f5c..908714a08a61a8a7e50a379b49bbf8f413f09b12 100644 --- a/cpp/src/db/DB.h +++ b/cpp/src/db/DB.h @@ -46,7 +46,6 @@ public: virtual Status Size(uint64_t& result) = 0; - virtual Status BuildIndex(const std::string& table_id) = 0; virtual Status CreateIndex(const std::string& table_id, const TableIndex& index) = 0; virtual Status DescribeIndex(const std::string& table_id, TableIndex& index) = 0; virtual Status DropIndex(const std::string& table_id) = 0; diff --git a/cpp/src/db/DBImpl.cpp b/cpp/src/db/DBImpl.cpp index a9386ff1eb8e865ccfd3579be2c6e930621c3fb7..f6e340f9be15c9833274764528941924f8b4cd7d 100644 --- a/cpp/src/db/DBImpl.cpp +++ b/cpp/src/db/DBImpl.cpp @@ -424,7 +424,14 @@ Status DBImpl::MergeFiles(const std::string& table_id, const meta::DateT& date, } //step 4: update table files state - table_file.file_type_ = meta::TableFileSchema::RAW; + //if index type isn't IDMAP, set file type to TO_INDEX if file size execeed index_file_size + //else set file type to RAW, no need to build index + if (table_file.engine_type_ != (int)EngineType::FAISS_IDMAP) { + table_file.file_type_ = (index->PhysicalSize() >= table_file.index_file_size_) ? + meta::TableFileSchema::TO_INDEX : meta::TableFileSchema::RAW; + } else { + table_file.file_type_ = meta::TableFileSchema::RAW; + } table_file.file_size_ = index->PhysicalSize(); table_file.row_count_ = index->Count(); updated.push_back(table_file); @@ -516,22 +523,6 @@ void DBImpl::StartBuildIndexTask(bool force) { } } -Status DBImpl::BuildIndex(const std::string& table_id) { - bool has = false; - meta_ptr_->HasNonIndexFiles(table_id, has); - int times = 1; - - while (has) { - ENGINE_LOG_DEBUG << "Non index files detected in " << table_id << "! Will build index " << times; - meta_ptr_->UpdateTableFilesToIndex(table_id); - /* StartBuildIndexTask(true); */ - std::this_thread::sleep_for(std::chrono::milliseconds(std::min(10*1000, times*100))); - meta_ptr_->HasNonIndexFiles(table_id, has); - times++; - } - return Status::OK(); -} - Status DBImpl::CreateIndex(const std::string& table_id, const TableIndex& index) { { std::unique_lock lock(build_index_mutex_); @@ -558,7 +549,6 @@ Status DBImpl::CreateIndex(const std::string& table_id, const TableIndex& index) } //step 3: update index info - status = meta_ptr_->UpdateTableIndexParam(table_id, index); if (!status.ok()) { ENGINE_LOG_ERROR << "Failed to update table index info"; diff --git a/cpp/src/db/DBImpl.h b/cpp/src/db/DBImpl.h index 65ac489c49ac3e6f9e9e658575b2e8692f04e16d..a6e52bd3c3bce3aa0cb8398662f0ac65e78e2830 100644 --- a/cpp/src/db/DBImpl.h +++ b/cpp/src/db/DBImpl.h @@ -82,8 +82,6 @@ class DBImpl : public DB { Status Size(uint64_t &result) override; - Status BuildIndex(const std::string& table_id) override; - Status CreateIndex(const std::string& table_id, const TableIndex& index) override; Status DescribeIndex(const std::string& table_id, TableIndex& index) override; diff --git a/cpp/src/db/insert/MemTableFile.cpp b/cpp/src/db/insert/MemTableFile.cpp index 48e52d0d9b333639bff0e61d2109849cf174e77d..2054a5047e30cba72166583f860b0d1430736d5e 100644 --- a/cpp/src/db/insert/MemTableFile.cpp +++ b/cpp/src/db/insert/MemTableFile.cpp @@ -87,8 +87,14 @@ Status MemTableFile::Serialize() { table_file_schema_.file_size_ = execution_engine_->PhysicalSize(); table_file_schema_.row_count_ = execution_engine_->Count(); - table_file_schema_.file_type_ = (size >= table_file_schema_.index_file_size_) ? - meta::TableFileSchema::TO_INDEX : meta::TableFileSchema::RAW; + //if index type isn't IDMAP, set file type to TO_INDEX if file size execeed index_file_size + //else set file type to RAW, no need to build index + if (table_file_schema_.engine_type_ != (int)EngineType::FAISS_IDMAP) { + table_file_schema_.file_type_ = (size >= table_file_schema_.index_file_size_) ? + meta::TableFileSchema::TO_INDEX : meta::TableFileSchema::RAW; + } else { + table_file_schema_.file_type_ = meta::TableFileSchema::RAW; + } auto status = meta_->UpdateTableFile(table_file_schema_); diff --git a/cpp/src/db/meta/MetaTypes.h b/cpp/src/db/meta/MetaTypes.h index 0727528fe935f5401c2acce79a71ad9736869e8a..a75bcc07dc7652ce244e55e0e4c99dbebb9e0981 100644 --- a/cpp/src/db/meta/MetaTypes.h +++ b/cpp/src/db/meta/MetaTypes.h @@ -19,8 +19,8 @@ namespace meta { constexpr int32_t DEFAULT_ENGINE_TYPE = (int)EngineType::FAISS_IDMAP; constexpr int32_t DEFAULT_NLIST = 16384; -constexpr int32_t DEFAULT_INDEX_FILE_SIZE = ONE_GB; constexpr int32_t DEFAULT_METRIC_TYPE = (int)MetricType::L2; +constexpr int32_t DEFAULT_INDEX_FILE_SIZE = ONE_GB; constexpr int64_t FLAG_MASK_USERID = 1; diff --git a/cpp/src/db/meta/MySQLMetaImpl.cpp b/cpp/src/db/meta/MySQLMetaImpl.cpp index 92af8e2a677d9697c7dd9a03302e0908c25d17e4..cb1ec199cf08a8914bce88f46279179d5f2c75f7 100644 --- a/cpp/src/db/meta/MySQLMetaImpl.cpp +++ b/cpp/src/db/meta/MySQLMetaImpl.cpp @@ -536,6 +536,7 @@ Status MySQLMetaImpl::DropTableIndex(const std::string &table_id) { Query dropTableIndexQuery = connectionPtr->query(); + //soft delete index files dropTableIndexQuery << "UPDATE TableFiles " << "SET file_type = " << std::to_string(TableFileSchema::TO_DELETE) << "," << "updated_time = " << utils::GetMicroSecTimeStamp() << " " << @@ -550,6 +551,7 @@ Status MySQLMetaImpl::DropTableIndex(const std::string &table_id) { dropTableIndexQuery.error()); } + //set all backup file to raw dropTableIndexQuery << "UPDATE TableFiles " << "SET file_type = " << std::to_string(TableFileSchema::RAW) << "," << "updated_time = " << utils::GetMicroSecTimeStamp() << " " << @@ -564,6 +566,21 @@ Status MySQLMetaImpl::DropTableIndex(const std::string &table_id) { dropTableIndexQuery.error()); } + //set table index type to raw + dropTableIndexQuery << "UPDATE Tables " << + "SET engine_type = " << std::to_string(DEFAULT_ENGINE_TYPE) << "," << + "nlist = " << std::to_string(DEFAULT_NLIST) << " " << + "metric_type = " << std::to_string(DEFAULT_METRIC_TYPE) << " " << + "WHERE table_id = " << quote << table_id << ";"; + + ENGINE_LOG_DEBUG << "MySQLMetaImpl::DropTableIndex: " << dropTableIndexQuery.str(); + + if (!dropTableIndexQuery.exec()) { + ENGINE_LOG_ERROR << "QUERY ERROR WHEN DROP TABLE INDEX"; + return Status::DBTransactionError("QUERY ERROR WHEN DROP TABLE INDEX", + dropTableIndexQuery.error()); + } + } //Scoped Connection } catch (const BadQuery &er) { diff --git a/cpp/src/db/meta/SqliteMetaImpl.cpp b/cpp/src/db/meta/SqliteMetaImpl.cpp index a06c391ca5f2c68003313d7fb63413a540e8553c..3f427130ae80e4c0c0140c4833597581d7668a5a 100644 --- a/cpp/src/db/meta/SqliteMetaImpl.cpp +++ b/cpp/src/db/meta/SqliteMetaImpl.cpp @@ -461,6 +461,17 @@ Status SqliteMetaImpl::DropTableIndex(const std::string &table_id) { c(&TableFileSchema::file_type_) == (int) TableFileSchema::BACKUP )); + //set table index type to raw + ConnectorPtr->update_all( + set( + c(&TableSchema::engine_type_) = DEFAULT_ENGINE_TYPE, + c(&TableSchema::nlist_) = DEFAULT_NLIST, + c(&TableSchema::metric_type_) = DEFAULT_METRIC_TYPE + ), + where( + c(&TableSchema::table_id_) == table_id + )); + } catch (std::exception &e) { return HandleException("Encounter exception when delete table index files", e); } @@ -798,11 +809,15 @@ Status SqliteMetaImpl::FilesToSearch(const std::string &table_id, } files[table_file.date_].push_back(table_file); } - + if(files.empty()) { + std::cout << "ERROR" << std::endl; + } } catch (std::exception &e) { return HandleException("Encounter exception when iterate index files", e); } + + return Status::OK(); } diff --git a/cpp/unittest/db/db_tests.cpp b/cpp/unittest/db/db_tests.cpp index 2426846c15c3ffe9321e56915d91ea232372df30..e5e16bf6fd417b2fc8f8d1cf9742efd1e3fd4396 100644 --- a/cpp/unittest/db/db_tests.cpp +++ b/cpp/unittest/db/db_tests.cpp @@ -34,7 +34,6 @@ namespace { engine::meta::TableSchema table_info; table_info.dimension_ = TABLE_DIM; table_info.table_id_ = TABLE_NAME; - table_info.engine_type_ = (int)engine::EngineType::FAISS_IDMAP; return table_info; } @@ -263,7 +262,9 @@ TEST_F(DBTest, SEARCH_TEST) { ASSERT_STATS(stat); } - db_->BuildIndex(TABLE_NAME); // wait until build index finish + engine::TableIndex index; + index.engine_type_ = (int)engine::EngineType::FAISS_IDMAP; + db_->CreateIndex(TABLE_NAME, index); // wait until build index finish { engine::QueryResults results; @@ -273,7 +274,7 @@ TEST_F(DBTest, SEARCH_TEST) { {//search by specify index file engine::meta::DatesT dates; - std::vector file_ids = {"4", "5", "6"}; + std::vector file_ids = {"1", "2", "3", "4", "5", "6"}; engine::QueryResults results; stat = db_->Query(TABLE_NAME, file_ids, k, nq, 10, xq.data(), dates, results); ASSERT_STATS(stat); @@ -305,7 +306,12 @@ TEST_F(DBTest, PRELOADTABLE_TEST) { db_->InsertVectors(TABLE_NAME, nb, xb.data(), target_ids); ASSERT_EQ(target_ids.size(), nb); } - db_->BuildIndex(TABLE_NAME); + + sleep(2); + + engine::TableIndex index; + index.engine_type_ = (int)engine::EngineType::FAISS_IDMAP; + db_->CreateIndex(TABLE_NAME, index); // wait until build index finish int64_t prev_cache_usage = cache::CpuCacheMgr::GetInstance()->CacheUsage(); stat = db_->PreloadTable(TABLE_NAME);