MS-427 Describe index error after drop index

Former-commit-id: 1c4f1319ec349f05784d038fa4cbd0c0b8279100

cpp/CHANGELOG.md

@@ -10,6 +10,7 @@ Please mark all change in change log and use the ticket from JIRA.
 - MS-416 - ExecutionEngineImpl::GpuCache has not return value cause crash
 - MS-417 - YAML sequence load disable cause scheduler startup failed
 - MS-413 - Create index failed and server exited
+- MS-427 - Describe index error after drop index
 
 ## Improvement
 - MS-327 - Clean code for milvus

cpp/src/db/DB.h

@@ -46,7 +46,6 @@ public:
 
     virtual Status Size(uint64_t& result) = 0;
 
-    virtual Status BuildIndex(const std::string& table_id) = 0;
     virtual Status CreateIndex(const std::string& table_id, const TableIndex& index) = 0;
     virtual Status DescribeIndex(const std::string& table_id, TableIndex& index) = 0;
     virtual Status DropIndex(const std::string& table_id) = 0;

cpp/src/db/DBImpl.cpp

@@ -424,7 +424,14 @@ Status DBImpl::MergeFiles(const std::string& table_id, const meta::DateT& date,
     }
 
     //step 4: update table files state
+    //if index type isn't IDMAP, set file type to TO_INDEX if file size execeed index_file_size
+    //else set file type to RAW, no need to build index
+    if (table_file.engine_type_ != (int)EngineType::FAISS_IDMAP) {
+        table_file.file_type_ = (index->PhysicalSize() >= table_file.index_file_size_) ?
+                                meta::TableFileSchema::TO_INDEX : meta::TableFileSchema::RAW;
+    } else {
         table_file.file_type_ = meta::TableFileSchema::RAW;
+    }
     table_file.file_size_ = index->PhysicalSize();
     table_file.row_count_ = index->Count();
     updated.push_back(table_file);
@@ -516,22 +523,6 @@ void DBImpl::StartBuildIndexTask(bool force) {
     }
 }
 
-Status DBImpl::BuildIndex(const std::string& table_id) {
-    bool has = false;
-    meta_ptr_->HasNonIndexFiles(table_id, has);
-    int times = 1;
-
-    while (has) {
-        ENGINE_LOG_DEBUG << "Non index files detected in " << table_id << "! Will build index " << times;
-        meta_ptr_->UpdateTableFilesToIndex(table_id);
-        /* StartBuildIndexTask(true); */
-        std::this_thread::sleep_for(std::chrono::milliseconds(std::min(10*1000, times*100)));
-        meta_ptr_->HasNonIndexFiles(table_id, has);
-        times++;
-    }
-    return Status::OK();
-}
-
 Status DBImpl::CreateIndex(const std::string& table_id, const TableIndex& index) {
     {
         std::unique_lock<std::mutex> lock(build_index_mutex_);
@@ -558,7 +549,6 @@ Status DBImpl::CreateIndex(const std::string& table_id, const TableIndex& index)
         }
 
         //step 3: update index info
-
         status = meta_ptr_->UpdateTableIndexParam(table_id, index);
         if (!status.ok()) {
             ENGINE_LOG_ERROR << "Failed to update table index info";

cpp/src/db/DBImpl.h

@@ -82,8 +82,6 @@ class DBImpl : public DB {
 
     Status Size(uint64_t &result) override;
 
-    Status BuildIndex(const std::string& table_id) override;
-
     Status CreateIndex(const std::string& table_id, const TableIndex& index) override;
 
     Status DescribeIndex(const std::string& table_id, TableIndex& index) override;

cpp/src/db/insert/MemTableFile.cpp

@@ -87,8 +87,14 @@ Status MemTableFile::Serialize() {
     table_file_schema_.file_size_ = execution_engine_->PhysicalSize();
     table_file_schema_.row_count_ = execution_engine_->Count();
 
+    //if index type isn't IDMAP, set file type to TO_INDEX if file size execeed index_file_size
+    //else set file type to RAW, no need to build index
+    if (table_file_schema_.engine_type_ != (int)EngineType::FAISS_IDMAP) {
         table_file_schema_.file_type_ = (size >= table_file_schema_.index_file_size_) ?
                                         meta::TableFileSchema::TO_INDEX : meta::TableFileSchema::RAW;
+    } else {
+        table_file_schema_.file_type_ = meta::TableFileSchema::RAW;
+    }
 
     auto status = meta_->UpdateTableFile(table_file_schema_);
 

cpp/src/db/meta/MetaTypes.h

@@ -19,8 +19,8 @@ namespace meta {
 
 constexpr int32_t DEFAULT_ENGINE_TYPE = (int)EngineType::FAISS_IDMAP;
 constexpr int32_t DEFAULT_NLIST = 16384;
-constexpr int32_t DEFAULT_INDEX_FILE_SIZE = ONE_GB;
 constexpr int32_t DEFAULT_METRIC_TYPE = (int)MetricType::L2;
+constexpr int32_t DEFAULT_INDEX_FILE_SIZE = ONE_GB;
 
 constexpr int64_t FLAG_MASK_USERID = 1;
 

cpp/src/db/meta/MySQLMetaImpl.cpp

@@ -536,6 +536,7 @@ Status MySQLMetaImpl::DropTableIndex(const std::string &table_id) {
 
             Query dropTableIndexQuery = connectionPtr->query();
 
+            //soft delete index files
             dropTableIndexQuery << "UPDATE TableFiles " <<
                                 "SET file_type = " << std::to_string(TableFileSchema::TO_DELETE) << "," <<
                                 "updated_time = " << utils::GetMicroSecTimeStamp() << " " <<
@@ -550,6 +551,7 @@ Status MySQLMetaImpl::DropTableIndex(const std::string &table_id) {
                                                   dropTableIndexQuery.error());
             }
 
+            //set all backup file to raw
             dropTableIndexQuery << "UPDATE TableFiles " <<
                                 "SET file_type = " << std::to_string(TableFileSchema::RAW) << "," <<
                                 "updated_time = " << utils::GetMicroSecTimeStamp() << " " <<
@@ -564,6 +566,21 @@ Status MySQLMetaImpl::DropTableIndex(const std::string &table_id) {
                                                   dropTableIndexQuery.error());
             }
 
+            //set table index type to raw
+            dropTableIndexQuery << "UPDATE Tables " <<
+                                "SET engine_type = " << std::to_string(DEFAULT_ENGINE_TYPE) << "," <<
+                                "nlist = " << std::to_string(DEFAULT_NLIST) << " " <<
+                                "metric_type = " << std::to_string(DEFAULT_METRIC_TYPE) << " " <<
+                                "WHERE table_id = " << quote << table_id << ";";
+
+            ENGINE_LOG_DEBUG << "MySQLMetaImpl::DropTableIndex: " << dropTableIndexQuery.str();
+
+            if (!dropTableIndexQuery.exec()) {
+                ENGINE_LOG_ERROR << "QUERY ERROR WHEN DROP TABLE INDEX";
+                return Status::DBTransactionError("QUERY ERROR WHEN DROP TABLE INDEX",
+                                                  dropTableIndexQuery.error());
+            }
+
         } //Scoped Connection
 
     } catch (const BadQuery &er) {

cpp/src/db/meta/SqliteMetaImpl.cpp

@@ -461,6 +461,17 @@ Status SqliteMetaImpl::DropTableIndex(const std::string &table_id) {
                         c(&TableFileSchema::file_type_) == (int) TableFileSchema::BACKUP
                 ));
 
+        //set table index type to raw
+        ConnectorPtr->update_all(
+                set(
+                        c(&TableSchema::engine_type_) = DEFAULT_ENGINE_TYPE,
+                        c(&TableSchema::nlist_) = DEFAULT_NLIST,
+                        c(&TableSchema::metric_type_) = DEFAULT_METRIC_TYPE
+                ),
+                where(
+                        c(&TableSchema::table_id_) == table_id
+                ));
+
     } catch (std::exception &e) {
         return HandleException("Encounter exception when delete table index files", e);
     }
@@ -798,11 +809,15 @@ Status SqliteMetaImpl::FilesToSearch(const std::string &table_id,
             }
             files[table_file.date_].push_back(table_file);
         }
-
+        if(files.empty()) {
+            std::cout << "ERROR" << std::endl;
+        }
     } catch (std::exception &e) {
         return HandleException("Encounter exception when iterate index files", e);
     }
 
+
+
     return Status::OK();
 }
 

cpp/unittest/db/db_tests.cpp

@@ -34,7 +34,6 @@ namespace {
         engine::meta::TableSchema table_info;
         table_info.dimension_ = TABLE_DIM;
         table_info.table_id_ = TABLE_NAME;
-        table_info.engine_type_ = (int)engine::EngineType::FAISS_IDMAP;
         return table_info;
     }
 
@@ -263,7 +262,9 @@ TEST_F(DBTest, SEARCH_TEST) {
         ASSERT_STATS(stat);
     }
 
-    db_->BuildIndex(TABLE_NAME); // wait until build index finish
+    engine::TableIndex index;
+    index.engine_type_ = (int)engine::EngineType::FAISS_IDMAP;
+    db_->CreateIndex(TABLE_NAME, index); // wait until build index finish
 
     {
         engine::QueryResults results;
@@ -273,7 +274,7 @@ TEST_F(DBTest, SEARCH_TEST) {
 
     {//search by specify index file
         engine::meta::DatesT dates;
-        std::vector<std::string> file_ids = {"4", "5", "6"};
+        std::vector<std::string> file_ids = {"1", "2", "3", "4", "5", "6"};
         engine::QueryResults results;
         stat = db_->Query(TABLE_NAME, file_ids, k, nq, 10, xq.data(), dates, results);
         ASSERT_STATS(stat);
@@ -305,7 +306,12 @@ TEST_F(DBTest, PRELOADTABLE_TEST) {
         db_->InsertVectors(TABLE_NAME, nb, xb.data(), target_ids);
         ASSERT_EQ(target_ids.size(), nb);
     }
-    db_->BuildIndex(TABLE_NAME);
+
+    sleep(2);
+
+    engine::TableIndex index;
+    index.engine_type_ = (int)engine::EngineType::FAISS_IDMAP;
+    db_->CreateIndex(TABLE_NAME, index); // wait until build index finish
 
     int64_t prev_cache_usage = cache::CpuCacheMgr::GetInstance()->CacheUsage();
     stat = db_->PreloadTable(TABLE_NAME);