Merge branch 'yuncong' into 'branch-0.3.1-yuncong'

feat(db): impl build index

See merge request megasearch/milvus!181

Former-commit-id: 6ccdee807e0b8577f88a24005b54a9f55c200f4e

cpp/src/db/DBImpl.cpp

@@ -408,10 +408,10 @@ void DBImpl::BackgroundCompaction(std::set<std::string> table_ids) {
     meta_ptr_->CleanUpFilesWithTTL(ttl);
 }
 
-void DBImpl::StartBuildIndexTask() {
+void DBImpl::StartBuildIndexTask(bool force) {
     static uint64_t index_clock_tick = 0;
     index_clock_tick++;
-    if(index_clock_tick%INDEX_ACTION_INTERVAL != 0) {
+    if(!force && (index_clock_tick%INDEX_ACTION_INTERVAL != 0)) {
         return;
     }
 
@@ -431,8 +431,20 @@ void DBImpl::StartBuildIndexTask() {
 }
 
 Status DBImpl::BuildIndex(const std::string& table_id) {
+    bool has = false;
+    meta_ptr_->HasNonIndexFiles(table_id, has);
+    int times = 1;
+
+    while (has) {
+        ENGINE_LOG_DEBUG << "Non index files detected! Will build index " << times;
         meta_ptr_->UpdateTableFilesToIndex(table_id);
-    return BuildIndexByTable(table_id);
+        StartBuildIndexTask(true);
+        std::this_thread::sleep_for(std::chrono::milliseconds(std::min(10*1000, times*100)));
+        meta_ptr_->HasNonIndexFiles(table_id, has);
+        times++;
+    }
+    return Status::OK();
+    /* return BuildIndexByTable(table_id); */
 }
 
 Status DBImpl::BuildIndex(const meta::TableFileSchema& file) {

cpp/src/db/DBImpl.h

@@ -109,7 +109,7 @@ class DBImpl : public DB {
     Status BackgroundMergeFiles(const std::string &table_id);
     void BackgroundCompaction(std::set<std::string> table_ids);
 
-    void StartBuildIndexTask();
+    void StartBuildIndexTask(bool force=false);
     void BackgroundBuildIndex();
 
     Status

cpp/src/db/DBMetaImpl.cpp

@@ -317,6 +317,28 @@ Status DBMetaImpl::DescribeTable(TableSchema &table_schema) {
     return Status::OK();
 }
 
+Status DBMetaImpl::HasNonIndexFiles(const std::string& table_id, bool& has) {
+    has = false;
+    try {
+        auto selected = ConnectorPtr->select(columns(&TableFileSchema::id_),
+                                             where((c(&TableFileSchema::file_type_) == (int) TableFileSchema::RAW
+                                                    or
+                                                    c(&TableFileSchema::file_type_) == (int) TableFileSchema::TO_INDEX)
+                                                   and c(&TableFileSchema::table_id_) == table_id
+                                             ));
+
+        if (selected.size() >= 1) {
+            has = true;
+        } else {
+            has = false;
+        }
+
+    } catch (std::exception &e) {
+        return HandleException("Encounter exception when check non index files", e);
+    }
+    return Status::OK();
+}
+
 Status DBMetaImpl::HasTable(const std::string &table_id, bool &has_or_not) {
     has_or_not = false;
 

cpp/src/db/DBMetaImpl.h

@@ -35,6 +35,8 @@ public:
                                  const std::vector<size_t>& ids,
                                  TableFilesSchema& table_files) override;
 
+    virtual Status HasNonIndexFiles(const std::string& table_id, bool& has) override;
+
     virtual Status UpdateTableFilesToIndex(const std::string& table_id) override;
 
     virtual Status UpdateTableFile(TableFileSchema& file_schema) override;

cpp/src/db/Meta.h

@@ -58,6 +58,8 @@ public:
 
     virtual Status FilesToIndex(TableFilesSchema&) = 0;
 
+    virtual Status HasNonIndexFiles(const std::string& table_id, bool& has) = 0;
+
     virtual Status CleanUp() = 0;
     virtual Status CleanUpFilesWithTTL(uint16_t) = 0;
 

cpp/src/db/MySQLMetaImpl.cpp

@@ -436,6 +436,11 @@ namespace meta {
         return Status::OK();
     }
 
+    Status MySQLMetaImpl::HasNonIndexFiles(const std::string& table_id, bool& has) {
+        // TODO
+        return Status::OK();
+    }
+
     Status MySQLMetaImpl::DeleteTable(const std::string& table_id) {
 
 //        std::lock_guard<std::recursive_mutex> lock(mysql_mutex);

cpp/src/db/MySQLMetaImpl.h

@@ -40,6 +40,8 @@ namespace meta {
                                      const std::vector<size_t>& ids,
                                      TableFilesSchema& table_files) override;
 
+        virtual Status HasNonIndexFiles(const std::string& table_id, bool& has) override;
+
         virtual Status UpdateTableFile(TableFileSchema& file_schema) override;
 
         virtual Status UpdateTableFilesToIndex(const std::string& table_id) override;