From c6d38f0af2d326f4bf6898f1ccec58dc2af8fad6 Mon Sep 17 00:00:00 2001
From: "peng.xu" <peng.xu@zilliz.com>
Date: Tue, 9 Jul 2019 22:08:21 +0800
Subject: [PATCH] feat(db): impl build index

Former-commit-id: 3fd41e5fa861957e108a4d077a152dc0c30f2691
---
 cpp/src/db/DBImpl.cpp        | 20 ++++++++++++++++----
 cpp/src/db/DBImpl.h          |  2 +-
 cpp/src/db/DBMetaImpl.cpp    | 22 ++++++++++++++++++++++
 cpp/src/db/DBMetaImpl.h      |  2 ++
 cpp/src/db/Meta.h            |  2 ++
 cpp/src/db/MySQLMetaImpl.cpp |  5 +++++
 cpp/src/db/MySQLMetaImpl.h   |  2 ++
 7 files changed, 50 insertions(+), 5 deletions(-)

diff --git a/cpp/src/db/DBImpl.cpp b/cpp/src/db/DBImpl.cpp
index 94a2e76c..6f6cfcb8 100644
--- a/cpp/src/db/DBImpl.cpp
+++ b/cpp/src/db/DBImpl.cpp
@@ -408,10 +408,10 @@ void DBImpl::BackgroundCompaction(std::set<std::string> table_ids) {
     meta_ptr_->CleanUpFilesWithTTL(ttl);
 }
 
-void DBImpl::StartBuildIndexTask() {
+void DBImpl::StartBuildIndexTask(bool force) {
     static uint64_t index_clock_tick = 0;
     index_clock_tick++;
-    if(index_clock_tick%INDEX_ACTION_INTERVAL != 0) {
+    if(!force && (index_clock_tick%INDEX_ACTION_INTERVAL != 0)) {
         return;
     }
 
@@ -431,8 +431,20 @@ void DBImpl::StartBuildIndexTask() {
 }
 
 Status DBImpl::BuildIndex(const std::string& table_id) {
-    meta_ptr_->UpdateTableFilesToIndex(table_id);
-    return BuildIndexByTable(table_id);
+    bool has = false;
+    meta_ptr_->HasNonIndexFiles(table_id, has);
+    int times = 1;
+
+    while (has) {
+        ENGINE_LOG_DEBUG << "Non index files detected! Will build index " << times;
+        meta_ptr_->UpdateTableFilesToIndex(table_id);
+        StartBuildIndexTask(true);
+        std::this_thread::sleep_for(std::chrono::milliseconds(std::min(10*1000, times*100)));
+        meta_ptr_->HasNonIndexFiles(table_id, has);
+        times++;
+    }
+    return Status::OK();
+    /* return BuildIndexByTable(table_id); */
 }
 
 Status DBImpl::BuildIndex(const meta::TableFileSchema& file) {
diff --git a/cpp/src/db/DBImpl.h b/cpp/src/db/DBImpl.h
index e5042354..064d6d00 100644
--- a/cpp/src/db/DBImpl.h
+++ b/cpp/src/db/DBImpl.h
@@ -109,7 +109,7 @@ class DBImpl : public DB {
     Status BackgroundMergeFiles(const std::string &table_id);
     void BackgroundCompaction(std::set<std::string> table_ids);
 
-    void StartBuildIndexTask();
+    void StartBuildIndexTask(bool force=false);
     void BackgroundBuildIndex();
 
     Status
diff --git a/cpp/src/db/DBMetaImpl.cpp b/cpp/src/db/DBMetaImpl.cpp
index 0e2b52db..551073ff 100644
--- a/cpp/src/db/DBMetaImpl.cpp
+++ b/cpp/src/db/DBMetaImpl.cpp
@@ -316,6 +316,28 @@ Status DBMetaImpl::DescribeTable(TableSchema &table_schema) {
     return Status::OK();
 }
 
+Status DBMetaImpl::HasNonIndexFiles(const std::string& table_id, bool& has) {
+    has = false;
+    try {
+        auto selected = ConnectorPtr->select(columns(&TableFileSchema::id_),
+                                             where((c(&TableFileSchema::file_type_) == (int) TableFileSchema::RAW
+                                                    or
+                                                    c(&TableFileSchema::file_type_) == (int) TableFileSchema::TO_INDEX)
+                                                   and c(&TableFileSchema::table_id_) == table_id
+                                             ));
+
+        if (selected.size() >= 1) {
+            has = true;
+        } else {
+            has = false;
+        }
+
+    } catch (std::exception &e) {
+        return HandleException("Encounter exception when check non index files", e);
+    }
+    return Status::OK();
+}
+
 Status DBMetaImpl::HasTable(const std::string &table_id, bool &has_or_not) {
     has_or_not = false;
 
diff --git a/cpp/src/db/DBMetaImpl.h b/cpp/src/db/DBMetaImpl.h
index 597800c0..f0b17c41 100644
--- a/cpp/src/db/DBMetaImpl.h
+++ b/cpp/src/db/DBMetaImpl.h
@@ -35,6 +35,8 @@ public:
                                  const std::vector<size_t>& ids,
                                  TableFilesSchema& table_files) override;
 
+    virtual Status HasNonIndexFiles(const std::string& table_id, bool& has) override;
+
     virtual Status UpdateTableFilesToIndex(const std::string& table_id) override;
 
     virtual Status UpdateTableFile(TableFileSchema& file_schema) override;
diff --git a/cpp/src/db/Meta.h b/cpp/src/db/Meta.h
index beab6e22..1d83817f 100644
--- a/cpp/src/db/Meta.h
+++ b/cpp/src/db/Meta.h
@@ -58,6 +58,8 @@ public:
 
     virtual Status FilesToIndex(TableFilesSchema&) = 0;
 
+    virtual Status HasNonIndexFiles(const std::string& table_id, bool& has) = 0;
+
     virtual Status CleanUp() = 0;
     virtual Status CleanUpFilesWithTTL(uint16_t) = 0;
 
diff --git a/cpp/src/db/MySQLMetaImpl.cpp b/cpp/src/db/MySQLMetaImpl.cpp
index 4d4b858d..14ce38c7 100644
--- a/cpp/src/db/MySQLMetaImpl.cpp
+++ b/cpp/src/db/MySQLMetaImpl.cpp
@@ -436,6 +436,11 @@ namespace meta {
         return Status::OK();
     }
 
+    Status MySQLMetaImpl::HasNonIndexFiles(const std::string& table_id, bool& has) {
+        // TODO
+        return Status::OK();
+    }
+
     Status MySQLMetaImpl::DeleteTable(const std::string& table_id) {
 
 //        std::lock_guard<std::recursive_mutex> lock(mysql_mutex);
diff --git a/cpp/src/db/MySQLMetaImpl.h b/cpp/src/db/MySQLMetaImpl.h
index f8591c25..9daa9f83 100644
--- a/cpp/src/db/MySQLMetaImpl.h
+++ b/cpp/src/db/MySQLMetaImpl.h
@@ -40,6 +40,8 @@ namespace meta {
                                      const std::vector<size_t>& ids,
                                      TableFilesSchema& table_files) override;
 
+        virtual Status HasNonIndexFiles(const std::string& table_id, bool& has) override;
+
         virtual Status UpdateTableFile(TableFileSchema& file_schema) override;
 
         virtual Status UpdateTableFilesToIndex(const std::string& table_id) override;
-- 
GitLab