From 303d600ef3fb49806311b7a89218e264359f14db Mon Sep 17 00:00:00 2001
From: "peng.xu" <peng.xu@zilliz.com>
Date: Tue, 9 Jul 2019 18:22:43 +0800
Subject: [PATCH] feat(db): add build index api in db

Former-commit-id: dbd6da51e3c5fd666d7062798c0483c0f7fa7ffb
---
 cpp/src/db/DB.h           |  2 ++
 cpp/src/db/DBImpl.cpp     | 25 +++++++++++++++++++++++++
 cpp/src/db/DBImpl.h       |  6 ++++++
 cpp/src/db/DBMetaImpl.cpp | 17 +++++++++++++++++
 cpp/src/db/DBMetaImpl.h   |  2 ++
 cpp/src/db/Meta.h         |  2 ++
 6 files changed, 54 insertions(+)

diff --git a/cpp/src/db/DB.h b/cpp/src/db/DB.h
index d1198b70..a4163124 100644
--- a/cpp/src/db/DB.h
+++ b/cpp/src/db/DB.h
@@ -44,6 +44,8 @@ public:
 
     virtual Status Size(uint64_t& result) = 0;
 
+    virtual Status BuildIndex(const std::string& table_id) = 0;
+
     virtual Status DropAll() = 0;
 
     DB() = default;
diff --git a/cpp/src/db/DBImpl.cpp b/cpp/src/db/DBImpl.cpp
index 0a1e8651..d0363d3f 100644
--- a/cpp/src/db/DBImpl.cpp
+++ b/cpp/src/db/DBImpl.cpp
@@ -430,6 +430,11 @@ void DBImpl::StartBuildIndexTask() {
     }
 }
 
+Status DBImpl::BuildIndex(const std::string& table_id) {
+    meta_ptr_->UpdateTableFilesToIndex(table_id);
+    return BuildIndexByTable(table_id);
+}
+
 Status DBImpl::BuildIndex(const meta::TableFileSchema& file) {
     ExecutionEnginePtr to_index = EngineFactory::Build(file.dimension_, file.location_, (EngineType)file.engine_type_);
     if(to_index == nullptr) {
@@ -491,7 +496,27 @@ Status DBImpl::BuildIndex(const meta::TableFileSchema& file) {
     return Status::OK();
 }
 
+Status DBImpl::BuildIndexByTable(const std::string& table_id) {
+    std::unique_ptr<std::mutex> lock(build_index_mutex_);
+    meta::TableFilesSchema to_index_files;
+    meta_ptr_->FilesToIndex(to_index_files);
+
+    Status status;
+
+    for (auto& file : to_index_files) {
+        status = BuildIndex(file);
+        if (!status.ok()) {
+            ENGINE_LOG_ERROR << "Building index for " << file.id_ << " failed: " << status.ToString();
+            return status;
+        }
+        ENGINE_LOG_DEBUG << "Sync building index for " << file.id_ << " passed";
+    }
+
+    return status;
+}
+
 void DBImpl::BackgroundBuildIndex() {
+    std::unique_ptr<std::mutex> lock(build_index_mutex_);
     meta::TableFilesSchema to_index_files;
     meta_ptr_->FilesToIndex(to_index_files);
     Status status;
diff --git a/cpp/src/db/DBImpl.h b/cpp/src/db/DBImpl.h
index 9dcd174f..356c7afa 100644
--- a/cpp/src/db/DBImpl.h
+++ b/cpp/src/db/DBImpl.h
@@ -82,6 +82,8 @@ class DBImpl : public DB {
 
     Status Size(uint64_t &result) override;
 
+    Status BuildIndex(const std::string& table_id) override;
+
     ~DBImpl() override;
 
  private:
@@ -110,6 +112,8 @@ class DBImpl : public DB {
     void StartBuildIndexTask();
     void BackgroundBuildIndex();
 
+    Status
+    BuildIndexByTable(const std::string& table_id) {
     Status
     BuildIndex(const meta::TableFileSchema &);
 
@@ -132,6 +136,8 @@ class DBImpl : public DB {
     server::ThreadPool index_thread_pool_;
     std::list<std::future<void>> index_thread_results_;
 
+    std::mutex build_index_mutex_;
+
 }; // DBImpl
 
 
diff --git a/cpp/src/db/DBMetaImpl.cpp b/cpp/src/db/DBMetaImpl.cpp
index d13899dc..0e2b52db 100644
--- a/cpp/src/db/DBMetaImpl.cpp
+++ b/cpp/src/db/DBMetaImpl.cpp
@@ -791,6 +791,23 @@ Status DBMetaImpl::UpdateTableFile(TableFileSchema &file_schema) {
     return Status::OK();
 }
 
+Status DBMetaImpl::UpdateTableFilesToIndex(const std::string& table_id) {
+    try {
+        ConnectorPtr->update_all(
+            set(
+                c(&TableFileSchema::file_type_) = (int) TableFileSchema::TO_INDEX
+            ),
+            where(
+                c(&TableFileSchema::table_id_) == table_id and
+                c(&TableFileSchema::file_type_) == (int) TableFileSchema::RAW
+            ));
+    } catch (std::exception &e) {
+        return HandleException("Encounter exception when update table files to to_index", e);
+    }
+
+    return Status::OK();
+}
+
 Status DBMetaImpl::UpdateTableFiles(TableFilesSchema &files) {
     try {
         MetricCollector metric;
diff --git a/cpp/src/db/DBMetaImpl.h b/cpp/src/db/DBMetaImpl.h
index 1d5f4006..597800c0 100644
--- a/cpp/src/db/DBMetaImpl.h
+++ b/cpp/src/db/DBMetaImpl.h
@@ -35,6 +35,8 @@ public:
                                  const std::vector<size_t>& ids,
                                  TableFilesSchema& table_files) override;
 
+    virtual Status UpdateTableFilesToIndex(const std::string& table_id) override;
+
     virtual Status UpdateTableFile(TableFileSchema& file_schema) override;
 
     virtual Status UpdateTableFiles(TableFilesSchema& files) override;
diff --git a/cpp/src/db/Meta.h b/cpp/src/db/Meta.h
index 6c6dd28b..beab6e22 100644
--- a/cpp/src/db/Meta.h
+++ b/cpp/src/db/Meta.h
@@ -39,6 +39,8 @@ public:
             const std::vector<size_t>& ids,
             TableFilesSchema& table_files) = 0;
 
+    virtual Status UpdateTableFilesToIndex(const std::string& table_id) = 0;
+
     virtual Status UpdateTableFile(TableFileSchema& file_schema) = 0;
 
     virtual Status UpdateTableFiles(TableFilesSchema& files) = 0;
-- 
GitLab