From 938de62fa171aceb5e8e7542cb879d49e35392df Mon Sep 17 00:00:00 2001
From: starlord <chxiehust@gmail.com>
Date: Fri, 18 Oct 2019 11:46:15 +0800
Subject: [PATCH] #39 Create SQ8H index hang if using github server version

Former-commit-id: 056ccbf9cad2de039cc61a512932d3aaea25131d
---
 CHANGELOG.md                               | 1 +
 core/src/db/engine/ExecutionEngineImpl.cpp | 6 +++---
 core/src/scheduler/task/BuildIndexTask.cpp | 6 ++++++
 core/src/utils/ValidationUtil.cpp          | 9 +++++++++
 4 files changed, 19 insertions(+), 3 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 1119c820..f2e22d01 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -29,6 +29,7 @@ Please mark all change in change log and use the ticket from JIRA.
 - \#20 - C++ sdk example get grpc error 
 - \#23 - Add unittest to improve code coverage
 - \#31 - make clang-format failed after run build.sh -l
+- \#39 - Create SQ8H index hang if using github server version
 
 ## Improvement
 - MS-552 - Add and change the easylogging library
diff --git a/core/src/db/engine/ExecutionEngineImpl.cpp b/core/src/db/engine/ExecutionEngineImpl.cpp
index bde45cf1..df802c27 100644
--- a/core/src/db/engine/ExecutionEngineImpl.cpp
+++ b/core/src/db/engine/ExecutionEngineImpl.cpp
@@ -65,7 +65,7 @@ ExecutionEngineImpl::ExecutionEngineImpl(uint16_t dimension, const std::string&
     : location_(location), dim_(dimension), index_type_(index_type), metric_type_(metric_type), nlist_(nlist) {
     index_ = CreatetVecIndex(EngineType::FAISS_IDMAP);
     if (!index_) {
-        throw Exception(DB_ERROR, "Could not create VecIndex");
+        throw Exception(DB_ERROR, "Unsupported index type");
     }
 
     TempMetaConf temp_conf;
@@ -111,7 +111,7 @@ ExecutionEngineImpl::CreatetVecIndex(EngineType type) {
             break;
         }
         default: {
-            ENGINE_LOG_ERROR << "Invalid engine type";
+            ENGINE_LOG_ERROR << "Unsupported index type";
             return nullptr;
         }
     }
@@ -373,7 +373,7 @@ ExecutionEngineImpl::BuildIndex(const std::string& location, EngineType engine_t
 
     auto to_index = CreatetVecIndex(engine_type);
     if (!to_index) {
-        throw Exception(DB_ERROR, "Could not create VecIndex");
+        throw Exception(DB_ERROR, "Unsupported index type");
     }
 
     TempMetaConf temp_conf;
diff --git a/core/src/scheduler/task/BuildIndexTask.cpp b/core/src/scheduler/task/BuildIndexTask.cpp
index 26a2a44e..d8602c14 100644
--- a/core/src/scheduler/task/BuildIndexTask.cpp
+++ b/core/src/scheduler/task/BuildIndexTask.cpp
@@ -134,6 +134,7 @@ XBuildIndexTask::Execute() {
                 ENGINE_LOG_DEBUG << "Failed to update file to index, mark file: " << table_file.file_id_
                                  << " to to_delete";
 
+                build_index_job->BuildIndexDone(to_index_id_);
                 to_index_engine_ = nullptr;
                 return;
             }
@@ -148,6 +149,7 @@ XBuildIndexTask::Execute() {
             std::cout << "ERROR: failed to build index, index file is too large or gpu memory is not enough"
                       << std::endl;
 
+            build_index_job->BuildIndexDone(to_index_id_);
             build_index_job->GetStatus() = Status(DB_ERROR, msg);
             to_index_engine_ = nullptr;
             return;
@@ -158,6 +160,9 @@ XBuildIndexTask::Execute() {
         meta_ptr->HasTable(file_->table_id_, has_table);
         if (!has_table) {
             meta_ptr->DeleteTableFiles(file_->table_id_);
+
+            build_index_job->BuildIndexDone(to_index_id_);
+            build_index_job->GetStatus() = Status(DB_ERROR, "Table has been deleted, discard index file.");
             to_index_engine_ = nullptr;
             return;
         }
@@ -177,6 +182,7 @@ XBuildIndexTask::Execute() {
             std::cout << "ERROR: failed to persist index file: " << table_file.location_
                       << ", possible out of disk space" << std::endl;
 
+            build_index_job->BuildIndexDone(to_index_id_);
             build_index_job->GetStatus() = Status(DB_ERROR, msg);
             to_index_engine_ = nullptr;
             return;
diff --git a/core/src/utils/ValidationUtil.cpp b/core/src/utils/ValidationUtil.cpp
index b982a31f..e275f533 100644
--- a/core/src/utils/ValidationUtil.cpp
+++ b/core/src/utils/ValidationUtil.cpp
@@ -94,6 +94,15 @@ ValidationUtil::ValidateTableIndexType(int32_t index_type) {
         return Status(SERVER_INVALID_INDEX_TYPE, msg);
     }
 
+#ifndef CUSTOMIZATION
+    // special case, hybird index only available in customize faiss library
+    if (engine_type == static_cast<int>(engine::EngineType::FAISS_IVFSQ8H)) {
+        std::string msg = "Unsupported index type: " + std::to_string(index_type);
+        SERVER_LOG_ERROR << msg;
+        return Status(SERVER_INVALID_INDEX_TYPE, msg);
+    }
+#endif
+
     return Status::OK();
 }
 
-- 
GitLab