Merge branch 'branch-0.4.0' into 'branch-0.4.0'

MS-443 Create index hang again

See merge request megasearch/milvus!452

Former-commit-id: af09afeb0ca73ea1073dda57fe5d4cefb9e3f156

cpp/CHANGELOG.md

@@ -15,6 +15,7 @@ Please mark all change in change log and use the ticket from JIRA.
 - MS-431 - Search vectors params nprobe: 0/-1, expected result: raise exception
 - MS-331 - Crate Table : when table exists, error code is META_FAILED(code=15) rather than ILLEGAL TABLE NAME(code=9))
 - MS-430 - Search no result if index created with FLAT
+- MS-443 - Create index hang again
 
 ## Improvement
 - MS-327 - Clean code for milvus

cpp/src/db/DBImpl.cpp

@@ -581,6 +581,9 @@ Status DBImpl::CreateIndex(const std::string& table_id, const TableIndex& index)
 
     while (!file_ids.empty()) {
         ENGINE_LOG_DEBUG << "Non index files detected! Will build index " << times;
+        if(index.engine_type_ != (int)EngineType::FAISS_IDMAP) {
+            status = meta_ptr_->UpdateTableFilesToIndex(table_id);
+        }
 
         std::this_thread::sleep_for(std::chrono::milliseconds(std::min(10*1000, times*100)));
         status = meta_ptr_->FilesByType(table_id, file_types, file_ids);

cpp/src/db/engine/ExecutionEngineImpl.cpp

@@ -93,7 +93,7 @@ Status ExecutionEngineImpl::AddWithIds(long n, const float *xdata, const long *x
 
 size_t ExecutionEngineImpl::Count() const {
     if(index_ == nullptr) {
-        ENGINE_LOG_ERROR << "ExecutionEngineImpl::index is null";
+        ENGINE_LOG_ERROR << "ExecutionEngineImpl: index is null, return count 0";
         return 0;
     }
     return index_->Count();
@@ -105,7 +105,7 @@ size_t ExecutionEngineImpl::Size() const {
 
 size_t ExecutionEngineImpl::Dimension() const {
     if(index_ == nullptr) {
-        ENGINE_LOG_ERROR << "ExecutionEngineImpl::index is null";
+        ENGINE_LOG_ERROR << "ExecutionEngineImpl: index is null, return dimension " << dim_;
         return dim_;
     }
     return index_->Dimension();
@@ -126,12 +126,16 @@ Status ExecutionEngineImpl::Serialize() {
 Status ExecutionEngineImpl::Load(bool to_cache) {
     index_ = zilliz::milvus::cache::CpuCacheMgr::GetInstance()->GetIndex(location_);
     bool already_in_cache = (index_ != nullptr);
-    if (!index_) {
+    if (!already_in_cache) {
         try {
             double physical_size = PhysicalSize();
             server::CollectExecutionEngineMetrics metrics(physical_size);
             index_ = read_index(location_);
-            ENGINE_LOG_DEBUG << "Disk io from: " << location_;
+            if(index_ == nullptr) {
+                ENGINE_LOG_ERROR << "Failed to load index from " << location_;
+            } else {
+                ENGINE_LOG_DEBUG << "Disk io from: " << location_;
+            }
         } catch (knowhere::KnowhereException &e) {
             ENGINE_LOG_ERROR << e.what();
             return Status::Error(e.what());
@@ -152,6 +156,11 @@ Status ExecutionEngineImpl::CopyToGpu(uint64_t device_id) {
     if (already_in_cache) {
         index_ = index;
     } else {
+        if(index_ == nullptr) {
+            ENGINE_LOG_ERROR << "ExecutionEngineImpl: index is null, failed to copy to gpu";
+            return Status::Error("index is null");
+        }
+
         try {
             index_ = index_->CopyToGpu(device_id);
             ENGINE_LOG_DEBUG << "CPU to GPU" << device_id;
@@ -176,6 +185,11 @@ Status ExecutionEngineImpl::CopyToCpu() {
     if (already_in_cache) {
         index_ = index;
     } else {
+        if(index_ == nullptr) {
+            ENGINE_LOG_ERROR << "ExecutionEngineImpl: index is null, failed to copy to cpu";
+            return Status::Error("index is null");
+        }
+
         try {
             index_ = index_->CopyToCpu();
             ENGINE_LOG_DEBUG << "GPU to CPU";
@@ -194,6 +208,11 @@ Status ExecutionEngineImpl::CopyToCpu() {
 }
 
 ExecutionEnginePtr ExecutionEngineImpl::Clone() {
+    if(index_ == nullptr) {
+        ENGINE_LOG_ERROR << "ExecutionEngineImpl: index is null, failed to clone";
+        return nullptr;
+    }
+
     auto ret = std::make_shared<ExecutionEngineImpl>(dim_, location_, index_type_, metric_type_, nlist_);
     ret->Init();
     ret->index_ = index_->Clone();
@@ -220,6 +239,11 @@ Status ExecutionEngineImpl::Merge(const std::string &location) {
         }
     }
 
+    if(index_ == nullptr) {
+        ENGINE_LOG_ERROR << "ExecutionEngineImpl: index is null, failed to merge";
+        return Status::Error("index is null");
+    }
+
     if (auto file_index = std::dynamic_pointer_cast<BFIndex>(to_merge)) {
         auto ec = index_->Add(file_index->Count(), file_index->GetRawVectors(), file_index->GetRawIds());
         if (ec != server::KNOWHERE_SUCCESS) {
@@ -237,6 +261,11 @@ ExecutionEngineImpl::BuildIndex(const std::string &location, EngineType engine_t
     ENGINE_LOG_DEBUG << "Build index file: " << location << " from: " << location_;
 
     auto from_index = std::dynamic_pointer_cast<BFIndex>(index_);
+    if(from_index == nullptr) {
+        ENGINE_LOG_ERROR << "ExecutionEngineImpl: from_index is null, failed to build index";
+        return nullptr;
+    }
+
     auto to_index = CreatetVecIndex(engine_type);
     if (!to_index) {
         throw Exception("Create Empty VecIndex");
@@ -264,6 +293,11 @@ Status ExecutionEngineImpl::Search(long n,
                                    long nprobe,
                                    float *distances,
                                    long *labels) const {
+    if(index_ == nullptr) {
+        ENGINE_LOG_ERROR << "ExecutionEngineImpl: index is null, failed to search";
+        return Status::Error("index is null");
+    }
+
     ENGINE_LOG_DEBUG << "Search Params: [k]  " << k << " [nprobe] " << nprobe;
     auto ec = index_->Search(n, data, distances, labels, Config::object{{"k", k}, {"nprobe", nprobe}});
     if (ec != server::KNOWHERE_SUCCESS) {