Merge branch 'branch-0.3.1' into 'branch-0.3.1'

MS-251 Build/Merge background thread error handling

See merge request megasearch/milvus!302

Former-commit-id: 3820c658e6c2f86e683955979313d19009c10284

cpp/src/db/DBImpl.cpp

@@ -558,15 +558,25 @@ Status DBImpl::BuildIndex(const meta::TableFileSchema& file) {
         auto to_remove = file;
         to_remove.file_type_ = meta::TableFileSchema::TO_DELETE;
 
-        meta::TableFilesSchema update_files = {to_remove, table_file};
-        meta_ptr_->UpdateTableFiles(update_files);
-
-        ENGINE_LOG_DEBUG << "New index file " << table_file.file_id_ << " of size "
-                   << index->PhysicalSize() << " bytes"
-                   << " from file " << to_remove.file_id_;
-
-        if(options_.insert_cache_immediately_) {
-            index->Cache();
+        meta::TableFilesSchema update_files = {table_file, to_remove};
+        status = meta_ptr_->UpdateTableFiles(update_files);
+        if(status.ok()) {
+            ENGINE_LOG_DEBUG << "New index file " << table_file.file_id_ << " of size "
+                             << index->PhysicalSize() << " bytes"
+                             << " from file " << to_remove.file_id_;
+
+            if(options_.insert_cache_immediately_) {
+                index->Cache();
+            }
+        } else {
+            //failed to update meta, mark the new file as to_delete, don't delete old file
+            to_remove.file_type_ = meta::TableFileSchema::TO_INDEX;
+            status = meta_ptr_->UpdateTableFile(to_remove);
+            ENGINE_LOG_DEBUG << "Failed to update file to index, mark file: " << to_remove.file_id_ << " to to_index";
+
+            table_file.file_type_ = meta::TableFileSchema::TO_DELETE;
+            status = meta_ptr_->UpdateTableFile(table_file);
+            ENGINE_LOG_DEBUG << "Failed to update file to index, mark file: " << table_file.file_id_ << " to to_delete";
         }
 
     } catch (std::exception& ex) {