Merge branch 'branch-0.3.1' into branch-0.4.0

Former-commit-id: e17b77bd31bc3920df8d61fb428a2ab4de2230fc

cpp/CHANGELOG.md

@@ -33,6 +33,7 @@ Please mark all change in change log and use the ticket from JIRA.
 - MS-236 - Add MySQLMetaImpl::HasNonIndexFiles
 - MS-257 - Update bzip2 download url
 - MS-288 - Update compile scripts
+- MS-330 - Stability test failed caused by server core dumped
 
 ## Improvement
 - MS-156 - Add unittest for merge result functions

cpp/src/db/DBImpl.cpp

@@ -370,6 +370,7 @@ Status DBImpl::MergeFiles(const std::string& table_id, const meta::DateT& date,
         const meta::TableFilesSchema& files) {
     ENGINE_LOG_DEBUG << "Merge files for table " << table_id;
 
+    //step 1: create table file
     meta::TableFileSchema table_file;
     table_file.table_id_ = table_id;
     table_file.date_ = date;
@@ -381,6 +382,7 @@ Status DBImpl::MergeFiles(const std::string& table_id, const meta::DateT& date,
         return status;
     }
 
+    //step 2: merge files
     ExecutionEnginePtr index =
             EngineFactory::Build(table_file.dimension_, table_file.location_, (EngineType)table_file.engine_type_);
 
@@ -404,9 +406,25 @@ Status DBImpl::MergeFiles(const std::string& table_id, const meta::DateT& date,
         if (index_size >= options_.index_trigger_size) break;
     }
 
+    //step 3: serialize to disk
+    try {
+        index->Serialize();
+    } catch (std::exception& ex) {
+        //typical error: out of disk space or permition denied
+        std::string msg = "Serialize merged index encounter exception" + std::string(ex.what());
+        ENGINE_LOG_ERROR << msg;
+
+        table_file.file_type_ = meta::TableFileSchema::TO_DELETE;
+        status = meta_ptr_->UpdateTableFile(table_file);
+        ENGINE_LOG_DEBUG << "Failed to update file to index, mark file: " << table_file.file_id_ << " to to_delete";
 
-    index->Serialize();
+        std::cout << "ERROR: failed to persist merged index file: " << table_file.location_
+                  << ", possible out of disk space" << std::endl;
+
+        return Status::Error(msg);
+    }
 
+    //step 4: update table files state
     if (index_size >= options_.index_trigger_size) {
         table_file.file_type_ = meta::TableFileSchema::TO_INDEX;
     } else {
@@ -471,7 +489,7 @@ void DBImpl::BackgroundCompaction(std::set<std::string> table_ids) {
 
     meta_ptr_->Archive();
 
-    int ttl = 1;
+    int ttl = 5*meta::M_SEC;//default: file will be deleted after 5 minutes
     if (options_.mode == Options::MODE::CLUSTER) {
         ttl = meta::D_SEC;
     }

cpp/src/db/Utils.cpp

@@ -131,7 +131,9 @@ Status GetTableFilePath(const DBMetaOptions& options, meta::TableFileSchema& tab
         }
     }
 
-    return Status::Error("Table file doesn't exist: " + table_file.file_id_);
+    std::string msg = "Table file doesn't exist: " + table_file.file_id_;
+    ENGINE_LOG_ERROR << msg;
+    return Status::Error(msg);
 }
 
 Status DeleteTableFilePath(const DBMetaOptions& options, meta::TableFileSchema& table_file) {

cpp/src/db/scheduler/task/IndexLoadTask.cpp

@@ -46,7 +46,20 @@ std::shared_ptr<IScheduleTask> IndexLoadTask::Execute() {
     ExecutionEnginePtr index_ptr = EngineFactory::Build(file_->dimension_,
                                                         file_->location_,
                                                         (EngineType)file_->engine_type_);
-    index_ptr->Load();
+
+    try {
+        index_ptr->Load();
+    } catch (std::exception& ex) {
+        //typical error: out of disk space or permition denied
+        std::string msg = "Failed to load index file: " + std::string(ex.what());
+        ENGINE_LOG_ERROR << msg;
+
+        for(auto& context : search_contexts_) {
+            context->IndexSearchDone(file_->id_);//mark as done avoid dead lock, even failed
+        }
+
+        return nullptr;
+    }
 
     size_t file_size = index_ptr->PhysicalSize();