[CP] backup task cannot finish when backup disk is full

deps/oblib/src/lib/utility/ob_tracepoint.h

@@ -448,6 +448,7 @@ public:
     EN_STOP_TENANT_LOG_ARCHIVE_BACKUP = 186,
     EN_BACKUP_SERVER_DISK_IS_FULL = 188,
     EN_BACKUP_DATA_CLEAN_ERROR = 189,
+    EN_BACKUP_COMMON_HEADER_NOT_COMPLETED = 190,
     EN_BACKUP_SCHEDULER_WEAK_READ = 191,
     EN_CHECK_STANDBY_CLUSTER_SCHEMA_CONDITION = 201,
     EN_ALLOCATE_LOB_BUF_FAILED = 202,

src/share/backup/ob_backup_meta_store.cpp

@@ -81,7 +81,13 @@ int64_t ObBackupMeta::get_total_macro_block_count() const
 }
 
 ObBackupMetaFileStore::ObBackupMetaFileStore()
-    : is_inited_(false), pos_(-1), cur_mem_file_(), allocator_(), base_path_info_(), meta_file_path_list_()
+    : is_inited_(false),
+      pos_(-1),
+      cur_mem_file_(),
+      allocator_(),
+      base_path_info_(),
+      meta_file_path_list_(),
+      need_swith_next_file_(false)
 {}
 
 ObBackupMetaFileStore::~ObBackupMetaFileStore()
@@ -122,6 +128,8 @@ int ObBackupMetaFileStore::next(ObBackupMeta& backup_meta)
         LOG_WARN("memory file should not be null", KR(ret), K(mem_file));
       } else if (OB_FAIL(may_need_parse_common_header(mem_file))) {
         LOG_WARN("failed to parse common header", KR(ret));
+      } else if (need_swith_next_file_) {
+        FLOG_INFO("backup data has incomplete data, skip it", KPC(mem_file));
       } else if (OB_FAIL(parse_sealed_message(mem_file, backup_meta, end_of_one_block))) {
         LOG_WARN("failed to parse sealed message", KR(ret));
       } else if (mem_file->reader_.pos() == mem_file->end_pos_) {
@@ -227,7 +235,7 @@ int ObBackupMetaFileStore::get_next_mem_file(MemoryFile*& mem_file)
   if (IS_NOT_INIT) {
     ret = OB_NOT_INIT;
     LOG_WARN("backup meta file store not init", KR(ret));
-  } else if (cur_mem_file_.reader_.remain() > 0 && !cur_mem_file_.meet_file_end_mark_) {
+  } else if (cur_mem_file_.reader_.remain() > 0 && !cur_mem_file_.meet_file_end_mark_ && !need_swith_next_file_) {
     mem_file = &cur_mem_file_;
   } else {
     LOG_INFO("current memory file", K(cur_mem_file_.reader_));
@@ -246,6 +254,7 @@ int ObBackupMetaFileStore::get_next_mem_file(MemoryFile*& mem_file)
         LOG_WARN("failed to assign mem file", KR(ret), K(tmp_mem_file));
       } else {
         mem_file = &cur_mem_file_;
+        need_swith_next_file_ = false;
       }
     }
   }
@@ -262,6 +271,9 @@ int ObBackupMetaFileStore::may_need_parse_common_header(MemoryFile*& mem_file)
     LOG_WARN("backup meta file store do not init", KR(ret));
   } else if (!mem_file->need_parse_common_header_) {
     // do not need parse common header
+  } else if (reader.remain() < sizeof(ObBackupCommonHeader)) {
+    FLOG_INFO("backup data has incomplete data, skip it", KPC(mem_file));
+    need_swith_next_file_ = true;
   } else if (OB_FAIL(parse_common_header(reader, common_header))) {
     LOG_WARN("failed to parse common header", KR(ret));
   } else if (OB_FAIL(common_header.check_valid())) {

src/share/backup/ob_backup_meta_store.h

@@ -118,6 +118,7 @@ private:
   common::ObArenaAllocator allocator_;
   share::ObBackupBaseDataPathInfo base_path_info_;
   common::ObArray<share::ObBackupPath> meta_file_path_list_;
+  bool need_swith_next_file_;
 
 private:
   DISALLOW_COPY_AND_ASSIGN(ObBackupMetaFileStore);

src/storage/ob_partition_base_data_backup.cpp

@@ -1973,6 +1973,23 @@ int ObBackupFileAppender::sync_upload()
       STORAGE_LOG(WARN, "advance align size fail", K(ret), K(align_size), K(advance_size));
     }
 
+#ifdef ERRSIM
+    if (OB_SUCC(ret) && ObBackupFileType::BACKUP_META_INDEX == file_type_) {
+      ret = E(EventTable::EN_BACKUP_COMMON_HEADER_NOT_COMPLETED) OB_SUCCESS;
+      if (OB_FAIL(ret)) {
+        LOG_WARN("prepare uplaod incomplete header", K(ret), KPC(common_header_));
+        const int64_t head_length = sizeof(ObBackupCommonHeader);
+        if (0 == head_length / 2 || data_buffer_.length() < head_length) {
+          ret = OB_SUCCESS;
+        } else if (OB_FAIL(storage_appender_.pwrite(data_buffer_.data(), head_length / 2, file_offset_))) {
+          STORAGE_LOG(WARN, "storage_writer writer fail", K(ret), K(storage_appender_));
+        } else {
+          ret = OB_IO_ERROR;
+        }
+      }
+    }
+#endif
+
 #ifdef ERRSIM
     if (OB_SUCC(ret) && ObBackupFileType::BACKUP_META_INDEX == file_type_) {
       ret = E(EventTable::EN_BACKUP_META_INDEX_BUFFER_NOT_COMPLETED) OB_SUCCESS;
@@ -2956,7 +2973,7 @@ int ObBackupPhysicalPGCtx::fetch_retry_points(const ObString& path, const ObStri
       common_header = NULL;
       int64_t end_pos = 0;
       if (buffer_reader.remain() < sizeof(ObBackupCommonHeader)) {
-        STORAGE_LOG(INFO, "backup data has incomplete data, skip it", K(buffer_reader.remain()));
+        FLOG_INFO("backup data has incomplete data, skip it", K(buffer_reader.remain()), K(path), K(storage_info));
         break;
       } else if (OB_FAIL(buffer_reader.get(common_header))) {
         STORAGE_LOG(WARN, "read macro index common header fail", K(ret));

src/storage/ob_partition_base_data_physical_restore.cpp

@@ -753,7 +753,11 @@ int ObPhyRestoreMetaIndexStore::init_one_file(
     ObBackupMetaIndex meta_index;
     while (OB_SUCC(ret) && buffer_reader.remain() > 0) {
       common_header = NULL;
-      if (OB_FAIL(buffer_reader.get(common_header))) {
+
+      if (buffer_reader.remain() < sizeof(ObBackupCommonHeader)) {
+        FLOG_INFO("backup data has incomplete data, skip it", K(buffer_reader.remain()), K(path), K(storage_info));
+        break;
+      } else if (OB_FAIL(buffer_reader.get(common_header))) {
         STORAGE_LOG(WARN, "read meta index common header fail", K(ret));
       } else if (OB_ISNULL(common_header)) {
         ret = OB_ERR_UNEXPECTED;
@@ -1115,7 +1119,7 @@ int ObPhyRestoreMacroIndexStore::init_one_file(
     while (OB_SUCC(ret) && buffer_reader.remain() > 0) {
       common_header = NULL;
       if (buffer_reader.remain() < sizeof(ObBackupCommonHeader)) {
-        STORAGE_LOG(INFO, "backup data has incomplete data, skip it", K(buffer_reader.remain()));
+        FLOG_INFO("backup data has incomplete data, skip it", K(buffer_reader.remain()), K(path), K(storage_info));
         break;
       } else if (OB_FAIL(buffer_reader.get(common_header))) {
         STORAGE_LOG(WARN, "read macro index common header fail", K(ret));