diff --git a/deps/oblib/src/lib/utility/ob_tracepoint.h b/deps/oblib/src/lib/utility/ob_tracepoint.h index 8ed974c96a913dc1fcb8e8e18fd9ddd62344b798..7b1df77ab3410be199cad4ad8a175e68d89528d1 100644 --- a/deps/oblib/src/lib/utility/ob_tracepoint.h +++ b/deps/oblib/src/lib/utility/ob_tracepoint.h @@ -448,6 +448,7 @@ public: EN_STOP_TENANT_LOG_ARCHIVE_BACKUP = 186, EN_BACKUP_SERVER_DISK_IS_FULL = 188, EN_BACKUP_DATA_CLEAN_ERROR = 189, + EN_BACKUP_COMMON_HEADER_NOT_COMPLETED = 190, EN_BACKUP_SCHEDULER_WEAK_READ = 191, EN_CHECK_STANDBY_CLUSTER_SCHEMA_CONDITION = 201, EN_ALLOCATE_LOB_BUF_FAILED = 202, diff --git a/src/share/backup/ob_backup_meta_store.cpp b/src/share/backup/ob_backup_meta_store.cpp index 5e9921f3513764717d89eed01fc067197e17bd94..83b7075990be947e2156e1e96801cb7637de5ec9 100644 --- a/src/share/backup/ob_backup_meta_store.cpp +++ b/src/share/backup/ob_backup_meta_store.cpp @@ -81,7 +81,13 @@ int64_t ObBackupMeta::get_total_macro_block_count() const } ObBackupMetaFileStore::ObBackupMetaFileStore() - : is_inited_(false), pos_(-1), cur_mem_file_(), allocator_(), base_path_info_(), meta_file_path_list_() + : is_inited_(false), + pos_(-1), + cur_mem_file_(), + allocator_(), + base_path_info_(), + meta_file_path_list_(), + need_swith_next_file_(false) {} ObBackupMetaFileStore::~ObBackupMetaFileStore() @@ -122,6 +128,8 @@ int ObBackupMetaFileStore::next(ObBackupMeta& backup_meta) LOG_WARN("memory file should not be null", KR(ret), K(mem_file)); } else if (OB_FAIL(may_need_parse_common_header(mem_file))) { LOG_WARN("failed to parse common header", KR(ret)); + } else if (need_swith_next_file_) { + FLOG_INFO("backup data has incomplete data, skip it", KPC(mem_file)); } else if (OB_FAIL(parse_sealed_message(mem_file, backup_meta, end_of_one_block))) { LOG_WARN("failed to parse sealed message", KR(ret)); } else if (mem_file->reader_.pos() == mem_file->end_pos_) { @@ -227,7 +235,7 @@ int ObBackupMetaFileStore::get_next_mem_file(MemoryFile*& mem_file) if (IS_NOT_INIT) { ret = OB_NOT_INIT; LOG_WARN("backup meta file store not init", KR(ret)); - } else if (cur_mem_file_.reader_.remain() > 0 && !cur_mem_file_.meet_file_end_mark_) { + } else if (cur_mem_file_.reader_.remain() > 0 && !cur_mem_file_.meet_file_end_mark_ && !need_swith_next_file_) { mem_file = &cur_mem_file_; } else { LOG_INFO("current memory file", K(cur_mem_file_.reader_)); @@ -246,6 +254,7 @@ int ObBackupMetaFileStore::get_next_mem_file(MemoryFile*& mem_file) LOG_WARN("failed to assign mem file", KR(ret), K(tmp_mem_file)); } else { mem_file = &cur_mem_file_; + need_swith_next_file_ = false; } } } @@ -262,6 +271,9 @@ int ObBackupMetaFileStore::may_need_parse_common_header(MemoryFile*& mem_file) LOG_WARN("backup meta file store do not init", KR(ret)); } else if (!mem_file->need_parse_common_header_) { // do not need parse common header + } else if (reader.remain() < sizeof(ObBackupCommonHeader)) { + FLOG_INFO("backup data has incomplete data, skip it", KPC(mem_file)); + need_swith_next_file_ = true; } else if (OB_FAIL(parse_common_header(reader, common_header))) { LOG_WARN("failed to parse common header", KR(ret)); } else if (OB_FAIL(common_header.check_valid())) { diff --git a/src/share/backup/ob_backup_meta_store.h b/src/share/backup/ob_backup_meta_store.h index a3ae5610cd4042eb419f58d09ecfcd5bbf7b726a..52f5b5fc7f4c1b930e3918c2e43292cfc0763ea2 100644 --- a/src/share/backup/ob_backup_meta_store.h +++ b/src/share/backup/ob_backup_meta_store.h @@ -118,6 +118,7 @@ private: common::ObArenaAllocator allocator_; share::ObBackupBaseDataPathInfo base_path_info_; common::ObArray meta_file_path_list_; + bool need_swith_next_file_; private: DISALLOW_COPY_AND_ASSIGN(ObBackupMetaFileStore); diff --git a/src/storage/ob_partition_base_data_backup.cpp b/src/storage/ob_partition_base_data_backup.cpp index abf18f02a4a0592d608526b1d4fc18de7bbaf4fd..90449e44fd390cde7611b2d7a8dc3bf3e92bad34 100644 --- a/src/storage/ob_partition_base_data_backup.cpp +++ b/src/storage/ob_partition_base_data_backup.cpp @@ -1973,6 +1973,23 @@ int ObBackupFileAppender::sync_upload() STORAGE_LOG(WARN, "advance align size fail", K(ret), K(align_size), K(advance_size)); } +#ifdef ERRSIM + if (OB_SUCC(ret) && ObBackupFileType::BACKUP_META_INDEX == file_type_) { + ret = E(EventTable::EN_BACKUP_COMMON_HEADER_NOT_COMPLETED) OB_SUCCESS; + if (OB_FAIL(ret)) { + LOG_WARN("prepare uplaod incomplete header", K(ret), KPC(common_header_)); + const int64_t head_length = sizeof(ObBackupCommonHeader); + if (0 == head_length / 2 || data_buffer_.length() < head_length) { + ret = OB_SUCCESS; + } else if (OB_FAIL(storage_appender_.pwrite(data_buffer_.data(), head_length / 2, file_offset_))) { + STORAGE_LOG(WARN, "storage_writer writer fail", K(ret), K(storage_appender_)); + } else { + ret = OB_IO_ERROR; + } + } + } +#endif + #ifdef ERRSIM if (OB_SUCC(ret) && ObBackupFileType::BACKUP_META_INDEX == file_type_) { ret = E(EventTable::EN_BACKUP_META_INDEX_BUFFER_NOT_COMPLETED) OB_SUCCESS; @@ -2956,7 +2973,7 @@ int ObBackupPhysicalPGCtx::fetch_retry_points(const ObString& path, const ObStri common_header = NULL; int64_t end_pos = 0; if (buffer_reader.remain() < sizeof(ObBackupCommonHeader)) { - STORAGE_LOG(INFO, "backup data has incomplete data, skip it", K(buffer_reader.remain())); + FLOG_INFO("backup data has incomplete data, skip it", K(buffer_reader.remain()), K(path), K(storage_info)); break; } else if (OB_FAIL(buffer_reader.get(common_header))) { STORAGE_LOG(WARN, "read macro index common header fail", K(ret)); diff --git a/src/storage/ob_partition_base_data_physical_restore.cpp b/src/storage/ob_partition_base_data_physical_restore.cpp index 792aa6712953f972b08e120d19fc2099e53e11d1..b257a7a1c8ec44fde981ae8ffd12aee43e6c4672 100644 --- a/src/storage/ob_partition_base_data_physical_restore.cpp +++ b/src/storage/ob_partition_base_data_physical_restore.cpp @@ -753,7 +753,11 @@ int ObPhyRestoreMetaIndexStore::init_one_file( ObBackupMetaIndex meta_index; while (OB_SUCC(ret) && buffer_reader.remain() > 0) { common_header = NULL; - if (OB_FAIL(buffer_reader.get(common_header))) { + + if (buffer_reader.remain() < sizeof(ObBackupCommonHeader)) { + FLOG_INFO("backup data has incomplete data, skip it", K(buffer_reader.remain()), K(path), K(storage_info)); + break; + } else if (OB_FAIL(buffer_reader.get(common_header))) { STORAGE_LOG(WARN, "read meta index common header fail", K(ret)); } else if (OB_ISNULL(common_header)) { ret = OB_ERR_UNEXPECTED; @@ -1115,7 +1119,7 @@ int ObPhyRestoreMacroIndexStore::init_one_file( while (OB_SUCC(ret) && buffer_reader.remain() > 0) { common_header = NULL; if (buffer_reader.remain() < sizeof(ObBackupCommonHeader)) { - STORAGE_LOG(INFO, "backup data has incomplete data, skip it", K(buffer_reader.remain())); + FLOG_INFO("backup data has incomplete data, skip it", K(buffer_reader.remain()), K(path), K(storage_info)); break; } else if (OB_FAIL(buffer_reader.get(common_header))) { STORAGE_LOG(WARN, "read macro index common header fail", K(ret));