From 875077c2422c1203e71b0e590a8fa71c6fbf2af7 Mon Sep 17 00:00:00 2001 From: oceanoverflow Date: Wed, 2 Mar 2022 10:34:02 +0800 Subject: [PATCH] fix tenant level backup restore failure --- src/share/backup/ob_backup_info_mgr.cpp | 128 +++++++++++------- src/share/backup/ob_backup_info_mgr.h | 8 +- .../backup/ob_multi_backup_dest_util.cpp | 49 +++++-- src/share/backup/ob_multi_backup_dest_util.h | 11 +- 4 files changed, 135 insertions(+), 61 deletions(-) diff --git a/src/share/backup/ob_backup_info_mgr.cpp b/src/share/backup/ob_backup_info_mgr.cpp index 3bcfb17049..26668175bb 100644 --- a/src/share/backup/ob_backup_info_mgr.cpp +++ b/src/share/backup/ob_backup_info_mgr.cpp @@ -1142,60 +1142,25 @@ int ObRestoreBackupInfoUtil::get_restore_backup_info_v2_( const GetRestoreBackupInfoParam ¶m, ObRestoreBackupInfo &info) { int ret = OB_SUCCESS; - ObFakeBackupLeaseService fake_backup_lease; - ObExternSingleBackupSetInfoMgr backup_info_mgr; - ObLogArchiveBackupInfoMgr log_archive_backup_info_mgr; - ObExternTenantLocalityInfoMgr tenant_locality_info_mgr; - ObExternPGListMgr pg_list_mgr; - ObSimpleBackupSetPath simple_set_path; // largest backup set path - ObSimpleBackupPiecePath simple_piece_path; // smallest backup piece path - ObBackupPath piece_backup_path; const char *backup_dest = param.backup_dest_; const char *backup_cluster_name = param.backup_cluster_name_; const int64_t cluster_id = param.cluster_id_; const int64_t incarnation = param.incarnation_; const int64_t restore_timestamp = param.restore_timestamp_; - const char *passwd_array = param.passwd_array_; - const int64_t cluster_version = ObClusterVersion::get_instance().get_cluster_version(); - - ObExternLogArchiveBackupInfo log_archive_backup_info; - ObTenantLogArchiveStatus log_archive_status; - ObExternTenantLocalityInfo tenant_locality_info; ObBackupPieceInfo piece_info; ObBackupSetFileInfo backup_set_info; - - if (OB_FAIL(param.get_smallest_backup_piece_path(simple_piece_path))) { - LOG_WARN("failed to get smallest simple backup piece path", K(ret)); - } else if (OB_FAIL(param.get_largest_backup_set_path(simple_set_path))) { - LOG_WARN("failed to get smallest largest backup set path", K(ret)); - } else if (OB_FAIL(backup_info_mgr.init(simple_set_path, fake_backup_lease))) { - LOG_WARN("failed to init backup info mgr", K(ret), K(simple_set_path)); - } else if (OB_FAIL(backup_info_mgr.get_extern_backup_set_file_info(passwd_array, backup_set_info))) { - LOG_WARN("failed to find backup info", K(ret), K(restore_timestamp)); - } else if (OB_FAIL(piece_backup_path.init(simple_piece_path.get_simple_path()))) { - LOG_WARN("failed to init piece backup path", K(ret)); - } else if (OB_FAIL(piece_backup_path.join(OB_STR_TENANT_CLOG_SINGLE_BACKUP_PIECE_INFO))) { - LOG_WARN("failed to join single backup piece info"); - } else if (OB_FAIL(log_archive_backup_info_mgr.read_external_single_backup_piece_info( - piece_backup_path, simple_piece_path.get_storage_info(), piece_info, fake_backup_lease))) { - LOG_WARN("failed to read external single backup piece info", K(ret), K(piece_backup_path), K(simple_piece_path)); - } else if (backup_set_info.snapshot_version_ < piece_info.start_ts_) { + ObExternTenantLocalityInfo tenant_locality_info; + bool is_snapshot_restore = false; + if (OB_FAIL( + inner_get_restore_backup_set_info_(param, backup_set_info, tenant_locality_info, info.sys_pg_key_list_))) { + LOG_WARN("failed to inner get restore backup set info", K(ret)); + } else if (OB_FAIL(check_is_snapshot_restore_(param, backup_set_info, is_snapshot_restore))) { + LOG_WARN("failed to check is snapshot backup", K(ret), K(backup_set_info)); + } else if (!is_snapshot_restore && OB_FAIL(inner_get_restore_backup_piece_info_(param, piece_info))) { + LOG_WARN("failed to inner get restore backup piece info", K(ret)); + } else if (!is_snapshot_restore && backup_set_info.snapshot_version_ < piece_info.start_ts_) { ret = OB_ISOLATED_BACKUP_SET; LOG_WARN("log archive status is not continues with backup info", K(ret), K(backup_set_info)); - } else if (OB_FAIL(tenant_locality_info_mgr.init(simple_set_path, fake_backup_lease))) { - LOG_WARN("failed to init tenant locality info mgr", K(ret), K(simple_set_path)); - } else if (OB_FAIL(tenant_locality_info_mgr.get_extern_tenant_locality_info(tenant_locality_info))) { - LOG_WARN("failed to find tenant locality info", K(ret)); - } else if (backup_set_info.cluster_version_ > cluster_version) { - ret = OB_NOT_SUPPORTED; - LOG_ERROR("cannot restore from newer cluster to older cluster", K(ret), K(cluster_version), K(backup_set_info)); - } else if (OB_FAIL(pg_list_mgr.init(simple_set_path, fake_backup_lease))) { - LOG_WARN("failed to get sys pg list", K(ret), K(backup_set_info), K(simple_set_path)); - } else if (OB_FAIL(pg_list_mgr.get_sys_pg_list(info.sys_pg_key_list_))) { - LOG_WARN("failed to get sys pg list", K(ret), K(backup_set_info)); - } else if (info.sys_pg_key_list_.empty()) { - ret = OB_ERR_UNEXPECTED; - LOG_WARN("sys pg key list should not be empty", K(ret), K(backup_set_info)); } else { info.compat_mode_ = tenant_locality_info.compat_mode_; info.frozen_data_version_ = backup_set_info.backup_data_version_; @@ -1214,15 +1179,86 @@ int ObRestoreBackupInfoUtil::get_restore_backup_info_v2_( info.physical_restore_info_.inc_backup_set_id_ = backup_set_info.backup_set_id_; info.physical_restore_info_.incarnation_ = incarnation; info.physical_restore_info_.tenant_id_ = tenant_locality_info.tenant_id_; - info.physical_restore_info_.log_archive_round_ = piece_info.key_.round_id_; info.physical_restore_info_.compatible_ = backup_set_info.compatible_; info.physical_restore_info_.cluster_version_ = backup_set_info.cluster_version_; info.physical_restore_info_.backup_date_ = backup_set_info.date_; + if (!is_snapshot_restore) { + info.physical_restore_info_.log_archive_round_ = piece_info.key_.round_id_; + } FLOG_INFO("get_restore_backup_info", K(info), K(piece_info), K(backup_set_info)); } return ret; } +int ObRestoreBackupInfoUtil::check_is_snapshot_restore_( + const GetRestoreBackupInfoParam ¶m, const ObBackupSetFileInfo &backup_set_info, bool &is_snapshot_restore) +{ + int ret = OB_SUCCESS; + is_snapshot_restore = false; + if (param.backup_piece_path_list_.empty() && !param.backup_set_path_list_.empty()) { + is_snapshot_restore = backup_set_info.snapshot_version_ == param.restore_timestamp_; + LOG_INFO("check is snapshot backup", K(backup_set_info.snapshot_version_), K(param.restore_timestamp_)); + } + return ret; +} + +int ObRestoreBackupInfoUtil::inner_get_restore_backup_set_info_(const GetRestoreBackupInfoParam ¶m, + ObBackupSetFileInfo &backup_set_info, ObExternTenantLocalityInfo &tenant_locality_info, + common::ObIArray &sys_pg_key_list) +{ + int ret = OB_SUCCESS; + ObFakeBackupLeaseService fake_backup_lease; + ObExternPGListMgr pg_list_mgr; + ObExternSingleBackupSetInfoMgr backup_info_mgr; + ObExternTenantLocalityInfoMgr tenant_locality_info_mgr; + const char *passwd_array = param.passwd_array_; + const int64_t cluster_version = ObClusterVersion::get_instance().get_cluster_version(); + ObSimpleBackupSetPath simple_set_path; // largest backup set path + if (OB_FAIL(param.get_largest_backup_set_path(simple_set_path))) { + LOG_WARN("failed to get smallest largest backup set path", K(ret)); + } else if (OB_FAIL(backup_info_mgr.init(simple_set_path, fake_backup_lease))) { + LOG_WARN("failed to init backup info mgr", K(ret), K(simple_set_path)); + } else if (OB_FAIL(backup_info_mgr.get_extern_backup_set_file_info(passwd_array, backup_set_info))) { + LOG_WARN("failed to find backup info", K(ret), K(passwd_array)); + } else if (OB_FAIL(tenant_locality_info_mgr.init(simple_set_path, fake_backup_lease))) { + LOG_WARN("failed to init tenant locality info mgr", K(ret), K(simple_set_path)); + } else if (OB_FAIL(tenant_locality_info_mgr.get_extern_tenant_locality_info(tenant_locality_info))) { + LOG_WARN("failed to find tenant locality info", K(ret)); + } else if (backup_set_info.cluster_version_ > cluster_version) { + ret = OB_NOT_SUPPORTED; + LOG_ERROR("cannot restore from newer cluster to older cluster", K(ret), K(cluster_version), K(backup_set_info)); + } else if (OB_FAIL(pg_list_mgr.init(simple_set_path, fake_backup_lease))) { + LOG_WARN("failed to get sys pg list", K(ret), K(backup_set_info), K(simple_set_path)); + } else if (OB_FAIL(pg_list_mgr.get_sys_pg_list(sys_pg_key_list))) { + LOG_WARN("failed to get sys pg list", K(ret), K(backup_set_info)); + } else if (sys_pg_key_list.empty()) { + ret = OB_ERR_UNEXPECTED; + LOG_WARN("sys pg key list should not be empty", K(ret), K(backup_set_info)); + } + return ret; +} + +int ObRestoreBackupInfoUtil::inner_get_restore_backup_piece_info_( + const GetRestoreBackupInfoParam ¶m, ObBackupPieceInfo &piece_info) +{ + int ret = OB_SUCCESS; + ObFakeBackupLeaseService fake_backup_lease; + ObLogArchiveBackupInfoMgr log_archive_backup_info_mgr; + ObSimpleBackupPiecePath simple_piece_path; // smallest backup piece path + ObBackupPath piece_backup_path; + if (OB_FAIL(param.get_smallest_backup_piece_path(simple_piece_path))) { + LOG_WARN("failed to get smallest simple backup piece path", K(ret)); + } else if (OB_FAIL(piece_backup_path.init(simple_piece_path.get_simple_path()))) { + LOG_WARN("failed to init piece backup path", K(ret)); + } else if (OB_FAIL(piece_backup_path.join(OB_STR_TENANT_CLOG_SINGLE_BACKUP_PIECE_INFO))) { + LOG_WARN("failed to join single backup piece info"); + } else if (OB_FAIL(log_archive_backup_info_mgr.read_external_single_backup_piece_info( + piece_backup_path, simple_piece_path.get_storage_info(), piece_info, fake_backup_lease))) { + LOG_WARN("failed to read external single backup piece info", K(ret), K(piece_backup_path), K(simple_piece_path)); + } + return ret; +} + int ObRestoreBackupInfoUtil::get_restore_sys_table_ids( const ObPhysicalRestoreInfo &info, common::ObIArray &pkey_list) { diff --git a/src/share/backup/ob_backup_info_mgr.h b/src/share/backup/ob_backup_info_mgr.h index d0839a6cc2..d87d13fb51 100644 --- a/src/share/backup/ob_backup_info_mgr.h +++ b/src/share/backup/ob_backup_info_mgr.h @@ -45,7 +45,6 @@ struct ObLogArchiveSimpleInfo final { K_(cur_piece_create_date), K_(is_piece_freezing), K_(prev_piece_id), K_(prev_piece_create_date)); }; -// 用于切主后确认pg的checkpoint ts，可能会触发内部表的访问。 class ObLogArchiveInfoMgr final { public: static ObLogArchiveInfoMgr &get_instance(); @@ -171,6 +170,13 @@ private: static int get_restore_backup_info_v1_(const GetRestoreBackupInfoParam ¶m, ObRestoreBackupInfo &info); // get info from simple path level static int get_restore_backup_info_v2_(const GetRestoreBackupInfoParam ¶m, ObRestoreBackupInfo &info); + static int check_is_snapshot_restore_( + const GetRestoreBackupInfoParam ¶m, const ObBackupSetFileInfo &backup_set_info, bool &is_snapshot_restore); + static int inner_get_restore_backup_set_info_(const GetRestoreBackupInfoParam ¶m, + ObBackupSetFileInfo &backup_set_info, ObExternTenantLocalityInfo &tenant_locality_info, + common::ObIArray &sys_pg_keys); + static int inner_get_restore_backup_piece_info_( + const GetRestoreBackupInfoParam ¶m, ObBackupPieceInfo &piece_info); }; class ObRestoreFatalErrorReporter : public share::ObThreadPool { diff --git a/src/share/backup/ob_multi_backup_dest_util.cpp b/src/share/backup/ob_multi_backup_dest_util.cpp index aedb960341..48bb02ed5b 100644 --- a/src/share/backup/ob_multi_backup_dest_util.cpp +++ b/src/share/backup/ob_multi_backup_dest_util.cpp @@ -157,6 +157,7 @@ int ObMultiBackupDestUtil::get_multi_backup_path_list(const bool is_preview, con int64_t snapshot_version = 0; int64_t start_replay_log_ts = 0; bool is_compat_path = false; + bool is_snapshot_restore = false; if (OB_FAIL(get_backup_set_list(is_preview, cluster_name, cluster_id, @@ -166,8 +167,11 @@ int ObMultiBackupDestUtil::get_multi_backup_path_list(const bool is_preview, con set_list, snapshot_version, start_replay_log_ts, - is_compat_path))) { + is_compat_path, + is_snapshot_restore))) { LOG_WARN("failed to get backup set", KR(ret), K(tenant_id), K(restore_timestamp), K(list)); + } else if (is_snapshot_restore) { + LOG_INFO("tenant level backup no need get piece list", K(tenant_id)); } else { if (is_compat_path) { if (OB_FAIL(get_compat_backup_piece_list(cluster_name, @@ -675,7 +679,7 @@ int ObMultiBackupDestUtil::get_cluster_backup_dest(const ObBackupDest &backup_de int ObMultiBackupDestUtil::get_backup_set_list(const bool is_preview, const char *cluster_name, const int64_t cluster_id, const uint64_t tenant_id, const int64_t restore_timestamp, const common::ObString &backup_dest_str, common::ObArray &list, int64_t &snapshot_version, - int64_t &start_replay_log_ts, bool &is_compat_path) + int64_t &start_replay_log_ts, bool &is_compat_path, bool &is_snapshot_restore) { int ret = OB_SUCCESS; snapshot_version = 0; @@ -703,10 +707,13 @@ int ObMultiBackupDestUtil::get_backup_set_list(const bool is_preview, const char tmp_list, snapshot_version, start_replay_log_ts, - is_compat_path))) { + is_compat_path, + is_snapshot_restore))) { LOG_WARN("failed to inner get backup set list", KR(ret), K(tenant_id), K(backup_dest)); } else if (OB_FAIL(append(list, tmp_list))) { LOG_WARN("failed to add array", KR(ret), K(tmp_list)); + } else { + LOG_INFO("get backup set list", K(list)); } return ret; } @@ -714,7 +721,7 @@ int ObMultiBackupDestUtil::get_backup_set_list(const bool is_preview, const char int ObMultiBackupDestUtil::do_get_backup_set_list(const bool is_preview, const char *cluster_name, const int64_t cluster_id, const uint64_t tenant_id, const int64_t restore_timestamp, const ObBackupDest &backup_dest, common::ObArray &path_list, int64_t &snapshot_version, - int64_t &start_replay_log_ts, bool &is_compat_path) + int64_t &start_replay_log_ts, bool &is_compat_path, bool &is_snapshot_restore) { int ret = OB_SUCCESS; path_list.reset(); @@ -744,7 +751,8 @@ int ObMultiBackupDestUtil::do_get_backup_set_list(const bool is_preview, const c backup_dest, path_list, snapshot_version, - start_replay_log_ts))) { + start_replay_log_ts, + is_snapshot_restore))) { LOG_WARN("failed to do get cluster level backup set list", KR(ret), K(is_preview), @@ -768,6 +776,8 @@ int ObMultiBackupDestUtil::do_get_backup_set_list(const bool is_preview, const c } else { LOG_WARN("failed to get extern backup set file infos", KR(ret), K(simple_path)); } + } else if (OB_FAIL(check_is_snapshot_restore(info, restore_timestamp, is_snapshot_restore))) { + LOG_WARN("failed to check is snapshot backup", K(ret), K(info)); } else { simple_path.backup_set_id_ = info.backup_set_id_; simple_path.copy_id_ = info.copy_id_; @@ -787,7 +797,7 @@ int ObMultiBackupDestUtil::do_get_backup_set_list(const bool is_preview, const c int ObMultiBackupDestUtil::do_get_backup_set_list_from_cluster_level(const bool is_preview, const char *cluster_name, const int64_t cluster_id, const uint64_t tenant_id, const int64_t restore_timestamp, const ObBackupDest &backup_dest, common::ObArray &path_list, int64_t &snapshot_version, - int64_t &start_replay_log_ts) + int64_t &start_replay_log_ts, bool &is_snapshot_restore) { int ret = OB_SUCCESS; ObClusterBackupDest cluster_backup_dest; @@ -823,7 +833,8 @@ int ObMultiBackupDestUtil::do_get_backup_set_list_from_cluster_level(const bool file_infos, path_list, snapshot_version, - start_replay_log_ts))) { + start_replay_log_ts, + is_snapshot_restore))) { LOG_WARN("failed to do inner get backup set list", KR(ret), K(restore_timestamp)); } else { std::sort(path_list.begin(), path_list.end(), cmp_set); @@ -869,8 +880,10 @@ int ObMultiBackupDestUtil::do_get_backup_set_list_from_cluster_level(const bool ObClusterBackupDest cluster_backup_dest; ObBackupDest tmp_backup_dest; char tmp_simple_path_str[OB_MAX_BACKUP_DEST_LENGTH] = ""; - if (OB_FAIL(cluster_backup_dest.set( - tmp_file_info.backup_dest_.ptr(), cluster_name, cluster_id, OB_START_INCARNATION))) { + if (OB_FAIL(check_is_snapshot_restore(tmp_file_info, restore_timestamp, is_snapshot_restore))) { + LOG_WARN("failed to check is snaptshot backup", K(ret), K(tmp_file_info)); + } else if (OB_FAIL(cluster_backup_dest.set( + tmp_file_info.backup_dest_.ptr(), cluster_name, cluster_id, OB_START_INCARNATION))) { LOG_WARN("failed to set cluster backup dest"); } else if (OB_FAIL(base_data_path_info.set(cluster_backup_dest, tmp_file_info.tenant_id_, @@ -908,7 +921,8 @@ int ObMultiBackupDestUtil::do_get_backup_set_list_from_cluster_level(const bool int ObMultiBackupDestUtil::do_inner_get_backup_set_list(const char *cluster_name, const int64_t cluster_id, const int64_t restore_timestamp, const ObBackupDest &backup_dest, const ObArray &file_infos, - common::ObArray &path_list, int64_t &snapshot_version, int64_t &start_replay_log_ts) + common::ObArray &path_list, int64_t &snapshot_version, int64_t &start_replay_log_ts, + bool &is_snapshot_restore) { int ret = OB_SUCCESS; snapshot_version = -1; @@ -939,6 +953,8 @@ int ObMultiBackupDestUtil::do_inner_get_backup_set_list(const char *cluster_name const ObBackupSetFileInfo &info = file_infos.at(i); if (OB_SUCCESS != info.result_) { // do nothing + } else if (OB_FAIL(check_is_snapshot_restore(info, restore_timestamp, is_snapshot_restore))) { + LOG_WARN("failed to check is snaptshot backup", K(ret), K(info), K(restore_timestamp)); } else if (!ObBackupFileStatus::can_show_in_preview(info.file_status_)) { LOG_INFO("backup set info cannot list in preview", K(info)); } else { @@ -1426,5 +1442,18 @@ int ObMultiBackupDestUtil::check_backup_path_is_backup_backup(const char *cluste return ret; } +int ObMultiBackupDestUtil::check_is_snapshot_restore( + const share::ObBackupSetFileInfo &backup_set_info, const int64_t restore_timestamp, bool &is_snapshot_restore) +{ + int ret = OB_SUCCESS; + if (!backup_set_info.is_valid() || restore_timestamp <= 0) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("get invalid args", K(ret), K(backup_set_info), K(restore_timestamp)); + } else { + is_snapshot_restore = backup_set_info.snapshot_version_ == restore_timestamp; + } + return ret; +} + } // end namespace share } // end namespace oceanbase diff --git a/src/share/backup/ob_multi_backup_dest_util.h b/src/share/backup/ob_multi_backup_dest_util.h index 52c6155f67..7b30a6cff2 100644 --- a/src/share/backup/ob_multi_backup_dest_util.h +++ b/src/share/backup/ob_multi_backup_dest_util.h @@ -89,18 +89,19 @@ private: static int get_backup_set_list(const bool is_preview, const char *cluster_name, const int64_t cluster_id, const uint64_t tenant_id, const int64_t restore_timestamp, const common::ObString &backup_dest_str, common::ObArray &path_list, int64_t &snapshot_version, int64_t &start_replay_log_ts, - bool &is_compat_path); + bool &is_compat_path, bool &is_snapshot_restore); static int do_get_backup_set_list(const bool is_preview, const char *cluster_name, const int64_t cluster_id, const uint64_t tenant_id, const int64_t restore_timestamp, const ObBackupDest &backup_dest, common::ObArray &path_list, int64_t &snapshot_version, int64_t &start_replay_log_ts, - bool &is_compat_path); + bool &is_compat_path, bool &is_snapshot_restore); static int do_get_backup_set_list_from_cluster_level(const bool is_preview, const char *cluster_name, const int64_t cluster_id, const uint64_t tenant_id, const int64_t restore_timestamp, const ObBackupDest &backup_dest, common::ObArray &path_list, int64_t &snapshot_version, - int64_t &start_replay_log_ts); + int64_t &start_replay_log_ts, bool &is_snapshot_restore); static int do_inner_get_backup_set_list(const char *cluster_name, const int64_t cluster_id, const int64_t restore_timestamp, const ObBackupDest &backup_dest, const ObArray &file_infos, - common::ObArray &path_list, int64_t &snapshot_version, int64_t &start_replay_log_ts); + common::ObArray &path_list, int64_t &snapshot_version, int64_t &start_replay_log_ts, + bool &is_snapshot_restore); static int get_backup_piece_list(const bool is_preview, const char *cluster_name, const int64_t cluster_id, const uint64_t tenant_id, const int64_t snapshot_version, const int64_t start_replay_log_ts, const int64_t restore_timestamp, const common::ObString &backup_dest_str, @@ -125,6 +126,8 @@ private: static int check_backup_path_is_backup_backup(const char *cluster_name, const int64_t cluster_id, const common::ObString &root_path, const common::ObString &storage_info, const uint64_t tenant_id, bool &is_backup_backup); + static int check_is_snapshot_restore( + const share::ObBackupSetFileInfo &backup_set_info, const int64_t restore_timestamp, bool &is_snapshot_restore); private: DISALLOW_COPY_AND_ASSIGN(ObMultiBackupDestUtil); -- GitLab