Fix physical backup &restore reuse trans sstable bug

src/rootserver/backup/ob_root_validate.cpp

@@ -1836,7 +1836,7 @@ int ObTenantValidate::fetch_all_pg_list(const ObClusterBackupDest& backup_dest,
   ObExternPGListMgr extern_pg_list_mgr;
   ObBackupListDataMgr list_data_mgr;
   // TODO(yanfeng.yyy) full backup set id not use backup info
-  const int64_t compatible = OB_BACKUP_COMPATIBLE_VERSION_V3;
+  const int64_t compatible = OB_BACKUP_CURRENT_COMPAITBLE_VERSION;
   const int64_t fake_piece_id = 0;
   const int64_t fake_create_ts = 0;
 

src/rootserver/ob_root_backup.cpp

@@ -1178,7 +1178,7 @@ int ObRootBackup::do_extern_backup_set_infos(const ObBaseBackupInfoStruct &info,
                  inc_backup_set_id,
                  backup_dest,
                  extern_backup_info.date_,
-                 OB_BACKUP_COMPATIBLE_VERSION_V3,
+                 OB_BACKUP_CURRENT_COMPAITBLE_VERSION,
                  *backup_lease_service_))) {
     LOG_WARN("failed to init extern backup set info mgr", K(ret), K(tenant_id), K(full_backup_set_id));
   } else {
@@ -1233,7 +1233,7 @@ int ObRootBackup::do_extern_backup_tenant_locality_infos(const ObBaseBackupInfoS
                  inc_backup_set_id,
                  backup_dest,
                  extern_backup_info.date_,
-                 OB_BACKUP_COMPATIBLE_VERSION_V3,
+                 OB_BACKUP_CURRENT_COMPAITBLE_VERSION,
                  *backup_lease_service_))) {
     LOG_WARN("failed to init extern backup set info mgr", K(ret), K(tenant_id), K(full_backup_set_id));
   } else if (OB_FAIL(extern_tenant_locality_info.tenant_name_.assign(tenant_info->get_tenant_name()))) {
@@ -2897,7 +2897,7 @@ int ObRootBackup::calculate_tenant_start_replay_log_ts(const share::ObTenantBack
   const bool need_check_all_meta_files = true;
   const bool need_check_compeleted = true;
   ObArray<ObBackupMetaIndex> backup_meta_indexs;
-  ObBackupCompatibleVersion compatible = OB_BACKUP_COMPATIBLE_VERSION_V3;
+  ObBackupCompatibleVersion compatible = OB_BACKUP_CURRENT_COMPAITBLE_VERSION;
   start_replay_log_ts = 0;
 
   if (!tenant_task_info.is_valid() || ObTenantBackupTaskInfo::FINISH != tenant_task_info.status_) {

src/share/backup/ob_backup_struct.h

@@ -45,10 +45,14 @@ int get_backup_copy_id_range_by_copy_level(
 enum ObBackupCompatibleVersion          // used for data backup
 { OB_BACKUP_COMPATIBLE_VERSION_V1 = 1,  // since 2.2.60
   OB_BACKUP_COMPATIBLE_VERSION_V2 = 2,  // since 2.2.77
-  OB_BACKUP_COMPATIBLE_VERSION_V3 = 3,  // since 3.1 TODO(muwei): use v3 in 3.1
+  OB_BACKUP_COMPATIBLE_VERSION_V3 = 3,  // since 3.1 before BP6
+  OB_BACKUP_COMPATIBLE_VERSION_V4 = 4,  // since 3.1 after BP6
   OB_BACKUP_COMPATIBLE_VERSION_MAX,
 };
 
+const static ObBackupCompatibleVersion OB_BACKUP_CURRENT_COMPAITBLE_VERSION =
+    ObBackupCompatibleVersion::OB_BACKUP_COMPATIBLE_VERSION_V4;
+
 bool has_independ_inc_backup_set(const int64_t version);
 // for log archive and data backup, exclude backup lease service inner table
 enum ObBackupInnerTableVersion {

src/share/backup/ob_extern_backup_info_mgr.cpp

@@ -694,7 +694,7 @@ int ObExternBackupInfoMgr::get_extern_backup_info(const ObBaseBackupInfoStruct&
         extern_backup_info.frozen_snapshot_version_ = frozen_status.frozen_timestamp_;
         extern_backup_info.frozen_schema_version_ = schema_versions.at(0).schema_version_;
         extern_backup_info.status_ = ObExternBackupInfo::DOING;
-        extern_backup_info.compatible_ = OB_BACKUP_COMPATIBLE_VERSION_V3;
+        extern_backup_info.compatible_ = OB_BACKUP_CURRENT_COMPAITBLE_VERSION;
         extern_backup_info.cluster_version_ = ObClusterVersion::get_instance().get_cluster_version();
         extern_backup_info.encryption_mode_ = info.encryption_mode_;
         extern_backup_info.passwd_ = info.passwd_;
@@ -2421,7 +2421,7 @@ int ObExternTenantBackupDiagnoseMgr::upload_tenant_backup_diagnose_info(const Ob
                  full_backup_set_id_,
                  inc_backup_set_id_,
                  backup_date_,
-                 OB_BACKUP_COMPATIBLE_VERSION_V3))) {
+                 OB_BACKUP_CURRENT_COMPAITBLE_VERSION))) {
     LOG_WARN("failed to set path info", K(ret), K(backup_dest_), K(tenant_id_));
   } else if (OB_FAIL(ObBackupPathUtil::get_tenant_backup_diagnose_path(path_info, path))) {
     LOG_WARN("failed to get tenant data backup info path", K(ret), K(backup_dest_));
@@ -3143,7 +3143,7 @@ int ObExternSingleBackupSetInfoMgr::init(const uint64_t tenant_id, const int64_t
   int ret = OB_SUCCESS;
   ObBackupPath path;
   ObBackupBaseDataPathInfo path_info;
-  const int64_t compatible = OB_BACKUP_COMPATIBLE_VERSION_V3;
+  const int64_t compatible = OB_BACKUP_CURRENT_COMPAITBLE_VERSION;
 
   if (is_inited_) {
     ret = OB_INIT_TWICE;

src/storage/backup/ob_partition_base_data_physical_restore_v2.cpp

@@ -708,13 +708,47 @@ int ObPartitionMacroBlockRestoreReaderV2::get_macro_block_path(
 {
   int ret = OB_SUCCESS;
   path.reset();
+  bool deal_trans_sstable_as_minor = false;
+
   if (OB_UNLIKELY(!is_inited_)) {
     ret = OB_NOT_INIT;
     STORAGE_LOG(WARN, "not init", K(ret));
   } else if (!macro_index.is_valid()) {
     ret = OB_INVALID_ARGUMENT;
     STORAGE_LOG(WARN, "get macro block path get invalid argument", K(ret), K(macro_index));
+  } else if (backup_table_key_.is_trans_sstable()) {
+    if (OB_BACKUP_COMPATIBLE_VERSION_V4 == restore_info_->restore_info_.compatible_) {
+      if (OB_FAIL(get_minor_macro_block_path_(macro_index, path))) {
+        LOG_WARN("failed to get minor macro block path", K(ret), K(macro_index));
+      }
+    } else if (ObBackupCompatibleVersion::OB_BACKUP_COMPATIBLE_VERSION_V3 == restore_info_->restore_info_.compatible_) {
+      if (OB_FAIL(get_major_macro_block_path_(macro_index, path))) {
+        LOG_WARN("failed to get major macro block path", K(ret), K(macro_index));
+      }
+    }
   } else if (backup_table_key_.is_major_sstable()) {
+    if (OB_FAIL(get_major_macro_block_path_(macro_index, path))) {
+      LOG_WARN("failed to get major macro block path", K(ret), K(macro_index));
+    }
+  } else if (backup_table_key_.is_minor_sstable()) {
+    if (OB_FAIL(get_minor_macro_block_path_(macro_index, path))) {
+      LOG_WARN("failed to get minor macro block path", K(ret), K(macro_index));
+    }
+  }
+  return ret;
+}
+
+int ObPartitionMacroBlockRestoreReaderV2::get_major_macro_block_path_(
+    const ObBackupTableMacroIndex &macro_index, share::ObBackupPath &path)
+{
+  int ret = OB_SUCCESS;
+  if (OB_UNLIKELY(!is_inited_)) {
+    ret = OB_NOT_INIT;
+    STORAGE_LOG(WARN, "not init", K(ret));
+  } else if (!macro_index.is_valid()) {
+    ret = OB_INVALID_ARGUMENT;
+    STORAGE_LOG(WARN, "get macro block path get invalid argument", K(ret), K(macro_index));
+  } else {
     ObSimpleBackupSetPath simple_backup_set_path;
     ObArray<ObSimpleBackupSetPath> path_list;
     const int64_t backup_set_id = macro_index.backup_set_id_;
@@ -745,16 +779,28 @@ int ObPartitionMacroBlockRestoreReaderV2::get_macro_block_path(
                    path))) {
       LOG_WARN("fail to get major macro block file path", K(ret), K(simple_backup_set_path), K(macro_index));
     }
-  } else if (backup_table_key_.is_minor_sstable()) {
-    if (OB_FAIL(ObBackupPathUtil::get_minor_macro_block_file_path(simple_path_,
-            backup_pgkey_.get_table_id(),
-            backup_pgkey_.get_partition_id(),
-            macro_index.backup_set_id_,
-            macro_indexs_->get_backup_task_id(),
-            macro_index.sub_task_id_,
-            path))) {
-      STORAGE_LOG(WARN, "fail to get meta file path", K(ret));
-    }
+  }
+  return ret;
+}
+
+int ObPartitionMacroBlockRestoreReaderV2::get_minor_macro_block_path_(
+    const ObBackupTableMacroIndex &macro_index, share::ObBackupPath &path)
+{
+  int ret = OB_SUCCESS;
+  if (OB_UNLIKELY(!is_inited_)) {
+    ret = OB_NOT_INIT;
+    STORAGE_LOG(WARN, "not init", K(ret));
+  } else if (!macro_index.is_valid()) {
+    ret = OB_INVALID_ARGUMENT;
+    STORAGE_LOG(WARN, "get macro block path get invalid argument", K(ret), K(macro_index));
+  } else if (OB_FAIL(ObBackupPathUtil::get_minor_macro_block_file_path(simple_path_,
+                 backup_pgkey_.get_table_id(),
+                 backup_pgkey_.get_partition_id(),
+                 macro_index.backup_set_id_,
+                 macro_indexs_->get_backup_task_id(),
+                 macro_index.sub_task_id_,
+                 path))) {
+    STORAGE_LOG(WARN, "fail to get meta file path", K(ret));
   }
   return ret;
 }
@@ -2082,7 +2128,7 @@ int ObPhyRestoreMacroIndexStoreV2::get_major_macro_index_array(
   } else {
     for (MacroIndexMap::const_iterator iter = index_map_.begin(); iter != index_map_.end(); ++iter) {
       const ObITable::TableKey &table_key = iter->first;
-      if (!table_key.is_major_sstable()) {
+      if (!table_key.is_major_sstable() || table_key.is_trans_sstable()) {
         // do nothing
       } else if (table_id == table_key.table_id_) {
         index_list = iter->second;

src/storage/backup/ob_partition_base_data_physical_restore_v2.h

@@ -167,6 +167,8 @@ private:
   int trans_macro_block(
       const uint64_t table_id, blocksstable::ObMacroBlockMetaV2 &meta, blocksstable::ObBufferReader &data);
   int get_macro_block_path(const ObBackupTableMacroIndex &macro_index, share::ObBackupPath &path);
+  int get_major_macro_block_path_(const ObBackupTableMacroIndex &macro_index, share::ObBackupPath &path);
+  int get_minor_macro_block_path_(const ObBackupTableMacroIndex &macro_index, share::ObBackupPath &path);
 
 private:
   bool is_inited_;

src/storage/ob_partition_backup.cpp

@@ -1271,7 +1271,7 @@ int ObBackupPrepareTask::build_backup_major_sstable(
   } else {
     for (int64_t i = 0; OB_SUCC(ret) && i < local_tables.count(); ++i) {
       const ObITable::TableKey &local_table = local_tables.at(i);
-      if (local_table.is_major_sstable()) {
+      if (local_table.is_major_sstable() && !local_table.is_trans_sstable()) {
         ObMigrateTableInfo::SSTableInfo info;
         info.src_table_key_ = local_table;
         info.dest_base_version_ = local_table.get_base_version();
@@ -1301,7 +1301,7 @@ int ObBackupPrepareTask::build_backup_minor_sstable(
     ObMigrateTableInfo::SSTableInfo info;
     for (int64_t i = 0; OB_SUCC(ret) && i < local_tables.count(); ++i) {
       const ObITable::TableKey &local_table = local_tables.at(i);
-      if (local_table.is_minor_sstable()) {
+      if (local_table.is_minor_sstable() || local_table.is_trans_sstable()) {
         info.reset();
         info.src_table_key_ = local_table;
         info.dest_base_version_ = local_table.trans_version_range_.base_version_;
@@ -1534,7 +1534,8 @@ int ObBackupPrepareTask::fetch_backup_major_sstables(ObIArray<ObITable::TableKey
             if (OB_UNLIKELY(!major_table_key.is_valid())) {
               ret = OB_INVALID_ARGUMENT;
               LOG_WARN("invalid major table key", K(ret), K(major_table_key));
-            } else if (OB_UNLIKELY(!ObITable::is_major_sstable(major_table_key.table_type_))) {
+            } else if (OB_UNLIKELY(!ObITable::is_major_sstable(major_table_key.table_type_) ||
+                                   ObITable::is_trans_sstable(major_table_key.table_type_))) {
               ret = OB_ERR_SYS;
               LOG_ERROR("table type is not major sstable", K(ret), K(major_table_key), K(table_info));
             } else if (OB_FAIL(table_keys.push_back(major_table_key))) {
@@ -1573,9 +1574,10 @@ int ObBackupPrepareTask::fetch_backup_minor_sstables(ObIArray<ObITable::TableKey
             if (OB_UNLIKELY(!minor_table_key.is_valid())) {
               ret = OB_INVALID_ARGUMENT;
               LOG_WARN("invalid major table key", K(ret), K(minor_table_key));
-            } else if (OB_UNLIKELY(!ObITable::is_minor_sstable(minor_table_key.table_type_))) {
+            } else if (OB_UNLIKELY(!ObITable::is_minor_sstable(minor_table_key.table_type_) &&
+                                   !ObITable::is_trans_sstable(minor_table_key.table_type_))) {
               ret = OB_ERR_SYS;
-              LOG_ERROR("table type is not minor sstable", K(ret), K(minor_table_key), K(table_info));
+              LOG_ERROR("table type is not minor sstable or trans sstable", K(ret), K(minor_table_key), K(table_info));
             } else if (OB_FAIL(table_keys.push_back(minor_table_key))) {
               LOG_WARN("failed to push major table key into array", K(ret), K(minor_table_key));
             }

src/storage/ob_partition_base_data_backup.cpp

@@ -2806,7 +2806,8 @@ int ObBackupPhysicalPGCtx::fetch_prev_macro_index(const ObPhyRestoreMacroIndexSt
     ret = OB_NOT_OPEN;
     LOG_WARN("not opened yet", K(ret));
   } else if (OB_UNLIKELY(!macro_index_store.is_inited() || !macro_arg.is_valid() ||
-                         !macro_arg.table_key_ptr_->is_major_sstable())) {
+                         !macro_arg.table_key_ptr_->is_major_sstable() ||
+                         macro_arg.table_key_ptr_->is_trans_sstable())) {
     ret = OB_INVALID_ARGUMENT;
     STORAGE_LOG(WARN, "invalid argument", K(ret), K(macro_index_store), K(macro_arg));
   } else {
@@ -3616,7 +3617,7 @@ int ObBackupCopyPhysicalTask::get_datafile_appender(const ObITable::TableType& t
       STORAGE_LOG(WARN, "BandwidthThrottle should not be null here", K(ret));
     } else if (OB_FAIL(arg.get_backup_base_data_info(path_info))) {
       STORAGE_LOG(WARN, "get backup base data info fail", K(ret), K(arg), K(pg_key));
-    } else if (ObITable::is_major_sstable(table_type)) {
+    } else if (ObITable::is_major_sstable(table_type) && !ObITable::is_trans_sstable(table_type)) {
       if (OB_FAIL(ObBackupPathUtil::get_major_macro_block_file_path(path_info,
               pg_key.get_table_id(),
               pg_key.get_partition_id(),
@@ -3625,7 +3626,7 @@ int ObBackupCopyPhysicalTask::get_datafile_appender(const ObITable::TableType& t
               path))) {
         STORAGE_LOG(WARN, "failed to get macro file path", K(ret));
       }
-    } else if (ObITable::is_minor_sstable(table_type)) {
+    } else if (ObITable::is_minor_sstable(table_type) || ObITable::is_trans_sstable(table_type)) {
       if (OB_FAIL(ObBackupPathUtil::get_minor_macro_block_file_path(path_info,
               pg_key.get_table_id(),
               pg_key.get_partition_id(),

src/storage/ob_partition_base_data_physical_restore.cpp

@@ -169,7 +169,7 @@ int ObRestoreFileUtil::read_partition_meta(const ObString &path, const ObString
         } else if (OB_FAIL(partition_meta.copy_from_old_meta(tmp_meta))) {
           LOG_WARN("failed to copy_from_old_meta", K(ret), K(tmp_meta));
         }
-      } else if (OB_BACKUP_COMPATIBLE_VERSION_V3 == compatible) {
+      } else if (OB_BACKUP_COMPATIBLE_VERSION_V3 == compatible || OB_BACKUP_COMPATIBLE_VERSION_V4 == compatible) {
         if (OB_FAIL(buffer_reader.read_serialize(partition_meta))) {
           STORAGE_LOG(WARN, "read partition meta fail", K(ret), K(path), K(meta_index));
         }
@@ -820,7 +820,8 @@ int ObPhyRestoreMetaIndexStore::check_meta_index_completed(
   if (ObBackupCompatibleVersion::OB_BACKUP_COMPATIBLE_VERSION_V1 == compatible ||
       OB_BACKUP_COMPATIBLE_VERSION_V2 == compatible) {
     type = ObBackupMetaType::PARTITION_GROUP_META;
-  } else if (ObBackupCompatibleVersion::OB_BACKUP_COMPATIBLE_VERSION_V3 == compatible) {
+  } else if (ObBackupCompatibleVersion::OB_BACKUP_COMPATIBLE_VERSION_V3 == compatible ||
+             ObBackupCompatibleVersion::OB_BACKUP_COMPATIBLE_VERSION_V4 == compatible) {
     type = ObBackupMetaType::PARTITION_GROUP_META_INFO;
   } else {
     ret = OB_ERR_UNEXPECTED;

src/storage/ob_partition_migrator.cpp

@@ -5433,7 +5433,7 @@ int ObMigrateDag::init_for_restore_(ObMigrateCtx &ctx)
     STORAGE_LOG(WARN, "failed to get backup pgkey", K(ret), K(ctx.replica_op_arg_.phy_restore_arg_));
   } else if (FALSE_IT(group_task = reinterpret_cast<ObPartGroupMigrationTask *>(ctx.group_task_))) {
   } else if (OB_ISNULL(ctx.macro_indexs_)) {
-    if (OB_BACKUP_COMPATIBLE_VERSION_V3 == compatible) {
+    if (OB_BACKUP_COMPATIBLE_VERSION_V3 == compatible || OB_BACKUP_COMPATIBLE_VERSION_V4 == compatible) {
       const ObBackupMetaType meta_type = ObBackupMetaType::PARTITION_GROUP_META_INFO;
       ObPhyRestoreMacroIndexStoreV2 *phy_restore_macro_index_v2 = NULL;
       if (OB_ISNULL(phy_restore_macro_index_v2 = MIGRATOR.get_cp_fty()->get_phy_restore_macro_index_v2())) {
@@ -6178,7 +6178,7 @@ int ObMigratePrepareTask::prepare_restore_reader()
   if (!is_inited_) {
     ret = OB_NOT_INIT;
     LOG_ERROR("not inited", K(ret));
-  } else if (OB_BACKUP_COMPATIBLE_VERSION_V3 == compatible) {
+  } else if (OB_BACKUP_COMPATIBLE_VERSION_V3 == compatible || OB_BACKUP_COMPATIBLE_VERSION_V4 == compatible) {
     ObPartitionGroupMetaRestoreReaderV2 *restore_meta_reader_v2 = NULL;
     ObPhyRestoreMacroIndexStoreV2 *macro_index = NULL;
 
@@ -7108,7 +7108,7 @@ int ObMigratePrepareTask::choose_restore_migrate_src(
     } else {
       if (OB_BACKUP_COMPATIBLE_VERSION_V1 == compatible || OB_BACKUP_COMPATIBLE_VERSION_V2 == compatible) {
         ctx_->fetch_pg_info_compat_version_ = ObFetchPGInfoArg::FETCH_PG_INFO_ARG_COMPAT_VERSION_V1;
-      } else if (OB_BACKUP_COMPATIBLE_VERSION_V3 == compatible) {
+      } else if (OB_BACKUP_COMPATIBLE_VERSION_V3 == compatible || OB_BACKUP_COMPATIBLE_VERSION_V4 == compatible) {
         ctx_->fetch_pg_info_compat_version_ = ObFetchPGInfoArg::FETCH_PG_INFO_ARG_COMPAT_VERSION_V2;
       } else {
         ret = OB_ERR_UNEXPECTED;
@@ -10371,7 +10371,7 @@ int ObITableTaskGeneratorTask::get_base_meta_reader(
       if (OB_FAIL(get_base_meta_restore_reader_v1(arg.table_key_, reader))) {
         STORAGE_LOG(WARN, "fail to get_base_meta_restore_reader_v1", K(ret));
       }
-    } else if (OB_BACKUP_COMPATIBLE_VERSION_V3 == compatible) {
+    } else if (OB_BACKUP_COMPATIBLE_VERSION_V3 == compatible || OB_BACKUP_COMPATIBLE_VERSION_V4 == compatible) {
       if (OB_FAIL(get_base_meta_restore_reader_v2(arg.table_key_, reader))) {
         STORAGE_LOG(WARN, "fail to get_base_meta_restore_reader_v2", K(ret));
       }
@@ -11064,7 +11064,7 @@ int ObMigratePrepareTask::get_partition_table_info_reader(
       if (OB_FAIL(inner_get_partition_table_info_restore_reader_v1(src_info, reader))) {
         LOG_WARN("failed to get partition table info restore reader", K(ret));
       }
-    } else if (OB_BACKUP_COMPATIBLE_VERSION_V3 == compatible) {
+    } else if (OB_BACKUP_COMPATIBLE_VERSION_V3 == compatible || OB_BACKUP_COMPATIBLE_VERSION_V4 == compatible) {
       if (OB_FAIL(inner_get_partition_table_info_restore_reader_v2(src_info, reader))) {
         LOG_WARN("failed to get partition table info restore reader", K(ret));
       }