the backup data cannot be cleaned up when the disk space is full

src/rootserver/backup/ob_tenant_backup_data_clean_mgr.cpp

@@ -892,7 +892,7 @@ int ObTenantBackupBaseDataCleanTask::clean_backup_data(const ObSimpleBackupDataC
   int ret = OB_SUCCESS;
   ObArray<ObExternBackupInfo> extern_backup_infos;
   storage::ObPhyRestoreMetaIndexStore meta_index;
-
+  uint64_t tenant_id = simple_clean_tenant.tenant_id_;
   if (!is_inited_) {
     ret = OB_NOT_INIT;
     LOG_WARN("tenant backup base data clean task do not init", K(ret));
@@ -900,15 +900,15 @@ int ObTenantBackupBaseDataCleanTask::clean_backup_data(const ObSimpleBackupDataC
     LOG_WARN("failed to get tenant backup infos", K(ret), K(backup_set_id), K(simple_clean_tenant), K(clean_element));
   } else {
     LOG_INFO("get extern backup infos", K(extern_backup_infos), K(backup_set_id));
-
     for (int64_t i = 0; OB_SUCC(ret) && i < extern_backup_infos.count(); ++i) {
       const ObExternBackupInfo &backup_info = extern_backup_infos.at(i);
       if (OB_FAIL(data_clean_->check_can_do_task())) {
         LOG_WARN("failed to check can do task", K(ret));
+      } else if (OB_FAIL(try_clean_backup_set_info(tenant_id, clean_element, backup_set_id, backup_info))) {
+        LOG_WARN("failed to try clean backup set info", K(ret), K(tenant_id), K(backup_set_id));
       } else if (OB_FAIL(clean_backp_set(simple_clean_tenant, clean_element, backup_set_id, backup_info))) {
         LOG_WARN("failed to clean backup set", K(ret), K(simple_clean_tenant), K(clean_element), K(backup_info));
-      } else if (OB_FAIL(try_clean_backup_set_dir(
-                     simple_clean_tenant.tenant_id_, clean_element, backup_set_id, backup_info))) {
+      } else if (OB_FAIL(try_clean_backup_set_dir(tenant_id, clean_element, backup_set_id, backup_info))) {
         LOG_WARN("failed to try clean backup set dir", K(ret), K(backup_set_id), K(simple_clean_tenant));
       }
     }
@@ -1125,8 +1125,6 @@ int ObTenantBackupBaseDataCleanTask::try_clean_backup_set_dir(const uint64_t ten
   } else if (!backup_set_id.is_valid()) {
     ret = OB_INVALID_ARGUMENT;
     LOG_WARN("try clean backup set dir get invalid argument", K(ret), K(backup_set_id));
-  } else if (OB_FAIL(try_clean_backup_set_info(tenant_id, clean_element, backup_set_id, extern_backup_info))) {
-    LOG_WARN("failed to try clean backup set info", K(ret), K(tenant_id), K(backup_set_id));
   } else if (OB_FAIL(try_clean_backup_set_data_dir(tenant_id, clean_element, backup_set_id, extern_backup_info))) {
     LOG_WARN("failed to try clean backup set data dir", K(ret), K(tenant_id), K(backup_set_id));
   } else if (OB_FAIL(try_clean_full_backup_set_dir(tenant_id, clean_element, backup_set_id, extern_backup_info))) {
@@ -1193,15 +1191,11 @@ int ObTenantBackupBaseDataCleanTask::try_clean_backup_set_info(const uint64_t te
   if (!is_inited_) {
     ret = OB_NOT_INIT;
     LOG_WARN("tenant backup base data clean task do not init", K(ret));
+  } else if (!backup_set_id.is_valid()) {
+    ret = OB_INVALID_ARGUMENT;
+    LOG_WARN("try clean backup set info get invalid argument", K(ret), K(backup_set_id));
   } else if (OB_FAIL(backup_dest.set(clean_element.backup_dest_, clean_element.incarnation_))) {
     LOG_WARN("failed to set backup dest", K(ret), K(clean_element));
-  } else if (OB_FAIL(path_info.set(
-                 backup_dest, tenant_id, full_backup_set_id, inc_backup_set_id, backup_date, compatible))) {
-    LOG_WARN("failed to set backup path info", K(ret), K(backup_dest), K(tenant_id), K(extern_backup_info));
-  } else if (OB_FAIL(ObBackupPathUtil::get_tenant_data_full_backup_set_path(path_info, path))) {
-    LOG_WARN("failed to get tenant date full backup set path", K(ret), K(path_info));
-  } else if (OB_FAIL(ObBackupDataCleanUtil::delete_tmp_files(path, backup_dest.get_storage_info()))) {
-    LOG_WARN("failed to delete tmp files", K(ret), K(path), K(backup_dest));
   } else if (OB_FAIL(extern_backup_set_info_mgr.init(tenant_id,
                  full_backup_set_id,
                  inc_backup_set_id,
@@ -1221,25 +1215,36 @@ int ObTenantBackupBaseDataCleanTask::try_clean_backup_set_info(const uint64_t te
   } else if (ObBackupDataCleanMode::TOUCH == backup_set_id.clean_mode_) {
     if (clean_element.backup_dest_option_.auto_delete_obsolete_backup_) {
       // do nothing
-    } else if (OB_FAIL(extern_backup_set_info_mgr.touch_extern_backup_set_info())) {
-      LOG_WARN("failed to touch backup file", K(ret), K(backup_dest));
     } else if (compatible > ObBackupCompatibleVersion::OB_BACKUP_COMPATIBLE_VERSION_V1 &&
                OB_FAIL(extern_single_backup_set_info_mgr.touch_extern_backup_set_file_info())) {
       LOG_WARN("failed to touch extern backup set file info", K(ret), K(backup_dest));
+    } else if (OB_FAIL(extern_backup_set_info_mgr.touch_extern_backup_set_info())) {
+      LOG_WARN("failed to touch backup file", K(ret), K(backup_dest));
     }
   } else if (ObBackupDataCleanMode::CLEAN == backup_set_id.clean_mode_) {
     if (clean_element.backup_dest_option_.auto_touch_reserved_backup_) {
       // do nothing
-    } else if (OB_FAIL(extern_backup_set_info_mgr.delete_extern_backup_set_info())) {
-      LOG_WARN("failed to delete backup file", K(ret), K(backup_dest));
     } else if (compatible > ObBackupCompatibleVersion::OB_BACKUP_COMPATIBLE_VERSION_V1 &&
                OB_FAIL(extern_single_backup_set_info_mgr.delete_extern_backup_set_file_info())) {
       LOG_WARN("failed to delete extern backup set file info", K(ret), K(backup_dest));
+    } else if (OB_FAIL(extern_backup_set_info_mgr.delete_extern_backup_set_info())) {
+      LOG_WARN("failed to delete backup file", K(ret), K(backup_dest));
     }
   } else {
     ret = OB_NOT_SUPPORTED;
     LOG_WARN("try clean backup set info's clean mode is not supported", K(ret), K(backup_set_id));
   }
+
+  if (OB_FAIL(ret)) {
+  } else if (OB_FAIL(path_info.set(
+                 backup_dest, tenant_id, full_backup_set_id, inc_backup_set_id, backup_date, compatible))) {
+    LOG_WARN("failed to set backup path info", K(ret), K(backup_dest), K(tenant_id), K(extern_backup_info));
+  } else if (OB_FAIL(ObBackupPathUtil::get_tenant_data_full_backup_set_path(path_info, path))) {
+    LOG_WARN("failed to get tenant date full backup set path", K(ret), K(path_info));
+  } else if (OB_FAIL(ObBackupDataCleanUtil::delete_tmp_files(path, backup_dest.get_storage_info()))) {
+    LOG_WARN("failed to delete tmp files", K(ret), K(path), K(backup_dest));
+  }
+
   return ret;
 }
 
@@ -2381,14 +2386,14 @@ int ObTenantBackupClogDataCleanTask::handle_backup_clog_piece_infos(const uint64
     ret = OB_INVALID_ARGUMENT;
     LOG_WARN(
         "handle archive key get invalid argument", K(ret), K(tenant_id), K(backup_piece_info), K(delete_clog_mode));
+  } else if (OB_FAIL(handle_single_piece_info(tenant_id, cluster_backup_dest, backup_piece_info, delete_clog_mode))) {
+    LOG_WARN("failed to handle single piece info", K(ret), K(cluster_backup_dest));
   } else if (OB_FAIL(handle_data_and_index_dir(
                  tenant_id, cluster_backup_dest, backup_piece_info, delete_clog_mode, pg_keys))) {
     LOG_WARN("failed to handle data and index dir", K(ret), K(cluster_backup_dest), K(backup_piece_info));
   } else if (OB_FAIL(
                  handle_archive_key(tenant_id, cluster_backup_dest, backup_piece_info, delete_clog_mode, pg_keys))) {
     LOG_WARN("failed to handle archive key", K(ret), K(cluster_backup_dest), K(backup_piece_info));
-  } else if (OB_FAIL(handle_single_piece_info(tenant_id, cluster_backup_dest, backup_piece_info, delete_clog_mode))) {
-    LOG_WARN("failed to handle single piece info", K(ret), K(cluster_backup_dest));
   } else if (OB_FAIL(handle_backup_piece_dir(tenant_id, cluster_backup_dest, backup_piece_info, delete_clog_mode))) {
     LOG_WARN("failed to handle backup piece dir", K(ret), K(cluster_backup_dest));
   }

src/rootserver/ob_backup_data_clean.cpp

@@ -2587,8 +2587,6 @@ int ObBackupDataClean::mark_backup_set_info_deleting(
     LOG_WARN("failed to get need delete backup set ids", K(ret), K(clean_info));
   } else if (OB_FAIL(mark_backup_set_info_inner_table_deleting(clean_info, clean_element, backup_set_ids))) {
     LOG_WARN("failed to mark backup set info inner table deleted", K(ret), K(clean_info));
-  } else if (OB_FAIL(mark_extern_backup_set_info_deleting(clean_info, clean_element, backup_set_ids))) {
-    LOG_WARN("failed to mark extern backup set info deleted", K(ret), K(clean_info));
   }
   return ret;
 }
@@ -2847,9 +2845,6 @@ int ObBackupDataClean::mark_log_archive_info_deleting(
     LOG_WARN("failed to get need delete clog round and piece", K(ret), K(clean_info));
   } else if (OB_FAIL(mark_log_archive_info_inner_table_deleting(clean_info, backup_piece_keys, log_archive_rounds))) {
     LOG_WARN("failed to mark backup set info inner table deleted", K(clean_info));
-  } else if (OB_FAIL(mark_extern_log_archive_info_deleting(
-                 clean_info, clean_element, backup_piece_keys, log_archive_rounds))) {
-    LOG_WARN("failed to mark extern backup set info deleted", K(ret), K(clean_info));
   }
 
   return ret;
@@ -3319,6 +3314,11 @@ int ObBackupDataClean::delete_backup_extern_infos(
         LOG_WARN("failed to get need delete clog round and piece", K(ret), K(clean_info));
       } else if (OB_FAIL(delete_extern_tmp_files(clean_info, clean_element))) {
         LOG_WARN("failed to delete extern tmp files", K(ret), K(clean_info));
+      } else if (OB_FAIL(mark_extern_backup_set_info_deleting(clean_info, clean_element, backup_set_ids))) {
+        LOG_WARN("failed to mark extern backup set info deleted", K(ret), K(clean_info));
+      } else if (OB_FAIL(mark_extern_log_archive_info_deleting(
+                     clean_info, clean_element, backup_piece_keys, log_archive_rounds))) {
+        LOG_WARN("failed to mark extern backup set info deleted", K(ret), K(clean_info));
       } else if (OB_FAIL(delete_extern_backup_info_deleted(clean_info, clean_element, backup_set_ids))) {
         LOG_WARN("failed to makr extern backup info deleted", K(ret), K(clean_info), K(clean_element));
       } else if (OB_FAIL(delete_extern_clog_info_deleted(clean_info, clean_element, log_archive_rounds))) {

src/rootserver/restore/ob_restore_util.cpp

@@ -604,6 +604,12 @@ int ObRestoreUtil::fill_multi_backup_path(
     LOG_WARN("failed to filter duplicate path list", KR(ret));
   } else if (OB_FAIL(job.multi_restore_path_list_.set(backup_set_list, backup_piece_list))) {
     LOG_WARN("failed to set mutli restore path list", KR(ret));
+  } else if (OB_FAIL(ObMultiBackupDestUtil::check_can_restore_by_set_or_piece(backup_set_list, backup_piece_list))) {
+    if (OB_BACKUP_FILE_NOT_EXIST == ret) {
+      LOG_WARN("the backup file is missing and cannot be restored", K(ret));
+    } else {
+      LOG_WARN("failed to check can be restored", KR(ret));
+    }
   } else if (OB_FAIL(
                  get_multi_path_file_info_list(backup_set_list, backup_piece_list, set_info_list, piece_info_list))) {
     LOG_WARN("failed to get multi path file info list", KR(ret), K(backup_set_list), K(backup_piece_list));
@@ -685,6 +691,12 @@ int ObRestoreUtil::inner_fill_compat_backup_path(
     LOG_WARN("failed to get backup path list", KR(ret), K(uri_list));
   } else if (OB_FAIL(job.multi_restore_path_list_.set(backup_set_list, backup_piece_list))) {
     LOG_WARN("failed to set mutli restore path list", KR(ret));
+  } else if (OB_FAIL(ObMultiBackupDestUtil::check_can_restore_by_set_or_piece(backup_set_list, backup_piece_list))) {
+    if (OB_BACKUP_FILE_NOT_EXIST == ret) {
+      LOG_WARN("the backup file is missing and cannot be restored", K(ret));
+    } else {
+      LOG_WARN("failed to check can be restored", KR(ret));
+    }
   }
   return ret;
 }

src/share/backup/ob_multi_backup_dest_util.cpp

@@ -481,6 +481,45 @@ int ObMultiBackupDestUtil::get_path_type(
   return ret;
 }
 
+int ObMultiBackupDestUtil::check_can_restore_by_set_or_piece(
+    const common::ObArray<share::ObSimpleBackupSetPath> &backup_set_list,
+    const common::ObArray<share::ObSimpleBackupPiecePath> &backup_piece_list)
+{
+  int ret = OB_SUCCESS;
+  ObBackupPath backup_set_path, backup_piece_path;
+  ObStorageUtil util(false /*need_retry*/);
+
+  for (int64_t i = 0; OB_SUCC(ret) && i < backup_set_list.count(); ++i) {
+    const ObSimpleBackupSetPath &simple_path = backup_set_list.at(i);
+    bool set_exist = false;
+    if (OB_FAIL(get_backup_set_info_path(simple_path.get_simple_path(), backup_set_path))) {
+      LOG_WARN("failed to get backup set info path", KR(ret), K_(simple_path.backup_dest));
+    } else if (OB_FAIL(util.is_exist(
+                   backup_set_path.get_obstr(), simple_path.get_storage_info(), set_exist))) {
+      LOG_WARN("failed to check set file exist", KR(ret), K(backup_set_path));
+    } else if (!set_exist) {
+      ret = OB_BACKUP_FILE_NOT_EXIST;
+      LOG_WARN("single_backup_set_info is not exist", KR(ret), K_(simple_path.backup_set_id), K(backup_set_path));
+    }
+  }
+
+  for (int64_t i = 0; OB_SUCC(ret) && i < backup_piece_list.count(); ++i) {
+    const ObSimpleBackupPiecePath &simple_path = backup_piece_list.at(i);
+    bool piece_exist = false;
+    if (OB_FAIL(get_backup_piece_info_path(simple_path.get_simple_path(), backup_piece_path))) {
+      LOG_WARN("failed to get backup piece info path", KR(ret), K_(simple_path.backup_dest));
+    } else if (OB_FAIL(util.is_exist(
+                   backup_piece_path.get_obstr(), simple_path.get_storage_info(), piece_exist))) {
+      LOG_WARN("failed to check piece file exist", KR(ret), K(backup_piece_path));
+    } else if (!piece_exist) {
+      ret = OB_BACKUP_FILE_NOT_EXIST;
+      LOG_WARN("single_piece_info is not exist", KR(ret), K_(simple_path.backup_piece_id), K(backup_piece_path));
+    }
+  }
+
+  return ret;
+}
+
 int ObMultiBackupDestUtil::inner_get_backup_tenant_id_from_set_or_piece(
     const common::ObArray<common::ObString> &path_list, const int64_t restore_timestamp, uint64_t &tenant_id)
 {

src/share/backup/ob_multi_backup_dest_util.h

@@ -62,6 +62,8 @@ public:
   static int check_multi_path_is_complete(const int64_t restore_timestamp,
       common::ObArray<share::ObBackupSetFileInfo> &set_info_list,
       common::ObArray<share::ObBackupPieceInfo> &piece_info_list, bool &is_complete);
+  static int check_can_restore_by_set_or_piece(const common::ObArray<share::ObSimpleBackupSetPath> &backup_set_list,
+      const common::ObArray<share::ObSimpleBackupPiecePath> &backup_piece_list);
 
 private:
   static int check_is_compat_backup_path(const char *cluster_name, const int64_t cluster_id,