diff --git a/deps/oblib/src/lib/utility/ob_tracepoint.h b/deps/oblib/src/lib/utility/ob_tracepoint.h index 922126d483cc8e7128b23415bc4eca3f5d6286b1..861add148783675cf724c3dbc22889090db9a037 100644 --- a/deps/oblib/src/lib/utility/ob_tracepoint.h +++ b/deps/oblib/src/lib/utility/ob_tracepoint.h @@ -446,6 +446,7 @@ public: EN_BACKUP_BACKUPSET_DO_SCHEDULE = 184, EN_BACKUP_CHECK_BACKUP_POINT_EXIST = 185, EN_STOP_TENANT_LOG_ARCHIVE_BACKUP = 186, + EN_BACKUP_SERVER_DISK_IS_FULL = 188, // 下面请从201开始 EN_CHECK_STANDBY_CLUSTER_SCHEMA_CONDITION = 201, EN_ALLOCATE_LOB_BUF_FAILED = 202, diff --git a/src/rootserver/ob_root_backup.cpp b/src/rootserver/ob_root_backup.cpp index 2e406994816872d1d61d35623a75803fefa67181..ef415bef9d295f51fa9cadfe66eb9ea17db847ca 100644 --- a/src/rootserver/ob_root_backup.cpp +++ b/src/rootserver/ob_root_backup.cpp @@ -671,6 +671,8 @@ int ObRootBackup::do_sys_tenant_backup( } else { if (OB_FAIL(add_backup_info_lock(info, updater, info_manager))) { LOG_WARN("failed to add backup info lock", K(ret), K(info)); + } else if (OB_FAIL(check_server_disk_stat(info, updater.get_trans()))) { + LOG_WARN("failed to check server disk stat", K(ret), K(info)); } else if (OB_FAIL(check_tenants_backup_task_failed(info, info_manager, updater.get_trans()))) { LOG_WARN("failed to check tenants backup task failed", K(ret), K(info)); } else if (OB_FAIL(do_with_all_finished_info(info, updater, info_manager))) { @@ -3123,6 +3125,80 @@ int ObRootBackup::do_tenant_update_task_his_and_backup_set_file( return ret; } +int ObRootBackup::check_server_disk_stat(const ObBaseBackupInfoStruct &info, common::ObISQLClient &sys_tenant_trans) +{ + int ret = OB_SUCCESS; + ObTenantBackupTaskInfo sys_task_info; + ObTenantBackupTaskInfo sys_dest_task_info; + const ObZone zone; + ObArray server_status_array; + const int64_t NO_LIMIT_PERCENT = 100; + + if (!is_inited_) { + ret = OB_NOT_INIT; + LOG_WARN("root backup do not init", K(ret)); + } else if (!info.is_valid()) { + ret = OB_INVALID_ARGUMENT; + LOG_WARN("check server disk stat get invalid argument", K(ret), K(info)); + } else if (OB_FAIL(get_tenant_backup_task( + info.tenant_id_, info.backup_set_id_, info.incarnation_, sys_tenant_trans, sys_task_info))) { + LOG_WARN("failed to get tenant backup task", K(ret), K(info)); + } else if (!sys_task_info.is_result_succeed()) { + // do nothing + } else if (OB_FAIL(server_mgr_->get_server_statuses(zone, server_status_array))) { + LOG_WARN("failed to get server statuses", K(ret)); + } else { + for (int64_t i = 0; OB_SUCC(ret) && i < server_status_array.count(); ++i) { + const ObServerStatus &status = server_status_array.at(i); + if (!status.is_active() || !status.in_service()) { + // do nothing + } else if (0 == status.resource_info_.disk_total_) { + } else { + const int64_t data_disk_usage_limit_percentage = GCONF.data_disk_usage_limit_percentage; + int64_t used_percent = (100 * status.resource_info_.disk_in_use_) / status.resource_info_.disk_total_; + +#ifdef ERRSIM + if (OB_SUCC(ret)) { + ret = E(EventTable::EN_BACKUP_SERVER_DISK_IS_FULL) OB_SUCCESS; + if (OB_FAIL(ret)) { + STORAGE_LOG(ERROR, "fake EN_BACKUP_SERVER_DISK_IS_FULL", K(ret)); + used_percent = 100; + ret = OB_SUCCESS; + } + } +#endif + + if (data_disk_usage_limit_percentage != NO_LIMIT_PERCENT && used_percent >= data_disk_usage_limit_percentage) { + FLOG_INFO("server disk is almost full", K(ret), K(used_percent), K(status)); + sys_dest_task_info = sys_task_info; + sys_dest_task_info.result_ = OB_CS_OUTOF_DISK_SPACE; + if (OB_FAIL(update_tenant_backup_task(sys_tenant_trans, sys_task_info, sys_dest_task_info))) { + LOG_WARN("failed to update tenant backup task", K(ret), K(sys_task_info), K(sys_dest_task_info)); + } else { + ROOTSERVICE_EVENT_ADD("backup", + "backup server disk is almost full", + "server", + status.server_, + "disk_in_use", + status.resource_info_.disk_in_use_, + "disk_total", + status.resource_info_.disk_total_, + "result", + OB_CS_OUTOF_DISK_SPACE); + break; + } + } + } + } + } + return ret; +} + +bool ObRootBackup::is_force_cancel_() const +{ + return GCONF.backup_dest.get_value_string().empty(); +} + ObTenantBackup::ObTenantBackup() : is_inited_(false), schema_service_(NULL), diff --git a/src/rootserver/ob_root_backup.h b/src/rootserver/ob_root_backup.h index 7cb14325768b829e72103c327325e4ecadeb3687..7627b15ad65327ccd7489c779c36dd479cafc5f0 100644 --- a/src/rootserver/ob_root_backup.h +++ b/src/rootserver/ob_root_backup.h @@ -253,6 +253,10 @@ private: int insert_tenant_backup_set_file_failed(ObMySQLTransaction& trans, const share::ObTenantBackupTaskInfo& task_info); int do_tenant_update_task_his_and_backup_set_file( const share::ObTenantBackupTaskInfo& task_info, const ObBackupSetFileInfo& backup_set_file_info); + int check_server_disk_stat(const ObBaseBackupInfoStruct &info, common::ObISQLClient &sys_tenant_trans); + +private: + bool is_force_cancel_() const; private: bool is_inited_;