fix idempotence of offline ddl when drop tenant simulatenously.

src/rootserver/ddl_task/ob_constraint_task.cpp

@@ -652,7 +652,7 @@ int ObConstraintTask::release_snapshot(const int64_t snapshot_version)
   } else if (OB_ISNULL(table_schema)) {
     LOG_INFO("table not exist", K(ret), K(object_id_), K(target_object_id_));
   } else if (OB_FAIL(ObDDLUtil::get_tablets(tenant_id_, object_id_, tablet_ids))) {
-    if (OB_TABLE_NOT_EXIST == ret) {
+    if (OB_TABLE_NOT_EXIST == ret || OB_TENANT_NOT_EXIST == ret) {
       ret = OB_SUCCESS;
     } else {
       LOG_WARN("failed to get tablet snapshots", K(ret));

src/rootserver/ddl_task/ob_ddl_task.cpp

@@ -292,6 +292,7 @@ int ObDDLTask::switch_status(ObDDLTaskStatus new_status, const int ret_code)
   int tmp_ret = OB_SUCCESS;
   bool is_cancel = false;
   int real_ret_code = ret_code;
+  bool is_tenant_dropped = false;
   const ObDDLTaskStatus old_status = task_status_;
   if (OB_TMP_FAIL(SYS_TASK_STATUS_MGR.is_task_cancel(trace_id_, is_cancel))) {
     LOG_WARN("check task is canceled", K(tmp_ret), K(trace_id_));
@@ -310,6 +311,11 @@ int ObDDLTask::switch_status(ObDDLTaskStatus new_status, const int ret_code)
   if (OB_ISNULL(root_service = GCTX.root_service_)) {
     ret = OB_ERR_UNEXPECTED;
     LOG_WARN("error unexpected, root service must not be nullptr", K(ret));
+  } else if (OB_FAIL(root_service->get_schema_service().check_if_tenant_has_been_dropped(tenant_id_, is_tenant_dropped))) {
+    LOG_WARN("check if tenant has been dropped failed", K(ret), K(tenant_id_));
+  } else if (is_tenant_dropped) {
+    need_retry_ = false;
+    LOG_INFO("tenant has been dropped, exit anyway", K(ret), K(tenant_id_));
   } else if (OB_FAIL(trans.start(&root_service->get_sql_proxy(), tenant_id_))) {
     LOG_WARN("start transaction failed", K(ret));
   } else {

src/rootserver/ddl_task/ob_index_build_task.cpp

@@ -581,13 +581,13 @@ int ObIndexBuildTask::release_snapshot(const int64_t snapshot)
     const ObTableSchema *data_table_schema = nullptr;
     ObMultiVersionSchemaService &schema_service = ObMultiVersionSchemaService::get_instance();
     if (OB_FAIL(ObDDLUtil::get_tablets(tenant_id_, object_id_, tablet_ids))) {
-      if (OB_TABLE_NOT_EXIST == ret) {
+      if (OB_TABLE_NOT_EXIST == ret || OB_TENANT_NOT_EXIST == ret) {
         ret = OB_SUCCESS;
       } else {
         LOG_WARN("failed to get data table snapshot", K(ret));
       }
     } else if (OB_FAIL(ObDDLUtil::get_tablets(tenant_id_, target_object_id_, tablet_ids))) {
-      if (OB_TABLE_NOT_EXIST == ret) {
+      if (OB_TABLE_NOT_EXIST == ret || OB_TENANT_NOT_EXIST == ret) {
         ret = OB_SUCCESS;
       } else {
         LOG_WARN("failed to get dest table snapshot", K(ret));

src/sql/engine/cmd/ob_ddl_executor_util.cpp

@@ -63,6 +63,7 @@ int ObDDLExecutorUtil::wait_ddl_finish(
     LOG_WARN("invalid argument", K(ret), K(tenant_id), K(task_id), KP(common_rpc_proxy));
   } else {
     int tmp_ret = OB_SUCCESS;
+    bool is_tenant_dropped = false;
     while (OB_SUCC(ret)) {
       if (OB_SUCCESS == ObDDLErrorMessageTableOperator::get_ddl_error_message(
           tenant_id, task_id, -1 /* target_object_id */, unused_addr, false /* is_ddl_retry_task */, *GCTX.sql_proxy_, error_message, unused_user_msg_len)) {
@@ -71,6 +72,13 @@ int ObDDLExecutorUtil::wait_ddl_finish(
           FORWARD_USER_ERROR(ret, error_message.user_message_);
         }
         break;
+      } else if (OB_TMP_FAIL(GSCHEMASERVICE.check_if_tenant_has_been_dropped(
+                               tenant_id, is_tenant_dropped))) {
+        LOG_WARN("check if tenant has been dropped failed", K(tmp_ret), K(tenant_id));
+      } else if (is_tenant_dropped) {
+        ret = OB_TENANT_HAS_BEEN_DROPPED;
+        LOG_WARN("tenant has been dropped", K(ret), K(tenant_id));
+        break;
       } else if (OB_FAIL(handle_session_exception(session))) {
         LOG_WARN("session exeception happened", K(ret), K(is_support_cancel));
         if (is_support_cancel && OB_TMP_FAIL(cancel_ddl_task(tenant_id, common_rpc_proxy))) {
@@ -90,6 +98,8 @@ int ObDDLExecutorUtil::wait_ddl_finish(
 int ObDDLExecutorUtil::wait_build_index_finish(const uint64_t tenant_id, const int64_t task_id, bool &is_finish)
 {
   int ret = OB_SUCCESS;
+  int tmp_ret = OB_SUCCESS;
+  bool is_tenant_dropped = false;
   ObAddr unused_addr;
   int64_t unused_user_msg_len = 0;
   THIS_WORKER.set_timeout_ts(ObTimeUtility::current_time() + OB_MAX_USER_SPECIFIED_TIMEOUT);
@@ -106,6 +116,12 @@ int ObDDLExecutorUtil::wait_build_index_finish(const uint64_t tenant_id, const i
       FORWARD_USER_ERROR(ret, error_message.user_message_);
     }
     is_finish = true;
+  } else if (OB_TMP_FAIL(GSCHEMASERVICE.check_if_tenant_has_been_dropped(
+                 tenant_id, is_tenant_dropped))) {
+    LOG_WARN("check if tenant has been dropped failed", K(tmp_ret), K(tenant_id));
+  } else if (is_tenant_dropped) {
+    ret = OB_TENANT_HAS_BEEN_DROPPED;
+    LOG_WARN("tenant has been dropped", K(ret), K(tenant_id));
   }
   return ret;
 }
@@ -129,6 +145,7 @@ int ObDDLExecutorUtil::wait_ddl_retry_task_finish(
     ret = OB_INVALID_ARGUMENT;
     LOG_WARN("invalid argument", K(ret), K(tenant_id), K(task_id), KP(common_rpc_proxy));
   } else {
+    bool is_tenant_dropped = false;
     int tmp_ret = OB_SUCCESS;
     while (OB_SUCC(ret)) {
       if (OB_SUCCESS == ObDDLErrorMessageTableOperator::get_ddl_error_message(
@@ -165,6 +182,13 @@ int ObDDLExecutorUtil::wait_ddl_retry_task_finish(
           }
         }
         break;
+      } else if (OB_TMP_FAIL(GSCHEMASERVICE.check_if_tenant_has_been_dropped(
+                    tenant_id, is_tenant_dropped))) {
+        LOG_WARN("check if tenant has been dropped failed", K(tmp_ret), K(tenant_id));
+      } else if (is_tenant_dropped) {
+        ret = OB_TENANT_HAS_BEEN_DROPPED;
+        LOG_WARN("tenant has been dropped", K(ret), K(tenant_id));
+        break;
       } else if (OB_FAIL(handle_session_exception(session))) {
         LOG_WARN("session exeception happened", K(ret));
         if (OB_TMP_FAIL(cancel_ddl_task(tenant_id, common_rpc_proxy))) {

src/sql/engine/cmd/ob_index_executor.cpp

@@ -291,6 +291,8 @@ int ObDropIndexExecutor::wait_drop_index_finish(
     int64_t unused_user_msg_len = 0;
     const int64_t retry_interval = 100 * 1000;
     while (OB_SUCC(ret)) {
+      int tmp_ret = OB_SUCCESS;
+      bool is_tenant_dropped = false;
       if (OB_SUCCESS == share::ObDDLErrorMessageTableOperator::get_ddl_error_message(
           tenant_id, task_id, -1 /* target_object_id */, unused_addr, false /* is_ddl_retry_task */, *GCTX.sql_proxy_, error_message, unused_user_msg_len)) {
         ret = error_message.ret_code_;
@@ -298,6 +300,13 @@ int ObDropIndexExecutor::wait_drop_index_finish(
           FORWARD_USER_ERROR(ret, error_message.user_message_);
         }
         break;
+      } else if (OB_TMP_FAIL(GSCHEMASERVICE.check_if_tenant_has_been_dropped(
+                               tenant_id, is_tenant_dropped))) {
+        LOG_WARN("check if tenant has been dropped failed", K(tmp_ret), K(tenant_id));
+      } else if (is_tenant_dropped) {
+        ret = OB_TENANT_HAS_BEEN_DROPPED;
+        LOG_WARN("tenant has been dropped", K(ret), K(tenant_id));
+        break;
       } else if (OB_FAIL(session.check_session_status())) {
         LOG_WARN("session exeception happened", K(ret));
       } else {

src/storage/ddl/ob_ddl_redo_log_writer.cpp

@@ -669,7 +669,7 @@ int ObDDLRedoLogWriter::write_ddl_start_log(ObDDLKvMgrHandle &ddl_kv_mgr_handle,
       if (tmp_cb->is_success()) {
         finish = true;
       } else if (tmp_cb->is_failed()) {
-        ret = OB_ERR_SYS;
+        ret = OB_NOT_MASTER;
       }
       if (OB_SUCC(ret) && !finish) {
         const int64_t current_time = ObTimeUtility::current_time();
@@ -810,7 +810,7 @@ int ObDDLRedoLogHandle::wait(const int64_t timeout)
       if (cb_->is_success()) {
         finish = true;
       } else if (cb_->is_failed()) {
-        ret = OB_ERR_SYS;
+        ret = OB_NOT_MASTER;
       }
       if (OB_SUCC(ret) && !finish) {
         const int64_t current_time = ObTimeUtility::current_time();
@@ -847,7 +847,7 @@ int ObDDLCommitLogHandle::wait(const int64_t timeout)
       if (cb_->is_success()) {
         finish = true;
       } else if (cb_->is_failed()) {
-        ret = OB_ERR_SYS;
+        ret = OB_NOT_MASTER;
       }
       if (OB_SUCC(ret) && !finish) {
         const int64_t current_time = ObTimeUtility::current_time();

src/storage/tablet/ob_tablet.cpp

@@ -2308,7 +2308,7 @@ int ObTablet::write_tablet_schema_version_change_clog(
         ret = OB_TIMEOUT;
         LOG_WARN("submit schema version change log timeout", K(ret));
       } else if (tmp_cb->is_failed()) {
-        ret = OB_ERR_UNEXPECTED;
+        ret = OB_NOT_MASTER;
         LOG_WARN("submit schema version change log failed", K(ret));
       } else {
         LOG_INFO("submit schema version change log succeed", K(tablet_id), K(schema_version));