fix deadlock between ls_lock and ls_restore_handler_lock

src/share/restore/ob_ls_restore_status.h

@@ -88,8 +88,8 @@ public:
     return ((status_ >= Status::RESTORE_START && status_ <= Status::RESTORE_SYS_TABLETS) ||
              status_ == Status::RESTORE_FAILED);
   }
-  // if restore status is not in [RESTORE_START, WAIT_RESTORE_TABLETS_META], log_replay_service can replay log.
-  bool can_replay_log() const { return ! (status_ > Status::RESTORE_NONE && status_ < Status::QUICK_RESTORE); }
+  // if restore status is not in [RESTORE_START, RESTORE_SYS_TABLETS], log_replay_service can replay log.
+  bool can_replay_log() const { return ! (status_ >= Status::RESTORE_START && status_ <= Status::RESTORE_SYS_TABLETS); }
   bool can_restore_log() const { return ! (status_ > Status::RESTORE_NONE && status_ < Status::QUICK_RESTORE); }
   Status get_status() const { return status_; }
   int set_status(int32_t status);

src/storage/restore/ob_ls_restore_handler.cpp

@@ -98,13 +98,22 @@ int ObLSRestoreHandler::offline()
   } else if (!is_online_) {
     LOG_INFO("ls restore handler is already offline");
   } else {
-    lib::ObMutexGuard guard(mtx_);
-    if (OB_FAIL(cancel_task_())) {
-      LOG_WARN("failed to cancel task", K(ret), KPC(ls_));
-    } else {
-      is_online_ = false;
-      LOG_INFO("ls restore handler offline finish");
-    }
+    int retry_cnt = 0;
+    do {
+      // if lock failed, retry 3 times.
+      if (OB_FAIL(mtx_.trylock())) {
+        LOG_WARN("lock restore handler failed, retry later", K(ret), KPC(ls_));
+        sleep(1);
+      } else {
+        if (OB_FAIL(cancel_task_())) {
+          LOG_WARN("failed to cancel task", K(ret), KPC(ls_));
+        } else {
+          is_online_ = false;
+          LOG_INFO("ls restore handler offline finish");
+        }
+        mtx_.unlock();
+      }
+    } while (retry_cnt ++ < 3/*max retry cnt*/ && OB_EAGAIN == ret);
   }
   return ret;
 }